chg: [add Hosts module]

bin/modules/DomClassifier.py

@@ -53,37 +53,38 @@ class DomClassifier(AbstractModule):
 
 
     def compute(self, message, r_result=False):
-        item = Item(message)
+        host, id = message.split()
 
-        item_content = item.get_content()
+        item = Item(id)
         item_basename = item.get_basename()
         item_date = item.get_date()
         item_source = item.get_source()
         try:
-            mimetype = item_basic.get_item_mimetype(item.get_id())
 
-            if mimetype.split('/')[0] == "text":
-                self.c.text(rawtext=item_content)
-                self.c.potentialdomain()
-                self.c.validdomain(passive_dns=True, extended=False)
-                #self.redis_logger.debug(self.c.vdomain)
+            self.c.text(rawtext=host)
+            print(self.c.domain)
+            self.c.validdomain(passive_dns=True, extended=False)
+            #self.redis_logger.debug(self.c.vdomain)
 
-                if self.c.vdomain and d4.is_passive_dns_enabled():
-                    for dns_record in self.c.vdomain:
-                        self.send_message_to_queue(dns_record)
+            print(self.c.vdomain)
+            print()
 
-                localizeddomains = self.c.include(expression=self.cc_tld)
-                if localizeddomains:
-                    print(localizeddomains)
-                    self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}")
+            if self.c.vdomain and d4.is_passive_dns_enabled():
+                for dns_record in self.c.vdomain:
+                    self.send_message_to_queue(dns_record)
 
-                localizeddomains = self.c.localizedomain(cc=self.cc)
-                if localizeddomains:
-                    print(localizeddomains)
-                    self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}")
+            localizeddomains = self.c.include(expression=self.cc_tld)
+            if localizeddomains:
+                print(localizeddomains)
+                self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}")
 
-                if r_result:
-                    return self.c.vdomain
+            localizeddomains = self.c.localizedomain(cc=self.cc)
+            if localizeddomains:
+                print(localizeddomains)
+                self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}")
+
+            if r_result:
+                return self.c.vdomain
 
         except IOError as err:
             self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")

bin/packages/modules.cfg

@@ -22,8 +22,12 @@ subscribe = Redis_Duplicate
 [Indexer]
 subscribe = Redis_Global
 
-[DomClassifier]
+[Hosts]
 subscribe = Redis_Global
+publish = Redis_Host
+
+[DomClassifier]
+subscribe = Redis_Host
 publish = Redis_D4_client
 
 [D4_client]

tests/test_modules.py

@@ -59,9 +59,10 @@ class Test_Module_Categ(unittest.TestCase):
 
     def test_module(self):
         item_id = 'tests/2021/01/01/categ.gz'
-        test_categ = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
+        test_categ = ['CreditCards', 'Mail', 'Onion', 'Urls', 'Credential', 'Cve']
 
         result = self.module_obj.compute(item_id, r_result=True)
+        print(result)
         self.assertCountEqual(result, test_categ)
 
 class Test_Module_CreditCards(unittest.TestCase):
@@ -87,8 +88,10 @@ class Test_Module_DomClassifier(unittest.TestCase):
         self.module_obj = DomClassifier()
 
     def test_module(self):
+        test_host = 'foo.be'
         item_id = 'tests/2021/01/01/domain_classifier.gz'
-        result = self.module_obj.compute(item_id, r_result=True)
+        msg = f'{test_host} {item_id}'
+        result = self.module_obj.compute(msg, r_result=True)
         self.assertTrue(len(result))
 
 class Test_Module_Global(unittest.TestCase):