diff --git a/bin/modules/DomClassifier.py b/bin/modules/DomClassifier.py index 240916eb..08f202ab 100755 --- a/bin/modules/DomClassifier.py +++ b/bin/modules/DomClassifier.py @@ -53,37 +53,38 @@ class DomClassifier(AbstractModule): def compute(self, message, r_result=False): - item = Item(message) + host, id = message.split() - item_content = item.get_content() + item = Item(id) item_basename = item.get_basename() item_date = item.get_date() item_source = item.get_source() try: - mimetype = item_basic.get_item_mimetype(item.get_id()) - if mimetype.split('/')[0] == "text": - self.c.text(rawtext=item_content) - self.c.potentialdomain() - self.c.validdomain(passive_dns=True, extended=False) - #self.redis_logger.debug(self.c.vdomain) + self.c.text(rawtext=host) + print(self.c.domain) + self.c.validdomain(passive_dns=True, extended=False) + #self.redis_logger.debug(self.c.vdomain) - if self.c.vdomain and d4.is_passive_dns_enabled(): - for dns_record in self.c.vdomain: - self.send_message_to_queue(dns_record) + print(self.c.vdomain) + print() - localizeddomains = self.c.include(expression=self.cc_tld) - if localizeddomains: - print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}") + if self.c.vdomain and d4.is_passive_dns_enabled(): + for dns_record in self.c.vdomain: + self.send_message_to_queue(dns_record) - localizeddomains = self.c.localizedomain(cc=self.cc) - if localizeddomains: - print(localizeddomains) - self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") + localizeddomains = self.c.include(expression=self.cc_tld) + if localizeddomains: + print(localizeddomains) + self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}") - if r_result: - return self.c.vdomain + localizeddomains = self.c.localizedomain(cc=self.cc) + if localizeddomains: + print(localizeddomains) + self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}") + + if r_result: + return self.c.vdomain except IOError as err: self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed") diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 4cc5ba6a..4da8fd2e 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -22,8 +22,12 @@ subscribe = Redis_Duplicate [Indexer] subscribe = Redis_Global -[DomClassifier] +[Hosts] subscribe = Redis_Global +publish = Redis_Host + +[DomClassifier] +subscribe = Redis_Host publish = Redis_D4_client [D4_client] diff --git a/tests/test_modules.py b/tests/test_modules.py index 728294fe..e65edcc7 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -59,9 +59,10 @@ class Test_Module_Categ(unittest.TestCase): def test_module(self): item_id = 'tests/2021/01/01/categ.gz' - test_categ = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve'] + test_categ = ['CreditCards', 'Mail', 'Onion', 'Urls', 'Credential', 'Cve'] result = self.module_obj.compute(item_id, r_result=True) + print(result) self.assertCountEqual(result, test_categ) class Test_Module_CreditCards(unittest.TestCase): @@ -87,8 +88,10 @@ class Test_Module_DomClassifier(unittest.TestCase): self.module_obj = DomClassifier() def test_module(self): + test_host = 'foo.be' item_id = 'tests/2021/01/01/domain_classifier.gz' - result = self.module_obj.compute(item_id, r_result=True) + msg = f'{test_host} {item_id}' + result = self.module_obj.compute(msg, r_result=True) self.assertTrue(len(result)) class Test_Module_Global(unittest.TestCase):