diff --git a/misp_modules/modules/expansion/ocr_enrich.py b/misp_modules/modules/expansion/ocr_enrich.py index cd6baca8..ff0a70c7 100644 --- a/misp_modules/modules/expansion/ocr_enrich.py +++ b/misp_modules/modules/expansion/ocr_enrich.py @@ -6,14 +6,21 @@ import pytesseract misperrors = {'error': 'Error'} mispattributes = {'input': ['attachment'], - 'output': ['freetext', 'text']} -moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'output': ['freetext']} +moduleinfo = {'version': '0.2', 'author': 'Sascha Rommelfangen', 'description': 'OCR decoder', 'module-type': ['expansion']} moduleconfig = [] +def filter_decoded(decoded): + for line in decoded.split('\n'): + decoded_line = line.strip('\t\x0b\x0c\r ') + if decoded_line: + yield decoded_line + + def handler(q=False): if q is False: return False @@ -31,9 +38,16 @@ def handler(q=False): image = img_array image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) try: - decoded = pytesseract.image_to_string(image) - return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename}, - {'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]} + decoded = pytesseract.image_to_string(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) + return { + 'results': [ + { + 'types': ['freetext'], + 'values': list(filter_decoded(decoded)), + 'comment': f"OCR from file {filename}" + } + ] + } except Exception as e: print(e) err = "Couldn't analyze file type. Only images are supported right now." diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index 4d7bba5f..d3f661ec 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -3,78 +3,75 @@ import sys try: import dns.resolver - resolver = dns.resolver.Resolver() - resolver.timeout = 0.2 - resolver.lifetime = 0.2 except ImportError: print("dnspython3 is missing, use 'pip install dnspython3' to install it.") sys.exit(0) misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']} -moduleinfo = {'version': '0.1', 'author': 'Christian Studer', +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', 'description': 'Check an IPv4 address against known RBLs.', 'module-type': ['expansion', 'hover']} -moduleconfig = [] +moduleconfig = ['timeout'] -rbls = { - 'spam.spamrats.com': 'http://www.spamrats.com', - 'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/', - 'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'web.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net', - 'virus.rbl.jp': 'http://www.rbl.jp', - 'dul.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/', - 'psbl.surriel.com': 'http://psbl.surriel.com', - 'misc.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'httpbl.abuse.ch': 'http://dnsbl.abuse.ch', - 'combined.njabl.org': 'http://combined.njabl.org', - 'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'korea.services.net': 'http://korea.services.net', - 'drone.abuse.ch': 'http://dnsbl.abuse.ch', - 'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org', - 'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', - 'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request', - 'bl.spamcannibal.org': 'http://www.spamcannibal.org', - 'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/', - 'zen.spamhaus.org': 'http://www.spamhaus.org/zen/', - 'rbl.suresupport.com': 'http://suresupport.com/postmaster', - 'db.wpbl.info': 'http://www.wpbl.info', - 'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/', - 'http.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index', - 'rbl.interserver.net': 'http://rbl.interserver.net', - 'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/', - 'dnsbl.sorbs.net': 'http://www.sorbs.net', - 'virbl.bit.nl': 'http://virbl.bit.nl', - 'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/', - 'socks.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'short.rbl.jp': 'http://www.rbl.jp', - 'dnsbl.dronebl.org': 'http://www.dronebl.org', - 'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp', - 'dyna.spamrats.com': 'http://www.spamrats.com', - 'spamrbl.imp.ch': 'http://antispam.imp.ch', - 'spam.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'wormrbl.imp.ch': 'http://antispam.imp.ch', - 'query.senderbase.org': 'http://www.senderbase.org/about', - 'opm.tornevall.org': 'http://dnsbl.tornevall.org', - 'netblock.pedantic.org': 'http://pedantic.org', - 'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33', - 'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', - 'multi.surbl.org': 'http://www.surbl.org', - 'noptr.spamrats.com': 'http://www.spamrats.com', - 'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en', - 'bl.spamcop.net': 'http://bl.spamcop.net', - 'cbl.abuseat.org': 'http://cbl.abuseat.org', - 'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php', - 'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'dnsbl.njabl.org': 'http://dnsbl.njabl.org', - 'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl', - 'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html' -} +rbls = ( + "spam.spamrats.com", + "spamguard.leadmon.net", + "rbl-plus.mail-abuse.org", + "web.dnsbl.sorbs.net", + "ix.dnsbl.manitu.net", + "virus.rbl.jp", + "dul.dnsbl.sorbs.net", + "bogons.cymru.com", + "psbl.surriel.com", + "misc.dnsbl.sorbs.net", + "httpbl.abuse.ch", + "combined.njabl.org", + "smtp.dnsbl.sorbs.net", + "korea.services.net", + "drone.abuse.ch", + "rbl.efnetrbl.org", + "cbl.anti-spam.org.cn", + "b.barracudacentral.org", + "bl.spamcannibal.org", + "xbl.spamhaus.org", + "zen.spamhaus.org", + "rbl.suresupport.com", + "db.wpbl.info", + "sbl.spamhaus.org", + "http.dnsbl.sorbs.net", + "csi.cloudmark.com", + "rbl.interserver.net", + "ubl.unsubscore.com", + "dnsbl.sorbs.net", + "virbl.bit.nl", + "pbl.spamhaus.org", + "socks.dnsbl.sorbs.net", + "short.rbl.jp", + "dnsbl.dronebl.org", + "blackholes.mail-abuse.org", + "truncate.gbudb.net", + "dyna.spamrats.com", + "spamrbl.imp.ch", + "spam.dnsbl.sorbs.net", + "wormrbl.imp.ch", + "query.senderbase.org", + "opm.tornevall.org", + "netblock.pedantic.org", + "access.redhawk.org", + "cdl.anti-spam.org.cn", + "multi.surbl.org", + "noptr.spamrats.com", + "dnsbl.inps.de", + "bl.spamcop.net", + "cbl.abuseat.org", + "dsn.rfc-ignorant.org", + "zombie.dnsbl.sorbs.net", + "dnsbl.njabl.org", + "relays.mail-abuse.org", + "rbl.spamlab.com", + "all.bl.blocklist.de" +) def handler(q=False): @@ -88,18 +85,23 @@ def handler(q=False): else: misperrors['error'] = "Unsupported attributes type" return misperrors - listeds = [] - infos = [] + resolver = dns.resolver.Resolver() + try: + timeout = float(request['config']['timeout']) + except (KeyError, ValueError): + timeout = 0.4 + resolver.timeout = timeout + resolver.lifetime = timeout + infos = {} ipRev = '.'.join(ip.split('.')[::-1]) for rbl in rbls: query = '{}.{}'.format(ipRev, rbl) try: txt = resolver.query(query, 'TXT') - listeds.append(query) - infos.append([str(t) for t in txt]) + infos[query] = [str(t) for t in txt] except Exception: continue - result = "\n".join([f"{listed}: {' - '.join(info)}" for listed, info in zip(listeds, infos)]) + result = "\n".join([f"{rbl}: {' - '.join(info)}" for rbl, info in infos.items()]) if not result: return {'error': 'No data found by querying known RBLs'} return {'results': [{'types': mispattributes.get('output'), 'values': result}]} diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 2674c64d..d6b7b337 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -116,7 +116,7 @@ class TestExpansions(unittest.TestCase): self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC / -0.0005355700 BTC)')) except Exception: - self.assertEqual(self.get_values(response), 'Not a valid BTC address, or Balance has changed') + self.assertTrue(self.get_values(response).startswith('Not a valid BTC address')) def test_btc_scam_check(self): query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} @@ -358,7 +358,7 @@ class TestExpansions(unittest.TestCase): query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertEqual(self.get_values(response), 'circl.lu') + self.assertIn('www.circl.lu', response.json()['results'][0]['values']) except Exception: self.assertIn(self.get_errors(response), ('We hit an error, time to bail!', 'API quota exceeded.')) else: @@ -402,7 +402,7 @@ class TestExpansions(unittest.TestCase): query = {"module": "rbl", "ip-src": "8.8.8.8"} response = self.misp_modules_post(query) try: - self.assertTrue(self.get_values(response).startswith('8.8.8.8.query.senderbase.org')) + self.assertTrue(self.get_values(response).startswith('8.8.8.8.bl.spamcannibal.org')) except Exception: self.assertEqual(self.get_errors(response), "No data found by querying known RBLs") @@ -431,11 +431,18 @@ class TestExpansions(unittest.TestCase): def test_shodan(self): module_name = "shodan" - query = {"module": module_name, "ip-src": "149.13.33.14"} + query = { + "module": module_name, + "attribute": { + "uuid": "a21aae0c-7426-4762-9b79-854314d69059", + "type": "ip-src", + "value": "149.13.33.14" + } + } if module_name in self.configs: query['config'] = self.configs[module_name] response = self.misp_modules_post(query) - self.assertIn("circl.lu", self.get_values(response)) + self.assertEqual(self.get_object(response), 'ip-api-address') else: response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), 'Shodan authentication is missing') @@ -517,16 +524,33 @@ class TestExpansions(unittest.TestCase): def test_virustotal_public(self): module_name = "virustotal_public" - query_types = ('domain', 'ip-src', 'sha256', 'url') - query_values = ('circl.lu', '149.13.33.14', - 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - 'http://194.169.88.56:49151/.i') + attributes = ( + { + "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3", + "type": "domain", + "value": "circl.lu" + }, + { + "uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979", + "type": "ip-src", + "value": "149.13.33.14" + }, + { + "uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f", + "type": "sha256", + "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3" + }, + { + "uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51", + "type": "url", + "value": "http://194.169.88.56:49151/.i" + } + ) results = ('whois', 'asn', 'file', 'virustotal-report') if module_name in self.configs: - for query_type, query_value, result in zip(query_types, query_values, results): + for attribute, result in zip(attributes, results): query = {"module": module_name, - "attribute": {"type": query_type, - "value": query_value}, + "attribute": attribute, "config": self.configs[module_name]} response = self.misp_modules_post(query) try: @@ -534,24 +558,42 @@ class TestExpansions(unittest.TestCase): except Exception: self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") else: - query = {"module": module_name, - "attribute": {"type": query_types[0], - "value": query_values[0]}} + query = { + "module": module_name, + "attribute": attributes[0] + } response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") def test_virustotal(self): module_name = "virustotal" - query_types = ('domain', 'ip-src', 'sha256', 'url') - query_values = ('circl.lu', '149.13.33.14', - 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - 'http://194.169.88.56:49151/.i') + attributes = ( + { + "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3", + "type": "domain", + "value": "circl.lu" + }, + { + "uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979", + "type": "ip-src", + "value": "149.13.33.14" + }, + { + "uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f", + "type": "sha256", + "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3" + }, + { + "uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51", + "type": "url", + "value": "http://194.169.88.56:49151/.i" + } + ) results = ('domain-ip', 'asn', 'virustotal-report', 'virustotal-report') if module_name in self.configs: - for query_type, query_value, result in zip(query_types, query_values, results): + for attribute, result in zip(attributes, results): query = {"module": module_name, - "attribute": {"type": query_type, - "value": query_value}, + "attribute": attribute, "config": self.configs[module_name]} response = self.misp_modules_post(query) try: @@ -559,9 +601,10 @@ class TestExpansions(unittest.TestCase): except Exception: self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") else: - query = {"module": module_name, - "attribute": {"type": query_types[0], - "value": query_values[0]}} + query = { + "module": module_name, + "attribute": attributes[0] + } response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.")