Merge branch 'main' of github.com:MISP/misp-modules into main

2021-04-19 08:08:11 +02:00 · 2021-04-19 08:08:11 +02:00 · e25ebd3aff
parent d5cf82f849 0b7bb587be
commit e25ebd3aff
3 changed files with 157 additions and 98 deletions
--- a/misp_modules/modules/expansion/ocr_enrich.py
+++ b/misp_modules/modules/expansion/ocr_enrich.py
@ -6,14 +6,21 @@ import pytesseract

 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['attachment'],
-                  'output': ['freetext', 'text']}
-moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+                  'output': ['freetext']}
+moduleinfo = {'version': '0.2', 'author': 'Sascha Rommelfangen',
              'description': 'OCR decoder',
              'module-type': ['expansion']}

 moduleconfig = []


+def filter_decoded(decoded):
+    for line in decoded.split('\n'):
+        decoded_line = line.strip('\t\x0b\x0c\r ')
+        if decoded_line:
+            yield decoded_line
+
+
 def handler(q=False):
    if q is False:
        return False
@ -31,9 +38,16 @@ def handler(q=False):
    image = img_array
    image = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
    try:
-        decoded = pytesseract.image_to_string(image)
-        return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename},
-                {'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]}
+        decoded = pytesseract.image_to_string(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+        return {
+            'results': [
+                {
+                    'types': ['freetext'],
+                    'values': list(filter_decoded(decoded)),
+                    'comment': f"OCR from file {filename}"
+                }
+            ]
+        }
    except Exception as e:
        print(e)
        err = "Couldn't analyze file type. Only images are supported right now."
--- a/misp_modules/modules/expansion/rbl.py
+++ b/misp_modules/modules/expansion/rbl.py
@ -3,78 +3,75 @@ import sys

 try:
    import dns.resolver
-    resolver = dns.resolver.Resolver()
-    resolver.timeout = 0.2
-    resolver.lifetime = 0.2
 except ImportError:
    print("dnspython3 is missing, use 'pip install dnspython3' to install it.")
    sys.exit(0)

 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']}
-moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
+moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
              'description': 'Check an IPv4 address against known RBLs.',
              'module-type': ['expansion', 'hover']}
-moduleconfig = []
+moduleconfig = ['timeout']

-rbls = {
-    'spam.spamrats.com': 'http://www.spamrats.com',
-    'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/',
-    'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
-    'web.dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net',
-    'virus.rbl.jp': 'http://www.rbl.jp',
-    'dul.dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/',
-    'psbl.surriel.com': 'http://psbl.surriel.com',
-    'misc.dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'httpbl.abuse.ch': 'http://dnsbl.abuse.ch',
-    'combined.njabl.org': 'http://combined.njabl.org',
-    'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'korea.services.net': 'http://korea.services.net',
-    'drone.abuse.ch': 'http://dnsbl.abuse.ch',
-    'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org',
-    'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
-    'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request',
-    'bl.spamcannibal.org': 'http://www.spamcannibal.org',
-    'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/',
-    'zen.spamhaus.org': 'http://www.spamhaus.org/zen/',
-    'rbl.suresupport.com': 'http://suresupport.com/postmaster',
-    'db.wpbl.info': 'http://www.wpbl.info',
-    'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/',
-    'http.dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index',
-    'rbl.interserver.net': 'http://rbl.interserver.net',
-    'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/',
-    'dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'virbl.bit.nl': 'http://virbl.bit.nl',
-    'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/',
-    'socks.dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'short.rbl.jp': 'http://www.rbl.jp',
-    'dnsbl.dronebl.org': 'http://www.dronebl.org',
-    'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
-    'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp',
-    'dyna.spamrats.com': 'http://www.spamrats.com',
-    'spamrbl.imp.ch': 'http://antispam.imp.ch',
-    'spam.dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'wormrbl.imp.ch': 'http://antispam.imp.ch',
-    'query.senderbase.org': 'http://www.senderbase.org/about',
-    'opm.tornevall.org': 'http://dnsbl.tornevall.org',
-    'netblock.pedantic.org': 'http://pedantic.org',
-    'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33',
-    'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US',
-    'multi.surbl.org': 'http://www.surbl.org',
-    'noptr.spamrats.com': 'http://www.spamrats.com',
-    'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en',
-    'bl.spamcop.net': 'http://bl.spamcop.net',
-    'cbl.abuseat.org': 'http://cbl.abuseat.org',
-    'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php',
-    'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net',
-    'dnsbl.njabl.org': 'http://dnsbl.njabl.org',
-    'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html',
-    'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl',
-    'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html'
-}
+rbls = (
+    "spam.spamrats.com",
+    "spamguard.leadmon.net",
+    "rbl-plus.mail-abuse.org",
+    "web.dnsbl.sorbs.net",
+    "ix.dnsbl.manitu.net",
+    "virus.rbl.jp",
+    "dul.dnsbl.sorbs.net",
+    "bogons.cymru.com",
+    "psbl.surriel.com",
+    "misc.dnsbl.sorbs.net",
+    "httpbl.abuse.ch",
+    "combined.njabl.org",
+    "smtp.dnsbl.sorbs.net",
+    "korea.services.net",
+    "drone.abuse.ch",
+    "rbl.efnetrbl.org",
+    "cbl.anti-spam.org.cn",
+    "b.barracudacentral.org",
+    "bl.spamcannibal.org",
+    "xbl.spamhaus.org",
+    "zen.spamhaus.org",
+    "rbl.suresupport.com",
+    "db.wpbl.info",
+    "sbl.spamhaus.org",
+    "http.dnsbl.sorbs.net",
+    "csi.cloudmark.com",
+    "rbl.interserver.net",
+    "ubl.unsubscore.com",
+    "dnsbl.sorbs.net",
+    "virbl.bit.nl",
+    "pbl.spamhaus.org",
+    "socks.dnsbl.sorbs.net",
+    "short.rbl.jp",
+    "dnsbl.dronebl.org",
+    "blackholes.mail-abuse.org",
+    "truncate.gbudb.net",
+    "dyna.spamrats.com",
+    "spamrbl.imp.ch",
+    "spam.dnsbl.sorbs.net",
+    "wormrbl.imp.ch",
+    "query.senderbase.org",
+    "opm.tornevall.org",
+    "netblock.pedantic.org",
+    "access.redhawk.org",
+    "cdl.anti-spam.org.cn",
+    "multi.surbl.org",
+    "noptr.spamrats.com",
+    "dnsbl.inps.de",
+    "bl.spamcop.net",
+    "cbl.abuseat.org",
+    "dsn.rfc-ignorant.org",
+    "zombie.dnsbl.sorbs.net",
+    "dnsbl.njabl.org",
+    "relays.mail-abuse.org",
+    "rbl.spamlab.com",
+    "all.bl.blocklist.de"
+)


 def handler(q=False):
@ -88,18 +85,23 @@ def handler(q=False):
    else:
        misperrors['error'] = "Unsupported attributes type"
        return misperrors
-    listeds = []
-    infos = []
+    resolver = dns.resolver.Resolver()
+    try:
+        timeout = float(request['config']['timeout'])
+    except (KeyError, ValueError):
+        timeout = 0.4
+    resolver.timeout = timeout
+    resolver.lifetime = timeout
+    infos = {}
    ipRev = '.'.join(ip.split('.')[::-1])
    for rbl in rbls:
        query = '{}.{}'.format(ipRev, rbl)
        try:
            txt = resolver.query(query, 'TXT')
-            listeds.append(query)
-            infos.append([str(t) for t in txt])
+            infos[query] = [str(t) for t in txt]
        except Exception:
            continue
-    result = "\n".join([f"{listed}: {'  -  '.join(info)}" for listed, info in zip(listeds, infos)])
+    result = "\n".join([f"{rbl}: {'  -  '.join(info)}" for rbl, info in infos.items()])
    if not result:
        return {'error': 'No data found by querying known RBLs'}
    return {'results': [{'types': mispattributes.get('output'), 'values': result}]}
--- a/tests/test_expansions.py
+++ b/tests/test_expansions.py
@ -116,7 +116,7 @@ class TestExpansions(unittest.TestCase):
            self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC / -0.0005355700 BTC)'))

        except Exception:
-            self.assertEqual(self.get_values(response), 'Not a valid BTC address, or Balance has changed')
+            self.assertTrue(self.get_values(response).startswith('Not a valid BTC address'))

    def test_btc_scam_check(self):
        query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"}
@ -358,7 +358,7 @@ class TestExpansions(unittest.TestCase):
            query["config"] = self.configs[module_name]
            response = self.misp_modules_post(query)
            try:
-                self.assertEqual(self.get_values(response), 'circl.lu')
+                self.assertIn('www.circl.lu', response.json()['results'][0]['values'])
            except Exception:
                self.assertIn(self.get_errors(response), ('We hit an error, time to bail!', 'API quota exceeded.'))
        else:
@ -402,7 +402,7 @@ class TestExpansions(unittest.TestCase):
        query = {"module": "rbl", "ip-src": "8.8.8.8"}
        response = self.misp_modules_post(query)
        try:
-            self.assertTrue(self.get_values(response).startswith('8.8.8.8.query.senderbase.org'))
+            self.assertTrue(self.get_values(response).startswith('8.8.8.8.bl.spamcannibal.org'))
        except Exception:
            self.assertEqual(self.get_errors(response), "No data found by querying known RBLs")

@ -431,11 +431,18 @@ class TestExpansions(unittest.TestCase):

    def test_shodan(self):
        module_name = "shodan"
-        query = {"module": module_name, "ip-src": "149.13.33.14"}
+        query = {
+            "module": module_name,
+            "attribute": {
+                "uuid": "a21aae0c-7426-4762-9b79-854314d69059",
+                "type": "ip-src",
+                "value": "149.13.33.14"
+            }
+        }
        if module_name in self.configs:
            query['config'] = self.configs[module_name]
            response = self.misp_modules_post(query)
-            self.assertIn("circl.lu", self.get_values(response))
+            self.assertEqual(self.get_object(response), 'ip-api-address')
        else:
            response = self.misp_modules_post(query)
            self.assertEqual(self.get_errors(response), 'Shodan authentication is missing')
@ -517,16 +524,33 @@ class TestExpansions(unittest.TestCase):

    def test_virustotal_public(self):
        module_name = "virustotal_public"
-        query_types = ('domain', 'ip-src', 'sha256', 'url')
-        query_values = ('circl.lu', '149.13.33.14',
-                        'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
-                        'http://194.169.88.56:49151/.i')
+        attributes = (
+            {
+                "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3",
+                "type": "domain",
+                "value": "circl.lu"
+            },
+            {
+                "uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979",
+                "type": "ip-src",
+                "value": "149.13.33.14"
+            },
+            {
+                "uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f",
+                "type": "sha256",
+                "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"
+            },
+            {
+                "uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51",
+                "type": "url",
+                "value": "http://194.169.88.56:49151/.i"
+            }
+        )
        results = ('whois', 'asn', 'file', 'virustotal-report')
        if module_name in self.configs:
-            for query_type, query_value, result in zip(query_types, query_values, results):
+            for attribute, result in zip(attributes, results):
                query = {"module": module_name,
-                         "attribute": {"type": query_type,
-                                       "value": query_value},
+                         "attribute": attribute,
                         "config": self.configs[module_name]}
                response = self.misp_modules_post(query)
                try:
@ -534,24 +558,42 @@ class TestExpansions(unittest.TestCase):
                except Exception:
                    self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.")
        else:
-            query = {"module": module_name,
-                     "attribute": {"type": query_types[0],
-                                   "value": query_values[0]}}
+            query = {
+                "module": module_name,
+                "attribute": attributes[0]
+            }
            response = self.misp_modules_post(query)
            self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.")

    def test_virustotal(self):
        module_name = "virustotal"
-        query_types = ('domain', 'ip-src', 'sha256', 'url')
-        query_values = ('circl.lu', '149.13.33.14',
-                        'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3',
-                        'http://194.169.88.56:49151/.i')
+        attributes = (
+            {
+                "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3",
+                "type": "domain",
+                "value": "circl.lu"
+            },
+            {
+                "uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979",
+                "type": "ip-src",
+                "value": "149.13.33.14"
+            },
+            {
+                "uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f",
+                "type": "sha256",
+                "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"
+            },
+            {
+                "uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51",
+                "type": "url",
+                "value": "http://194.169.88.56:49151/.i"
+            }
+        )
        results = ('domain-ip', 'asn', 'virustotal-report', 'virustotal-report')
        if module_name in self.configs:
-            for query_type, query_value, result in zip(query_types, query_values, results):
+            for attribute, result in zip(attributes, results):
                query = {"module": module_name,
-                         "attribute": {"type": query_type,
-                                       "value": query_value},
+                         "attribute": attribute,
                         "config": self.configs[module_name]}
                response = self.misp_modules_post(query)
                try:
@ -559,9 +601,10 @@ class TestExpansions(unittest.TestCase):
                except Exception:
                    self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.")
        else:
-            query = {"module": module_name,
-                     "attribute": {"type": query_types[0],
-                                   "value": query_values[0]}}
+            query = {
+                "module": module_name,
+                "attribute": attributes[0]
+            }
            response = self.misp_modules_post(query)
            self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.")