MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'main' of github.com:MISP/misp-modules into main

pull/496/head
Alexandre Dulaunoy 2021-04-19 08:08:11 +02:00
parent d5cf82f849 0b7bb587be
commit e25ebd3aff
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
3 changed files with 157 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
misp_modules/modules/expansion/ocr_enrich.py
View File

@ -6,14 +6,21 @@ import pytesseract
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
mispattributes = {'input': ['attachment'], mispattributes = {'input': ['attachment'],
'output': ['freetext', 'text']} 'output': ['freetext']}
moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', moduleinfo = {'version': '0.2', 'author': 'Sascha Rommelfangen',
'description': 'OCR decoder', 'description': 'OCR decoder',
'module-type': ['expansion']} 'module-type': ['expansion']}
moduleconfig = [] moduleconfig = []
def filter_decoded(decoded):
for line in decoded.split('\n'):
decoded_line = line.strip('\t\x0b\x0c\r ')
if decoded_line:
yield decoded_line
def handler(q=False): def handler(q=False):
if q is False: if q is False:
return False return False
@ -31,9 +38,16 @@ def handler(q=False):
image = img_array image = img_array
image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) image = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
try: try:
decoded = pytesseract.image_to_string(image) decoded = pytesseract.image_to_string(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename}, return {
{'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]} 'results': [
{
'types': ['freetext'],
'values': list(filter_decoded(decoded)),
'comment': f"OCR from file {filename}"
}
]
}
except Exception as e: except Exception as e:
print(e) print(e)
err = "Couldn't analyze file type. Only images are supported right now." err = "Couldn't analyze file type. Only images are supported right now."

138
misp_modules/modules/expansion/rbl.py
View File

@ -3,78 +3,75 @@ import sys
try: try:
import dns.resolver import dns.resolver
resolver = dns.resolver.Resolver()
resolver.timeout = 0.2
resolver.lifetime = 0.2
except ImportError: except ImportError:
print("dnspython3 is missing, use 'pip install dnspython3' to install it.") print("dnspython3 is missing, use 'pip install dnspython3' to install it.")
sys.exit(0) sys.exit(0)
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']} mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']}
moduleinfo = {'version': '0.1', 'author': 'Christian Studer', moduleinfo = {'version': '0.2', 'author': 'Christian Studer',
'description': 'Check an IPv4 address against known RBLs.', 'description': 'Check an IPv4 address against known RBLs.',
'module-type': ['expansion', 'hover']} 'module-type': ['expansion', 'hover']}
moduleconfig = [] moduleconfig = ['timeout']
rbls = { rbls = (
'spam.spamrats.com': 'http://www.spamrats.com', "spam.spamrats.com",
'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/', "spamguard.leadmon.net",
'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', "rbl-plus.mail-abuse.org",
'web.dnsbl.sorbs.net': 'http://www.sorbs.net', "web.dnsbl.sorbs.net",
'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net', "ix.dnsbl.manitu.net",
'virus.rbl.jp': 'http://www.rbl.jp', "virus.rbl.jp",
'dul.dnsbl.sorbs.net': 'http://www.sorbs.net', "dul.dnsbl.sorbs.net",
'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/', "bogons.cymru.com",
'psbl.surriel.com': 'http://psbl.surriel.com', "psbl.surriel.com",
'misc.dnsbl.sorbs.net': 'http://www.sorbs.net', "misc.dnsbl.sorbs.net",
'httpbl.abuse.ch': 'http://dnsbl.abuse.ch', "httpbl.abuse.ch",
'combined.njabl.org': 'http://combined.njabl.org', "combined.njabl.org",
'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net', "smtp.dnsbl.sorbs.net",
'korea.services.net': 'http://korea.services.net', "korea.services.net",
'drone.abuse.ch': 'http://dnsbl.abuse.ch', "drone.abuse.ch",
'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org', "rbl.efnetrbl.org",
'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', "cbl.anti-spam.org.cn",
'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request', "b.barracudacentral.org",
'bl.spamcannibal.org': 'http://www.spamcannibal.org', "bl.spamcannibal.org",
'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/', "xbl.spamhaus.org",
'zen.spamhaus.org': 'http://www.spamhaus.org/zen/', "zen.spamhaus.org",
'rbl.suresupport.com': 'http://suresupport.com/postmaster', "rbl.suresupport.com",
'db.wpbl.info': 'http://www.wpbl.info', "db.wpbl.info",
'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/', "sbl.spamhaus.org",
'http.dnsbl.sorbs.net': 'http://www.sorbs.net', "http.dnsbl.sorbs.net",
'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index', "csi.cloudmark.com",
'rbl.interserver.net': 'http://rbl.interserver.net', "rbl.interserver.net",
'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/', "ubl.unsubscore.com",
'dnsbl.sorbs.net': 'http://www.sorbs.net', "dnsbl.sorbs.net",
'virbl.bit.nl': 'http://virbl.bit.nl', "virbl.bit.nl",
'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/', "pbl.spamhaus.org",
'socks.dnsbl.sorbs.net': 'http://www.sorbs.net', "socks.dnsbl.sorbs.net",
'short.rbl.jp': 'http://www.rbl.jp', "short.rbl.jp",
'dnsbl.dronebl.org': 'http://www.dronebl.org', "dnsbl.dronebl.org",
'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', "blackholes.mail-abuse.org",
'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp', "truncate.gbudb.net",
'dyna.spamrats.com': 'http://www.spamrats.com', "dyna.spamrats.com",
'spamrbl.imp.ch': 'http://antispam.imp.ch', "spamrbl.imp.ch",
'spam.dnsbl.sorbs.net': 'http://www.sorbs.net', "spam.dnsbl.sorbs.net",
'wormrbl.imp.ch': 'http://antispam.imp.ch', "wormrbl.imp.ch",
'query.senderbase.org': 'http://www.senderbase.org/about', "query.senderbase.org",
'opm.tornevall.org': 'http://dnsbl.tornevall.org', "opm.tornevall.org",
'netblock.pedantic.org': 'http://pedantic.org', "netblock.pedantic.org",
'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33', "access.redhawk.org",
'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', "cdl.anti-spam.org.cn",
'multi.surbl.org': 'http://www.surbl.org', "multi.surbl.org",
'noptr.spamrats.com': 'http://www.spamrats.com', "noptr.spamrats.com",
'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en', "dnsbl.inps.de",
'bl.spamcop.net': 'http://bl.spamcop.net', "bl.spamcop.net",
'cbl.abuseat.org': 'http://cbl.abuseat.org', "cbl.abuseat.org",
'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php', "dsn.rfc-ignorant.org",
'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net', "zombie.dnsbl.sorbs.net",
'dnsbl.njabl.org': 'http://dnsbl.njabl.org', "dnsbl.njabl.org",
'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', "relays.mail-abuse.org",
'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl', "rbl.spamlab.com",
'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html' "all.bl.blocklist.de"
} )
def handler(q=False): def handler(q=False):
@ -88,18 +85,23 @@ def handler(q=False):
else: else:
misperrors['error'] = "Unsupported attributes type" misperrors['error'] = "Unsupported attributes type"
return misperrors return misperrors
listeds = [] resolver = dns.resolver.Resolver()
infos = [] try:
timeout = float(request['config']['timeout'])
except (KeyError, ValueError):
timeout = 0.4
resolver.timeout = timeout
resolver.lifetime = timeout
infos = {}
ipRev = '.'.join(ip.split('.')[::-1]) ipRev = '.'.join(ip.split('.')[::-1])
for rbl in rbls: for rbl in rbls:
query = '{}.{}'.format(ipRev, rbl) query = '{}.{}'.format(ipRev, rbl)
try: try:
txt = resolver.query(query, 'TXT') txt = resolver.query(query, 'TXT')
listeds.append(query) infos[query] = [str(t) for t in txt]
infos.append([str(t) for t in txt])
except Exception: except Exception:
continue continue
result = "\n".join([f"{listed}: {' - '.join(info)}" for listed, info in zip(listeds, infos)]) result = "\n".join([f"{rbl}: {' - '.join(info)}" for rbl, info in infos.items()])
if not result: if not result:
return {'error': 'No data found by querying known RBLs'} return {'error': 'No data found by querying known RBLs'}
return {'results': [{'types': mispattributes.get('output'), 'values': result}]} return {'results': [{'types': mispattributes.get('output'), 'values': result}]}

93
tests/test_expansions.py
View File

@ -116,7 +116,7 @@ class TestExpansions(unittest.TestCase):
self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC / -0.0005355700 BTC)')) self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC / -0.0005355700 BTC)'))
except Exception: except Exception:
self.assertEqual(self.get_values(response), 'Not a valid BTC address, or Balance has changed') self.assertTrue(self.get_values(response).startswith('Not a valid BTC address'))
def test_btc_scam_check(self): def test_btc_scam_check(self):
query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"}
@ -358,7 +358,7 @@ class TestExpansions(unittest.TestCase):
query["config"] = self.configs[module_name] query["config"] = self.configs[module_name]
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
try: try:
self.assertEqual(self.get_values(response), 'circl.lu') self.assertIn('www.circl.lu', response.json()['results'][0]['values'])
except Exception: except Exception:
self.assertIn(self.get_errors(response), ('We hit an error, time to bail!', 'API quota exceeded.')) self.assertIn(self.get_errors(response), ('We hit an error, time to bail!', 'API quota exceeded.'))
else: else:
@ -402,7 +402,7 @@ class TestExpansions(unittest.TestCase):
query = {"module": "rbl", "ip-src": "8.8.8.8"} query = {"module": "rbl", "ip-src": "8.8.8.8"}
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
try: try:
self.assertTrue(self.get_values(response).startswith('8.8.8.8.query.senderbase.org')) self.assertTrue(self.get_values(response).startswith('8.8.8.8.bl.spamcannibal.org'))
except Exception: except Exception:
self.assertEqual(self.get_errors(response), "No data found by querying known RBLs") self.assertEqual(self.get_errors(response), "No data found by querying known RBLs")
@ -431,11 +431,18 @@ class TestExpansions(unittest.TestCase):
def test_shodan(self): def test_shodan(self):
module_name = "shodan" module_name = "shodan"
query = {"module": module_name, "ip-src": "149.13.33.14"} query = {
"module": module_name,
"attribute": {
"uuid": "a21aae0c-7426-4762-9b79-854314d69059",
"type": "ip-src",
"value": "149.13.33.14"
}
}
if module_name in self.configs: if module_name in self.configs:
query['config'] = self.configs[module_name] query['config'] = self.configs[module_name]
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
self.assertIn("circl.lu", self.get_values(response)) self.assertEqual(self.get_object(response), 'ip-api-address')
else: else:
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
self.assertEqual(self.get_errors(response), 'Shodan authentication is missing') self.assertEqual(self.get_errors(response), 'Shodan authentication is missing')
@ -517,16 +524,33 @@ class TestExpansions(unittest.TestCase):
def test_virustotal_public(self): def test_virustotal_public(self):
module_name = "virustotal_public" module_name = "virustotal_public"
query_types = ('domain', 'ip-src', 'sha256', 'url') attributes = (
query_values = ('circl.lu', '149.13.33.14', {
'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3",
'http://194.169.88.56:49151/.i') "type": "domain",
"value": "circl.lu"
},
{
"uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979",
"type": "ip-src",
"value": "149.13.33.14"
},
{
"uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f",
"type": "sha256",
"value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"
},
{
"uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51",
"type": "url",
"value": "http://194.169.88.56:49151/.i"
}
)
results = ('whois', 'asn', 'file', 'virustotal-report') results = ('whois', 'asn', 'file', 'virustotal-report')
if module_name in self.configs: if module_name in self.configs:
for query_type, query_value, result in zip(query_types, query_values, results): for attribute, result in zip(attributes, results):
query = {"module": module_name, query = {"module": module_name,
"attribute": {"type": query_type, "attribute": attribute,
"value": query_value},
"config": self.configs[module_name]} "config": self.configs[module_name]}
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
try: try:
@ -534,24 +558,42 @@ class TestExpansions(unittest.TestCase):
except Exception: except Exception:
self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.")
else: else:
query = {"module": module_name, query = {
"attribute": {"type": query_types[0], "module": module_name,
"value": query_values[0]}} "attribute": attributes[0]
}
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.")
def test_virustotal(self): def test_virustotal(self):
module_name = "virustotal" module_name = "virustotal"
query_types = ('domain', 'ip-src', 'sha256', 'url') attributes = (
query_values = ('circl.lu', '149.13.33.14', {
'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3",
'http://194.169.88.56:49151/.i') "type": "domain",
"value": "circl.lu"
},
{
"uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979",
"type": "ip-src",
"value": "149.13.33.14"
},
{
"uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f",
"type": "sha256",
"value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"
},
{
"uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51",
"type": "url",
"value": "http://194.169.88.56:49151/.i"
}
)
results = ('domain-ip', 'asn', 'virustotal-report', 'virustotal-report') results = ('domain-ip', 'asn', 'virustotal-report', 'virustotal-report')
if module_name in self.configs: if module_name in self.configs:
for query_type, query_value, result in zip(query_types, query_values, results): for attribute, result in zip(attributes, results):
query = {"module": module_name, query = {"module": module_name,
"attribute": {"type": query_type, "attribute": attribute,
"value": query_value},
"config": self.configs[module_name]} "config": self.configs[module_name]}
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
try: try:
@ -559,9 +601,10 @@ class TestExpansions(unittest.TestCase):
except Exception: except Exception:
self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.")
else: else:
query = {"module": module_name, query = {
"attribute": {"type": query_types[0], "module": module_name,
"value": query_values[0]}} "attribute": attributes[0]
}
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.")