From 3d47eb74207585870c160d2269c4a123dddcfc40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 25 Jan 2019 10:45:02 +0100 Subject: [PATCH 01/46] fix: make flake8 happy --- misp_modules/modules/expansion/btc_steroids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py index 7011eda..430c67d 100755 --- a/misp_modules/modules/expansion/btc_steroids.py +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -191,7 +191,7 @@ def handler(q=False): value = float(tx['value'] / 100000000) u, e = convert(value, transactions['time']) mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0')) - #i += 1 + # i += 1 i += 1 r = { From 7a7b7b109f93126f14ea8697b27d359e096aa057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 4 Feb 2019 10:29:23 +0100 Subject: [PATCH 02/46] chg: Bump dependencies --- Pipfile.lock | 79 ++++++++++++++++++++++++++++------------------------ REQUIREMENTS | 10 +++---- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 02a61d1..19f32f0 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -281,17 +281,17 @@ }, "psutil": { "hashes": [ - "sha256:1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736", - "sha256:1c71b9716790e202a00ab0931a6d1e25db1aa1198bcacaea2f5329f75d257fff", - "sha256:3b7a4daf4223dae171a67a89314ac5ca0738e94064a78d99cfd751c55d05f315", - "sha256:3e19be3441134445347af3767fa7770137d472a484070840eee6653b94ac5576", - "sha256:6e265c8f3da00b015d24b842bfeb111f856b13d24f2c57036582568dc650d6c3", - "sha256:809c9cef0402e3e48b5a1dddc390a8a6ff58b15362ea5714494073fa46c3d293", - "sha256:b4d1b735bf5b120813f4c89db8ac22d89162c558cbd7fdd298866125fe906219", - "sha256:bbffac64cfd01c6bcf90eb1bedc6c80501c4dae8aef4ad6d6dd49f8f05f6fc5a", - "sha256:bfcea4f189177b2d2ce4a34b03c4ac32c5b4c22e21f5b093d9d315e6e253cd81" + "sha256:04d2071100aaad59f9bcbb801be2125d53b2e03b1517d9fed90b45eea51d297e", + "sha256:1aba93430050270750d046a179c5f3d6e1f5f8b96c20399ba38c596b28fc4d37", + "sha256:3ac48568f5b85fee44cd8002a15a7733deca056a191d313dbf24c11519c0c4a8", + "sha256:96f3fdb4ef7467854d46ad5a7e28eb4c6dc6d455d751ddf9640cd6d52bdb03d7", + "sha256:b755be689d6fc8ebc401e1d5ce5bac867e35788f10229e166338484eead51b12", + "sha256:c8ee08ad1b716911c86f12dc753eb1879006224fd51509f077987bb6493be615", + "sha256:d0c4230d60376aee0757d934020b14899f6020cd70ef8d2cb4f228b6ffc43e8f", + "sha256:d23f7025bac9b3e38adc6bd032cdaac648ac0074d18e36950a04af35458342e8", + "sha256:f0fcb7d3006dd4d9ccf3ccd0595d44c6abbfd433ec31b6ca177300ee3f19e54e" ], - "version": "==5.4.8" + "version": "==5.5.0" }, "pybgpranking": { "editable": true, @@ -333,7 +333,7 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "d4934cdf5f537c9f42ae37be7878de1848961de0" + "ref": "2c877f2aec11b7f5d2f23dfc5ce7398b2ce33b48" }, "pyonyphe": { "editable": true, @@ -400,10 +400,10 @@ }, "redis": { "hashes": [ - "sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf", - "sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a" + "sha256:74c892041cba46078ae1ef845241548baa3bd3634f9a6f0f952f006eb1619c71", + "sha256:7ba8612bbfd966dea8c62322543fed0095da2834dbd5a7c124afbc617a156aa7" ], - "version": "==3.0.1" + "version": "==3.1.0" }, "requests": { "hashes": [ @@ -443,10 +443,10 @@ }, "soupsieve": { "hashes": [ - "sha256:10687fc53eeb3518e01a0ac84d3d711da623d3298a3039459d3f649927c4a270", - "sha256:b23a0d7da0247200fe83c67c34de9d7599ad404106367313d8e65e04174d0b4b" + "sha256:466910df7561796a60748826781ebe9a888f7a1668a636ae86783f44d10aae73", + "sha256:87db12ae79194f0ff9808d2b1641c4f031ae39ffa3cab6b907ea7c1e5e5ed445" ], - "version": "==1.7.2" + "version": "==1.7.3" }, "sparqlwrapper": { "hashes": [ @@ -505,12 +505,12 @@ }, "vulners": { "hashes": [ - "sha256:8b468db8f8b0bad39ae51ebd4247f6ead90b6f53699e03b91ff9d63da70554d7", - "sha256:ad72378c842096cad9ebf83aa53d330117ece5d208ed7c419a21c70a8d5e2236", - "sha256:ffc92a099eeddea840fd199665992c0eb6d7ad69ac3a6730a286d00600bc5f2c" + "sha256:5f05404041cfaa8e5367bf884fc9ee319ebf34bedc495d7f84c433fa121cdb49", + "sha256:919b24df64ea55b6a8ba13e2a0530578f8a4be6a9cee257bf2214046e81c6f35", + "sha256:d45ecb13f5111947056a2dcc071b3e3fd45f6ad654eda06526245bba3850325e" ], "index": "pypi", - "version": "==1.3.6" + "version": "==1.4.0" }, "wand": { "hashes": [ @@ -564,10 +564,10 @@ "develop": { "atomicwrites": { "hashes": [ - "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", - "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.2.1" + "version": "==1.3.0" }, "attrs": { "hashes": [ @@ -634,13 +634,20 @@ ], "version": "==4.5.2" }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, "flake8": { "hashes": [ - "sha256:6a35f5b8761f45c5513e3405f110a86bea57982c3b75b766ce7b65217abe1670", - "sha256:c01f8a3963b3571a8e6bd7a4063359aff90749e160778e03817cd9b71c9e07d2" + "sha256:09b9bb539920776da542e67a570a5df96ff933c9a08b62cfae920bcc789e4383", + "sha256:e0f8cd519cfc0072c0ee31add5def09d2b3ef6040b34dc426445c3af9b02163c" ], "index": "pypi", - "version": "==3.6.0" + "version": "==3.7.4" }, "idna": { "hashes": [ @@ -689,25 +696,25 @@ }, "pycodestyle": { "hashes": [ - "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", - "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" ], - "version": "==2.4.0" + "version": "==2.5.0" }, "pyflakes": { "hashes": [ - "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", - "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" + "sha256:5e8c00e30c464c99e0b501dc160b13a14af7f27d4dffb529c556e30a159e231d", + "sha256:f277f9ca3e55de669fba45b7393a1449009cff5a37d1af10ebb76c52765269cd" ], - "version": "==2.0.0" + "version": "==2.1.0" }, "pytest": { "hashes": [ - "sha256:41568ea7ecb4a68d7f63837cf65b92ce8d0105e43196ff2b26622995bb3dc4b2", - "sha256:c3c573a29d7c9547fb90217ece8a8843aa0c1328a797e200290dc3d0b4b823be" + "sha256:65aeaa77ae87c7fc95de56285282546cfa9c886dc8e5dc78313db1c25e21bc07", + "sha256:6ac6d467d9f053e95aaacd79f831dbecfe730f419c6c7022cb316b365cd9199d" ], "index": "pypi", - "version": "==4.1.1" + "version": "==4.2.0" }, "requests": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index 709620a..c3c16e6 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@d4934cdf5f537c9f42ae37be7878de1848961de0#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@2c877f2aec11b7f5d2f23dfc5ce7398b2ce33b48#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe @@ -33,7 +33,7 @@ multidict==4.5.2 oauth2==1.9.0.post1 passivetotal==1.0.30 pillow==5.4.1 -psutil==5.4.8 +psutil==5.5.0 pyeupi==1.0 pygeoip==0.3.2 pyparsing==2.3.1 @@ -43,20 +43,20 @@ pytesseract==0.2.6 python-dateutil==2.7.5 pyyaml==3.13 rdflib==4.2.2 -redis==3.0.1 +redis==3.1.0 requests-cache==0.4.13 requests==2.21.0 shodan==1.10.4 sigmatools==0.7.1 six==1.12.0 -soupsieve==1.7.2 +soupsieve==1.7.3 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 tornado==5.1.1 url-normalize==1.4.1 urlarchiver==0.2 urllib3==1.24.1 -vulners==1.3.6 +vulners==1.4.0 wand==0.5.0 xlsxwriter==1.1.2 yara-python==3.8.1 From 454c9e0f437442d6cb3da71ca589df3c3c6f2a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 4 Feb 2019 11:05:51 +0100 Subject: [PATCH 03/46] fix: Pep8 related fixes. --- .../modules/expansion/circl_passivedns.py | 2 +- .../modules/expansion/xforceexchange.py | 207 +++++++++--------- misp_modules/modules/export_mod/liteexport.py | 112 +++++----- .../modules/export_mod/nexthinkexport.py | 2 +- .../modules/export_mod/osqueryexport.py | 2 +- misp_modules/modules/export_mod/pdfexport.py | 51 ++--- .../modules/import_mod/openiocimport.py | 2 +- .../import_mod/threatanalyzer_import.py | 4 +- 8 files changed, 192 insertions(+), 190 deletions(-) diff --git a/misp_modules/modules/expansion/circl_passivedns.py b/misp_modules/modules/expansion/circl_passivedns.py index 3da5bac..263b92a 100755 --- a/misp_modules/modules/expansion/circl_passivedns.py +++ b/misp_modules/modules/expansion/circl_passivedns.py @@ -32,7 +32,7 @@ def handler(q=False): res = x.query(toquery) out = '' for v in res: - out = out + "{} ".format(v['rdata']) + out = out + "{} ".format(v['rdata']) r = {'results': [{'types': mispattributes['output'], 'values': out}]} return r diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py index 0f01e44..6bb7126 100644 --- a/misp_modules/modules/expansion/xforceexchange.py +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -1,103 +1,104 @@ -import requests -import json -import sys - -BASEurl = "https://api.xforce.ibmcloud.com/" - -extensions = {"ip1": "ipr/%s", - "ip2": "ipr/malware/%s", - "url": "url/%s", - "hash": "malware/%s", - "vuln": "/vulnerabilities/search/%s", - "dns": "resolve/%s"} - -sys.path.append('./') - -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'vulnerability', 'md5', 'sha1', 'sha256'], - 'output': ['ip-src', 'ip-dst', 'text', 'domain']} - -# possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)', - 'description': 'IBM X-Force Exchange expansion module', - 'module-type': ['expansion', 'hover']} - -# config fields that your code expects from the site admin -moduleconfig = ["apikey", "event_limit"] -limit = 5000 # Default - - -def MyHeader(key=False): - global limit - if key is False: - return None - - return {"Authorization": "Basic %s " % key, - "Accept": "application/json", - 'User-Agent': 'Mozilla 5.0'} - - -def handler(q=False): - global limit - if q is False: - return False - - q = json.loads(q) - - key = q["config"]["apikey"] - limit = int(q["config"].get("event_limit", 5)) - - r = {"results": []} - - if "ip-src" in q: - r["results"] += apicall("dns", q["ip-src"], key) - if "ip-dst" in q: - r["results"] += apicall("dns", q["ip-dst"], key) - if "md5" in q: - r["results"] += apicall("hash", q["md5"], key) - if "sha1" in q: - r["results"] += apicall("hash", q["sha1"], key) - if "sha256" in q: - r["results"] += apicall("hash", q["sha256"], key) - if 'vulnerability' in q: - r["results"] += apicall("vuln", q["vulnerability"], key) - if "domain" in q: - r["results"] += apicall("dns", q["domain"], key) - - uniq = [] - for res in r["results"]: - if res not in uniq: - uniq.append(res) - r["results"] = uniq - return r - - -def apicall(indicator_type, indicator, key=False): - try: - myURL = BASEurl + (extensions[str(indicator_type)]) % indicator - jsondata = requests.get(myURL, headers=MyHeader(key)).json() - except Exception: - jsondata = None - redata = [] - # print(jsondata) - if jsondata is not None: - if indicator_type is "hash": - if "malware" in jsondata: - lopointer = jsondata["malware"] - redata.append({"type": "text", "values": lopointer["risk"]}) - if indicator_type is "dns": - if "records" in str(jsondata): - lopointer = jsondata["Passive"]["records"] - for dataset in lopointer: - redata.append({"type": "domain", "values": dataset["value"]}) - - return redata - - -def introspection(): - return mispattributes - - -def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo +import requests +import json +import sys + +BASEurl = "https://api.xforce.ibmcloud.com/" + +extensions = {"ip1": "ipr/%s", + "ip2": "ipr/malware/%s", + "url": "url/%s", + "hash": "malware/%s", + "vuln": "/vulnerabilities/search/%s", + "dns": "resolve/%s"} + +sys.path.append('./') + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'vulnerability', 'md5', 'sha1', 'sha256'], + 'output': ['ip-src', 'ip-dst', 'text', 'domain']} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)', + 'description': 'IBM X-Force Exchange expansion module', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ["apikey", "event_limit"] +limit = 5000 # Default + + +def MyHeader(key=False): + global limit + if key is False: + return None + + return {"Authorization": "Basic %s " % key, + "Accept": "application/json", + 'User-Agent': 'Mozilla 5.0'} + + +def handler(q=False): + global limit + if q is False: + return False + + q = json.loads(q) + + key = q["config"]["apikey"] + limit = int(q["config"].get("event_limit", 5)) + + r = {"results": []} + + if "ip-src" in q: + r["results"] += apicall("dns", q["ip-src"], key) + if "ip-dst" in q: + r["results"] += apicall("dns", q["ip-dst"], key) + if "md5" in q: + r["results"] += apicall("hash", q["md5"], key) + if "sha1" in q: + r["results"] += apicall("hash", q["sha1"], key) + if "sha256" in q: + r["results"] += apicall("hash", q["sha256"], key) + if 'vulnerability' in q: + r["results"] += apicall("vuln", q["vulnerability"], key) + if "domain" in q: + r["results"] += apicall("dns", q["domain"], key) + + uniq = [] + for res in r["results"]: + if res not in uniq: + uniq.append(res) + r["results"] = uniq + return r + + +def apicall(indicator_type, indicator, key=False): + try: + myURL = BASEurl + (extensions[str(indicator_type)]) % indicator + jsondata = requests.get(myURL, headers=MyHeader(key)).json() + except Exception: + jsondata = None + redata = [] + # print(jsondata) + if jsondata is not None: + if indicator_type == "hash": + if "malware" in jsondata: + lopointer = jsondata["malware"] + redata.append({"type": "text", "values": lopointer["risk"]}) + if indicator_type == "dns": + if "records" in str(jsondata): + lopointer = jsondata["Passive"]["records"] + for dataset in lopointer: + redata.append( + {"type": "domain", "values": dataset["value"]}) + + return redata + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/liteexport.py b/misp_modules/modules/export_mod/liteexport.py index e89c1c1..870f52a 100755 --- a/misp_modules/modules/export_mod/liteexport.py +++ b/misp_modules/modules/export_mod/liteexport.py @@ -16,73 +16,73 @@ responseType = "application/json" def handler(q=False): - if q is False: - return False + if q is False: + return False - request = json.loads(q) + request = json.loads(q) - config = {} - if "config" in request: - config = request["config"] - else: - config = {"indent_json_export": None} + config = {} + if "config" in request: + config = request["config"] + else: + config = {"indent_json_export": None} - if config['indent_json_export'] is not None: - try: - config['indent_json_export'] = int(config['indent_json_export']) - except Exception: - config['indent_json_export'] = None + if config['indent_json_export'] is not None: + try: + config['indent_json_export'] = int(config['indent_json_export']) + except Exception: + config['indent_json_export'] = None - if 'data' not in request: - return False + if 'data' not in request: + return False - # ~ Misp json structur - liteEvent = {'Event': {}} + # ~ Misp json structur + liteEvent = {'Event': {}} - for evt in request['data']: - rawEvent = evt['Event'] - liteEvent['Event']['info'] = rawEvent['info'] - liteEvent['Event']['Attribute'] = [] + for evt in request['data']: + rawEvent = evt['Event'] + liteEvent['Event']['info'] = rawEvent['info'] + liteEvent['Event']['Attribute'] = [] - attrs = evt['Attribute'] - for attr in attrs: - if 'Internal reference' not in attr['category']: - liteAttr = {} - liteAttr['category'] = attr['category'] - liteAttr['type'] = attr['type'] - liteAttr['value'] = attr['value'] - liteEvent['Event']['Attribute'].append(liteAttr) + attrs = evt['Attribute'] + for attr in attrs: + if 'Internal reference' not in attr['category']: + liteAttr = {} + liteAttr['category'] = attr['category'] + liteAttr['type'] = attr['type'] + liteAttr['value'] = attr['value'] + liteEvent['Event']['Attribute'].append(liteAttr) - return {'response': [], - 'data': str(base64.b64encode(bytes( - json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), 'utf-8')} + return {'response': [], + 'data': str(base64.b64encode(bytes( + json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), 'utf-8')} def introspection(): - modulesetup = {} - try: - responseType - modulesetup['responseType'] = responseType - except NameError: - pass - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/nexthinkexport.py b/misp_modules/modules/export_mod/nexthinkexport.py index f1a0d79..c87b3fb 100755 --- a/misp_modules/modules/export_mod/nexthinkexport.py +++ b/misp_modules/modules/export_mod/nexthinkexport.py @@ -86,7 +86,7 @@ def handler(q=False): for event in request["data"]: for attribute in event["Attribute"]: if attribute['type'] in types_to_use: - output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n' + output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n' r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')} return r diff --git a/misp_modules/modules/export_mod/osqueryexport.py b/misp_modules/modules/export_mod/osqueryexport.py index ba98fe6..6368875 100755 --- a/misp_modules/modules/export_mod/osqueryexport.py +++ b/misp_modules/modules/export_mod/osqueryexport.py @@ -80,7 +80,7 @@ def handler(q=False): for event in request["data"]: for attribute in event["Attribute"]: if attribute['type'] in types_to_use: - output = output + handlers[attribute['type']](attribute['value']) + '\n' + output = output + handlers[attribute['type']](attribute['value']) + '\n' r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')} return r diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 77a2e83..df7f879 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -152,36 +152,37 @@ def handler(q=False): command_line = 'asciidoctor-pdf -' args = shlex.split(command_line) with subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) as process: - cmd_out, cmd_err = process.communicate(input=report.report.encode('utf-8')) + cmd_out, cmd_err = process.communicate( + input=report.report.encode('utf-8')) return {'response': [], 'data': str(base64.b64encode(cmd_out), 'utf-8')} def introspection(): - modulesetup = {} - try: - responseType - modulesetup['responseType'] = responseType - except NameError: - pass + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index c237bdc..074a464 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -63,7 +63,7 @@ def handler(q=False): "comment": getattr(attrib, 'comment', '')} # add tag if q.get('config') and q['config'].get('default tag') is not None: - toAppend["tags"] = q['config']['default tag'].split(",") + toAppend["tags"] = q['config']['default tag'].split(",") r["results"].append(toAppend) return r diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index 4ae1cd2..ff0a5b1 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -325,7 +325,7 @@ def process_analysis_json(analysis_json): for stored_created_file in process['stored_files']['stored_created_file']: stored_created_file['@filename'] = cleanup_filepath(stored_created_file['@filename']) if stored_created_file['@filename']: - if stored_created_file['@filesize'] is not '0': + if stored_created_file['@filesize'] != '0': val = '{}|{}'.format(stored_created_file['@filename'], stored_created_file['@md5']) # print("stored_created_file filename|md5: {}|{} IDS:yes".format( # stored_created_file['@filename'], # filename @@ -346,7 +346,7 @@ def process_analysis_json(analysis_json): for stored_modified_file in process['stored_files']['stored_modified_file']: stored_modified_file['@filename'] = cleanup_filepath(stored_modified_file['@filename']) if stored_modified_file['@filename']: - if stored_modified_file['@filesize'] is not '0': + if stored_modified_file['@filesize'] != '0': val = '{}|{}'.format(stored_modified_file['@filename'], stored_modified_file['@md5']) # print("stored_modified_file MODIFY FILE: {}\t{}".format( # stored_modified_file['@filename'], # filename From d1000d82c4d14f50c68a606cec897849281bf2e0 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 5 Feb 2019 14:46:42 +0100 Subject: [PATCH 04/46] add: New module to check if a bitcoin address has been abused - Also related update of documentation --- README.md | 1 + doc/README.md | 1262 ++++++++++++++++- doc/documentation.md | 1243 ---------------- doc/expansion/btc_scam_check.json | 9 + doc/expansion/{btc.json => btc_steroids.json} | 0 doc/generate_documentation.py | 2 +- misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/btc_scam_check.py | 43 + 8 files changed, 1316 insertions(+), 1246 deletions(-) mode change 120000 => 100644 doc/README.md delete mode 100644 doc/documentation.md create mode 100644 doc/expansion/btc_scam_check.json rename doc/expansion/{btc.json => btc_steroids.json} (100%) create mode 100644 misp_modules/modules/expansion/btc_scam_check.py diff --git a/README.md b/README.md index 368ef6f..e8fa0d2 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Expansion modules * [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. +* [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused. * [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. diff --git a/doc/README.md b/doc/README.md deleted file mode 120000 index 0963ae8..0000000 --- a/doc/README.md +++ /dev/null @@ -1 +0,0 @@ -documentation.md \ No newline at end of file diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..e47470d --- /dev/null +++ b/doc/README.md @@ -0,0 +1,1261 @@ +# MISP modules documentation + +## Expansion Modules + +#### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) + +Query BGP Ranking (https://bgpranking-ng.circl.lu/). +- **features**: +>The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking. +> +> +- **input**: +>Autonomous system number. +- **output**: +>Text containing a description of the ASN, its history, and the position in BGP Ranking. +- **references**: +>https://github.com/D4-project/BGP-Ranking/ +- **requirements**: +>pybgpranking python library + +----- + +#### [btc_scam_check](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_scam_check.py) + + + +An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused. +- **features**: +>The module queries a dns blacklist directly with the bitcoin address and get a response if the address has been abused. +- **input**: +>btc address attribute. +- **output**: +>Text to indicate if the BTC address has been abused. +- **references**: +>https://btcblack.it/ +- **requirements**: +>dnspython3: dns python library + +----- + +#### [btc_steroids](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_steroids.py) + + + +An expansion hover module to get a blockchain balance from a BTC address in MISP. +- **input**: +>btc address attribute. +- **output**: +>Text to describe the blockchain balance and the transactions related to the btc address in input. + +----- + +#### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) + + + +Module to access CIRCL Passive DNS. +- **features**: +>This module takes a hostname, domain or ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive DNS REST API to get and display information about this input. +> +>To make it work a username and a password are thus required to authenticate to the CIRCL Passive DNS API. +- **input**: +>Hostname, domain, or ip-address attribute. +- **ouput**: +>Text describing passive DNS information related to the input attribute. +- **references**: +>https://www.circl.lu/services/passive-dns/, https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/ +- **requirements**: +>pypdns: Passive DNS python library, A CIRCL passive DNS account with username & password + +----- + +#### [circl_passivessl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) + + + +Modules to access CIRCL Passive SSL. +- **features**: +>This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive SSL REST API to get and display information about this input. +> +>To make it work a username and a password are thus required to authenticate to the CIRCL Passive SSL API. +- **input**: +>Ip-address attribute. +- **output**: +>Text describing passive SSL information related to the input attribute. +- **references**: +>https://www.circl.lu/services/passive-ssl/ +- **requirements**: +>pypssl: Passive SSL python library, A CIRCL passive SSL account with username & password + +----- + +#### [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) + +Module to expand country codes. +- **features**: +>The module takes a domain or a hostname as input, and returns the country it belongs to. +> +>For non country domains, a list of the most common possible extensions is used. +- **input**: +>Hostname or domain attribute. +- **output**: +>Text with the country code the input belongs to. + +----- + +#### [crowdstrike_falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) + + + +Module to query Crowdstrike Falcon. +- **features**: +>This module takes a MISP attribute as input to query a CrowdStrike Falcon API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. +> +>Please note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported. +- **input**: +>A MISP attribute included in the following list: +>- domain +>- email-attachment +>- email-dst +>- email-reply-to +>- email-src +>- email-subject +>- filename +>- hostname +>- ip-src +>- ip-dst +>- md5 +>- mutex +>- regkey +>- sha1 +>- sha256 +>- uri +>- url +>- user-agent +>- whois-registrant-email +>- x509-fingerprint-md5 +- **output**: +>MISP attributes mapped after the CrowdStrike API has been queried, included in the following list: +>- hostname +>- email-src +>- email-subject +>- filename +>- md5 +>- sha1 +>- sha256 +>- ip-dst +>- ip-dst +>- mutex +>- regkey +>- url +>- user-agent +>- x509-fingerprint-md5 +- **references**: +>https://www.crowdstrike.com/products/crowdstrike-falcon-faq/ +- **requirements**: +>A CrowdStrike API access (API id & key) + +----- + +#### [cve](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) + + + +An expansion hover module to expand information about CVE id. +- **features**: +>The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to get information about the vulnerability as it is described in the list of CVEs. +- **input**: +>Vulnerability attribute. +- **output**: +>Text giving information about the CVE related to the Vulnerability. +- **references**: +>https://cve.circl.lu/, https://cve.mitre.org/ + +----- + +#### [dbl_spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) + + + +Module to check Spamhaus DBL for a domain name. +- **features**: +>This modules takes a domain or a hostname in input and queries the Domain Block List provided by Spamhaus to determine what kind of domain it is. +> +>DBL then returns a response code corresponding to a certain classification of the domain we display. If the queried domain is not in the list, it is also mentionned. +> +>Please note that composite MISP attributes containing domain or hostname are supported as well. +- **input**: +>Domain or hostname attribute. +- **output**: +>Information about the nature of the input. +- **references**: +>https://www.spamhaus.org/faq/section/Spamhaus%20DBL +- **requirements**: +>dnspython3: DNS python3 library + +----- + +#### [dns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) + +A simple DNS expansion service to resolve IP address from domain MISP attributes. +- **features**: +>The module takes a domain of hostname attribute as input, and tries to resolve it. If no error is encountered, the IP address that resolves the domain is returned, otherwise the origin of the error is displayed. +> +>The address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8). +> +>Please note that composite MISP attributes containing domain or hostname are supported as well. +- **input**: +>Domain or hostname attribute. +- **output**: +>IP address resolving the input. +- **requirements**: +>dnspython3: DNS python3 library + +----- + +#### [domaintools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) + + + +DomainTools MISP expansion module. +- **features**: +>This module takes a MISP attribute as input to query the Domaintools API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. +> +>Please note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported. +- **input**: +>A MISP attribute included in the following list: +>- domain +>- hostname +>- email-src +>- email-dst +>- target-email +>- whois-registrant-email +>- whois-registrant-name +>- whois-registrant-phone +>- ip-src +>- ip-dst +- **output**: +>MISP attributes mapped after the Domaintools API has been queried, included in the following list: +>- whois-registrant-email +>- whois-registrant-phone +>- whois-registrant-name +>- whois-registrar +>- whois-creation-date +>- text +>- domain +- **references**: +>https://www.domaintools.com/ +- **requirements**: +>Domaintools python library, A Domaintools API access (username & apikey) + +----- + +#### [eupi](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) + + + +A module to query the Phishing Initiative service (https://phishing-initiative.lu). +- **features**: +>This module takes a domain, hostname or url MISP attribute as input to query the Phishing Initiative API. The API returns then the result of the query with some information about the value queried. +> +>Please note that composite attributes containing domain or hostname are also supported. +- **input**: +>A domain, hostname or url MISP attribute. +- **output**: +>Text containing information about the input, resulting from the query on Phishing Initiative. +- **references**: +>https://phishing-initiative.eu/?lang=en +- **requirements**: +>pyeupi: eupi python library, An access to the Phishing Initiative API (apikey & url) + +----- + +#### [farsight_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) + + + +Module to access Farsight DNSDB Passive DNS. +- **features**: +>This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. The API returns then the result of the query with some information about the value queried. +- **input**: +>A domain, hostname or IP address MISP attribute. +- **output**: +>Text containing information about the input, resulting from the query on the Farsight Passive DNS API. +- **references**: +>https://www.farsightsecurity.com/ +- **requirements**: +>An access to the Farsight Passive DNS API (apikey) + +----- + +#### [geoip_country](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) + + + +Module to query a local copy of Maxmind's Geolite database. +- **features**: +>This module takes an IP address MISP attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the location of this IP address. +> +>Please note that composite attributes domain|ip are also supported. +- **input**: +>An IP address MISP Attribute. +- **output**: +>Text containing information about the location of the IP address. +- **references**: +>https://www.maxmind.com/en/home +- **requirements**: +>A local copy of Maxmind's Geolite database + +----- + +#### [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) + +A hover module to check hashes against hashdd.com including NSLR dataset. +- **features**: +>This module takes a hash attribute as input to check its known level, using the hashdd API. This information is then displayed. +- **input**: +>A hash MISP attribute (md5). +- **output**: +>Text describing the known level of the hash in the hashdd databases. +- **references**: +>https://hashdd.com/ + +----- + +#### [intelmq_eventdb](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intelmq_eventdb.py) + + + +Module to access intelmqs eventdb. +- **features**: +>/!\ EXPERIMENTAL MODULE, some features may not work /!\ +> +>This module takes a domain, hostname, IP address or Autonomous system MISP attribute as input to query the IntelMQ database. The result of the query gives then additional information about the input. +- **input**: +>A hostname, domain, IP address or AS attribute. +- **output**: +>Text giving information about the input using IntelMQ database. +- **references**: +>https://github.com/certtools/intelmq, https://intelmq.readthedocs.io/en/latest/Developers-Guide/ +- **requirements**: +>psycopg2: Python library to support PostgreSQL, An access to the IntelMQ database (username, password, hostname and database reference) + +----- + +#### [ipasn](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) + +Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History). +- **features**: +>This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input. +- **input**: +>An IP address MISP attribute. +- **output**: +>Text describing additional information about the input after a query on the IPASN-history database. +- **references**: +>https://github.com/D4-project/IPASN-History +- **requirements**: +>pyipasnhistory: Python library to access IPASN-history instance + +----- + +#### [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) + +Module to query IPRep data for IP addresses. +- **features**: +>This module takes an IP address attribute as input and queries the database from packetmail.net to get some information about the reputation of the IP. +- **input**: +>An IP address MISP attribute. +- **output**: +>Text describing additional information about the input after a query on the IPRep API. +- **references**: +>https://github.com/mahesh557/packetmail +- **requirements**: +>An access to the packetmail API (apikey) + +----- + +#### [macaddress_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) + + + +MISP hover module for macaddress.io +- **features**: +>This module takes a MAC address attribute as input and queries macaddress.io for additional information. +> +>This information contains data about: +>- MAC address details +>- Vendor details +>- Block details +- **input**: +>MAC address MISP attribute. +- **output**: +>Text containing information on the MAC address fetched from a query on macaddress.io. +- **references**: +>https://macaddress.io/, https://github.com/CodeLineFi/maclookup-python +- **requirements**: +>maclookup: macaddress.io python library, An access to the macaddress.io API (apikey) + +----- + +#### [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) + + + +Module to process a query on Onyphe. +- **features**: +>This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted. +- **input**: +>A domain, hostname or IP address MISP attribute. +- **output**: +>MISP attributes fetched from the Onyphe query. +- **references**: +>https://www.onyphe.io/, https://github.com/sebdraven/pyonyphe +- **requirements**: +>onyphe python library, An access to the Onyphe API (apikey) + +----- + +#### [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) + + + +Module to process a full query on Onyphe. +- **features**: +>This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted. +> +>The parsing is here more advanced than the one on onyphe module, and is returning more attributes, since more fields of the query result are watched and parsed. +- **input**: +>A domain, hostname or IP address MISP attribute. +- **output**: +>MISP attributes fetched from the Onyphe query. +- **references**: +>https://www.onyphe.io/, https://github.com/sebdraven/pyonyphe +- **requirements**: +>onyphe python library, An access to the Onyphe API (apikey) + +----- + +#### [otx](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) + + + +Module to get information from AlienVault OTX. +- **features**: +>This module takes a MISP attribute as input to query the OTX Alienvault API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. +- **input**: +>A MISP attribute included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- md5 +>- sha1 +>- sha256 +>- sha512 +- **output**: +>MISP attributes mapped from the result of the query on OTX, included in the following list: +>- domain +>- ip-src +>- ip-dst +>- text +>- md5 +>- sha1 +>- sha256 +>- sha512 +>- email +- **references**: +>https://www.alienvault.com/open-threat-exchange +- **requirements**: +>An access to the OTX API (apikey) + +----- + +#### [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) + + + + +- **features**: +>The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register +- **input**: +>A MISP attribute included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- x509-fingerprint-sha1 +>- email-src +>- email-dst +>- target-email +>- whois-registrant-email +>- whois-registrant-phone +>- text +>- whois-registrant-name +>- whois-registrar +>- whois-creation-date +- **output**: +>MISP attributes mapped from the result of the query on PassiveTotal, included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- x509-fingerprint-sha1 +>- email-src +>- email-dst +>- target-email +>- whois-registrant-email +>- whois-registrant-phone +>- text +>- whois-registrant-name +>- whois-registrar +>- whois-creation-date +>- md5 +>- sha1 +>- sha256 +>- link +- **references**: +>https://www.passivetotal.org/register +- **requirements**: +>Passivetotal python library, An access to the PassiveTotal API (apikey) + +----- + +#### [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) + +Module to check an IPv4 address against known RBLs. +- **features**: +>This module takes an IP address attribute as input and queries multiple know Real-time Blackhost Lists to check if they have already seen this IP address. +> +>We display then all the information we get from those different sources. +- **input**: +>IP address attribute. +- **output**: +>Text with additional data from Real-time Blackhost Lists about the IP address. +- **references**: +>[RBLs list](https://github.com/MISP/misp-modules/blob/8817de476572a10a9c9d03258ec81ca70f3d926d/misp_modules/modules/expansion/rbl.py#L20) +- **requirements**: +>dnspython3: DNS python3 library + +----- + +#### [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) + +Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. +- **features**: +>The module takes an IP address as input and tries to find the hostname this IP address is resolved into. +> +>The address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8). +> +>Please note that composite MISP attributes containing IP addresses are supported as well. +- **input**: +>An IP address attribute. +- **output**: +>Hostname attribute the input is resolved into. +- **requirements**: +>DNS python library + +----- + +#### [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) + + + +An expansion modules for SecurityTrails. +- **features**: +>The module takes a domain, hostname or IP address attribute as input and queries the SecurityTrails API with it. +> +>Multiple parsing operations are then processed on the result of the query to extract a much information as possible. +> +>From this data extracted are then mapped MISP attributes. +- **input**: +>A domain, hostname or IP address attribute. +- **output**: +>MISP attributes resulting from the query on SecurityTrails API, included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- dns-soa-email +>- whois-registrant-email +>- whois-registrant-phone +>- whois-registrant-name +>- whois-registrar +>- whois-creation-date +>- domain +- **references**: +>https://securitytrails.com/ +- **requirements**: +>dnstrails python library, An access to the SecurityTrails API (apikey) + +----- + +#### [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) + + + +Module to query on Shodan. +- **features**: +>The module takes an IP address as input and queries the Shodan API to get some additional data about it. +- **input**: +>An IP address MISP attribute. +- **output**: +>Text with additional data about the input, resulting from the query on Shodan. +- **references**: +>https://www.shodan.io/ +- **requirements**: +>shodan python library, An access to the Shodan API (apikey) + +----- + +#### [sigma_queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) + + + +An expansion hover module to display the result of sigma queries. +- **features**: +>This module takes a Sigma rule attribute as input and tries all the different queries available to convert it into different formats recognized by SIEMs. +- **input**: +>A Sigma attribute. +- **output**: +>Text displaying results of queries on the Sigma attribute. +- **references**: +>https://github.com/Neo23x0/sigma/wiki +- **requirements**: +>Sigma python library + +----- + +#### [sigma_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) + + + +An expansion hover module to perform a syntax check on sigma rules. +- **features**: +>This module takes a Sigma rule attribute as input and performs a syntax check on it. +> +>It displays then that the rule is valid if it is the case, and the error related to the rule otherwise. +- **input**: +>A Sigma attribute. +- **output**: +>Text describing the validity of the Sigma rule. +- **references**: +>https://github.com/Neo23x0/sigma/wiki +- **requirements**: +>Sigma python library, Yaml python library + +----- + +#### [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) + +Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page. +- **features**: +>This module takes a link or url attribute as input and caches the related web page. It returns then a link of the cached page. +- **input**: +>A link or url attribute. +- **output**: +>A malware-sample attribute describing the cached page. +- **references**: +>https://github.com/adulau/url_archiver +- **requirements**: +>urlarchiver: python library to fetch and archive URL on the file-system + +----- + +#### [stix2_pattern_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) + + + +An expansion hover module to perform a syntax check on stix2 patterns. +- **features**: +>This module takes a STIX2 pattern attribute as input and performs a syntax check on it. +> +>It displays then that the rule is valid if it is the case, and the error related to the rule otherwise. +- **input**: +>A STIX2 pattern attribute. +- **output**: +>Text describing the validity of the STIX2 pattern. +- **references**: +>[STIX2.0 patterning specifications](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html) +- **requirements**: +>stix2patterns python library + +----- + +#### [threatcrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) + + + +Module to get information from ThreatCrowd. +- **features**: +>This module takes a MISP attribute as input and queries ThreatCrowd with it. +> +>The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. +- **input**: +>A MISP attribute included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- md5 +>- sha1 +>- sha256 +>- sha512 +>- whois-registrant-email +- **output**: +>MISP attributes mapped from the result of the query on ThreatCrowd, included in the following list: +>- domain +>- ip-src +>- ip-dst +>- text +>- md5 +>- sha1 +>- sha256 +>- sha512 +>- hostname +>- whois-registrant-email +- **references**: +>https://www.threatcrowd.org/ + +----- + +#### [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) + + + +Module to get information from ThreatMiner. +- **features**: +>This module takes a MISP attribute as input and queries ThreatMiner with it. +> +>The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. +- **input**: +>A MISP attribute included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- md5 +>- sha1 +>- sha256 +>- sha512 +- **output**: +>MISP attributes mapped from the result of the query on ThreatMiner, included in the following list: +>- domain +>- ip-src +>- ip-dst +>- text +>- md5 +>- sha1 +>- sha256 +>- sha512 +>- ssdeep +>- authentihash +>- filename +>- whois-registrant-email +>- url +>- link +- **references**: +>https://www.threatminer.org/ + +----- + +#### [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) + + + +An expansion module to query urlscan.io. +- **features**: +>This module takes a MISP attribute as input and queries urlscan.io with it. +> +>The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. +- **input**: +>A domain, hostname or url attribute. +- **output**: +>MISP attributes mapped from the result of the query on urlscan.io. +- **references**: +>https://urlscan.io/ +- **requirements**: +>An access to the urlscan.io API + +----- + +#### [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) + + + +Module to get information from virustotal. +- **features**: +>This module takes a MISP attribute as input and queries the VirusTotal API with it, in order to get additional data on the input attribute. +> +>Multiple recursive requests on the API can then be processed on some attributes found in the first request. A limit can be set to restrict the number of values to query again, and at the same time the number of request submitted to the API. +> +>This limit is important because the default user VirusTotal apikey only allows to process a certain nunmber of queries per minute. As a consequence it is recommended to have a larger number of requests or a private apikey. +> +>Data is then mapped into MISP attributes. +- **input**: +>A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute. +- **output**: +>MISP attributes mapped from the rersult of the query on VirusTotal API. +- **references**: +>https://www.virustotal.com/ +- **requirements**: +>An access to the VirusTotal API (apikey) + +----- + +#### [vmray_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) + + + +Module to submit a sample to VMRay. +- **features**: +>This module takes an attachment or malware-sample attribute as input to query the VMRay API. +> +>The sample contained within the attribute in then enriched with data from VMRay mapped into MISP attributes. +- **input**: +>An attachment or malware-sample attribute. +- **output**: +>MISP attributes mapped from the result of the query on VMRay API, included in the following list: +>- text +>- sha1 +>- sha256 +>- md5 +>- link +- **references**: +>https://www.vmray.com/ +- **requirements**: +>An access to the VMRay API (apikey & url) + +----- + +#### [vulndb](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) + + + +Module to query VulnDB (RiskBasedSecurity.com). +- **features**: +>This module takes a vulnerability attribute as input and queries VulnDB in order to get some additional data about it. +> +>The API gives the result of the query which can be displayed in the screen, and/or mapped into MISP attributes to add in the event. +- **input**: +>A vulnerability attribute. +- **output**: +>Additional data enriching the CVE input, fetched from VulnDB. +- **references**: +>https://vulndb.cyberriskanalytics.com/ +- **requirements**: +>An access to the VulnDB API (apikey, apisecret) + +----- + +#### [vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) + + + +An expansion hover module to expand information about CVE id using Vulners API. +- **features**: +>This module takes a vulnerability attribute as input and queries the Vulners API in order to get some additional data about it. +> +>The API then returns details about the vulnerability. +- **input**: +>A vulnerability attribute. +- **output**: +>Text giving additional information about the CVE in input. +- **references**: +>https://vulners.com/ +- **requirements**: +>Vulners python library, An access to the Vulners API + +----- + +#### [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) + +Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd). +- **features**: +>This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server). +- **input**: +>A domain or IP address attribute. +- **output**: +>Text describing the result of a whois request for the input value. +- **references**: +>https://github.com/rafiot/uwhoisd +- **requirements**: +>uwhois: A whois python library + +----- + +#### [wiki](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) + + + +An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis. +- **features**: +>This module takes a text attribute as input and queries the Wikidata API. If the text attribute is clear enough to define a specific term, the API returns a wikidata link in response. +- **input**: +>Text attribute. +- **output**: +>Text attribute. +- **references**: +>https://www.wikidata.org +- **requirements**: +>SPARQLWrapper python library + +----- + +#### [xforceexchange](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) + + + +An expansion module for IBM X-Force Exchange. +- **features**: +>This module takes a MISP attribute as input to query the X-Force API. The API returns then additional information known in their threats data, that is mapped into MISP attributes. +- **input**: +>A MISP attribute included in the following list: +>- ip-src +>- ip-dst +>- vulnerability +>- md5 +>- sha1 +>- sha256 +- **output**: +>MISP attributes mapped from the result of the query on X-Force Exchange. +- **references**: +>https://exchange.xforce.ibmcloud.com/ +- **requirements**: +>An access to the X-Force API (apikey) + +----- + +#### [yara_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) + + + +An expansion & hover module to translate any hash attribute into a yara rule. +- **features**: +>The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it. This YARA rule is also validated using the same method as in 'yara_syntax_validator' module. +>Both hover and expansion functionalities are supported with this module, where the hover part is displaying the resulting YARA rule and the expansion part allows you to add the rule as a new attribute, as usual with expansion modules. +- **input**: +>MISP Hash attribute (md5, sha1, sha256, imphash, or any of the composite attribute with filename and one of the previous hash type). +- **output**: +>YARA rule. +- **references**: +>https://virustotal.github.io/yara/, https://github.com/virustotal/yara-python +- **requirements**: +>yara-python python library + +----- + +#### [yara_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) + + + +An expansion hover module to perform a syntax check on if yara rules are valid or not. +- **features**: +>This modules simply takes a YARA rule as input, and checks its syntax. It returns then a confirmation if the syntax is valid, otherwise the syntax error is displayed. +- **input**: +>YARA rule attribute. +- **output**: +>Text to inform users if their rule is valid. +- **references**: +>http://virustotal.github.io/yara/ +- **requirements**: +>yara_python python library + +----- + +## Export Modules + +#### [cef_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) + +Module to export a MISP event in CEF format. +- **features**: +>The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in Common Event Format. +>Thus, there is no particular feature concerning MISP Events since any event can be exported. However, 4 configuration parameters recognized by CEF format are required and should be provided by users before exporting data: the device vendor, product and version, as well as the default severity of data. +- **input**: +>MISP Event attributes +- **output**: +>Common Event Format file +- **references**: +>https://community.softwaregrp.com/t5/ArcSight-Connectors/ArcSight-Common-Event-Format-CEF-Guide/ta-p/1589306?attachment-id=65537 + +----- + +#### [goamlexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) + + + +This module is used to export MISP events containing transaction objects into GoAML format. +- **features**: +>The module works as long as there is at least one transaction object in the Event. +> +>Then in order to have a valid GoAML document, please follow these guidelines: +>- For each transaction object, use either a bank-account, person, or legal-entity object to describe the origin of the transaction, and again one of them to describe the target of the transaction. +>- Create an object reference for both origin and target objects of the transaction. +>- A bank-account object needs a signatory, which is a person object, put as object reference of the bank-account. +>- A person can have an address, which is a geolocation object, put as object reference of the person. +> +>Supported relation types for object references that are recommended for each object are the folowing: +>- transaction: +> - 'from', 'from_my_client': Origin of the transaction - at least one of them is required. +> - 'to', 'to_my_client': Target of the transaction - at least one of them is required. +> - 'address': Location of the transaction - optional. +>- bank-account: +> - 'signatory': Signatory of a bank-account - the reference from bank-account to a signatory is required, but the relation-type is optional at the moment since this reference will always describe a signatory. +> - 'entity': Entity owning the bank account - optional. +>- person: +> - 'address': Address of a person - optional. +- **input**: +>MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target. +- **output**: +>GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities). +- **references**: +>http://goaml.unodc.org/ +- **requirements**: +>PyMISP, MISP objects + +----- + +#### [liteexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) + +Lite export of a MISP event. +- **features**: +>This module is simply producing a json MISP event format file, but exporting only Attributes from the Event. Thus, MISP Events exported with this module should have attributes that are not internal references, otherwise the resulting event would be empty. +- **input**: +>MISP Event attributes +- **output**: +>Lite MISP Event + +----- + +#### [nexthinkexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) + + + +Nexthink NXQL query export module +- **features**: +>This module export an event as Nexthink NXQL queries that can then be used in your own python3 tool or from wget/powershell +- **input**: +>MISP Event attributes +- **output**: +>Nexthink NXQL queries +- **references**: +>https://doc.nexthink.com/Documentation/Nexthink/latest/APIAndIntegrations/IntroducingtheWebAPIV2 + +----- + +#### [osqueryexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) + + + +OSQuery export of a MISP event. +- **features**: +>This module export an event as osquery queries that can be used in packs or in fleet management solution like Kolide. +- **input**: +>MISP Event attributes +- **output**: +>osquery SQL queries + +----- + +#### [pdfexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) + +Simple export of a MISP event to PDF. +- **features**: +>The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event. +- **input**: +>MISP Event +- **output**: +>MISP Event in a PDF file. +- **references**: +>https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html +- **requirements**: +>PyMISP, asciidoctor + +----- + +#### [testexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/testexport.py) + +Skeleton export module. + +----- + +#### [threatStream_misp_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) + + + +Module to export a structured CSV file for uploading to threatStream. +- **features**: +>The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatStream. +- **input**: +>MISP Event attributes +- **output**: +>ThreatStream CSV format file +- **references**: +>https://www.anomali.com/platform/threatstream, https://github.com/threatstream +- **requirements**: +>csv + +----- + +#### [threat_connect_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) + + + +Module to export a structured CSV file for uploading to ThreatConnect. +- **features**: +>The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatConnect. +>Users should then provide, as module configuration, the source of data they export, because it is required by the output format. +- **input**: +>MISP Event attributes +- **output**: +>ThreatConnect CSV format file +- **references**: +>https://www.threatconnect.com +- **requirements**: +>csv + +----- + +## Import Modules + +#### [csvimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) + +Module to import MISP attributes from a csv file. +- **features**: +>In order to parse data from a csv file, a header is required to let the module know which column is matching with known attribute fields / MISP types. +>This header is part of the configuration of the module and should be filled out in MISP plugin settings, each field separated by COMMAS. Fields that do not match with any type known in MISP can be ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment, '). +>There is also one type that is confused and can be either a MISP attribute type or an attribute field: 'comment'. In this case, using 'attrComment' specifies that the attribute field 'comment' should be considered, otherwise it will be considered as the MISP attribute type. +> +>For each MISP attribute type, an attribute is created. +>Attribute fields that are imported are the following: value, type, category, to-ids, distribution, comment, tag. +- **input**: +>CSV format file. +- **output**: +>MISP Event attributes +- **references**: +>https://tools.ietf.org/html/rfc4180, https://tools.ietf.org/html/rfc7111 +- **requirements**: +>PyMISP + +----- + +#### [cuckooimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) + + + +Module to import Cuckoo JSON. +- **features**: +>The module simply imports MISP Attributes from a Cuckoo JSON format file. There is thus no special feature to make it work. +- **input**: +>Cuckoo JSON file +- **output**: +>MISP Event attributes +- **references**: +>https://cuckoosandbox.org/, https://github.com/cuckoosandbox/cuckoo + +----- + +#### [email_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) + +Module to import emails in MISP. +- **features**: +>This module can be used to import e-mail text as well as attachments and urls. +>3 configuration parameters are then used to unzip attachments, guess zip attachment passwords, and extract urls: set each one of them to True or False to process or not the respective corresponding actions. +- **input**: +>E-mail file +- **output**: +>MISP Event attributes + +----- + +#### [goamlimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) + + + +Module to import MISP objects about financial transactions from GoAML files. +- **features**: +>Unlike the GoAML export module, there is here no special feature to import data from GoAML external files, since the module will import MISP Objects with their References on its own, as it is required for the export module to rebuild a valid GoAML document. +- **input**: +>GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities). +- **output**: +>MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target. +- **references**: +>http://goaml.unodc.org/ +- **requirements**: +>PyMISP + +----- + +#### [mispjson](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/mispjson.py) + +Module to import MISP JSON format for merging MISP events. +- **features**: +>The module simply imports MISP Attributes from an other MISP Event in order to merge events together. There is thus no special feature to make it work. +- **input**: +>MISP Event +- **output**: +>MISP Event attributes + +----- + +#### [ocr](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) + +Optical Character Recognition (OCR) module for MISP. +- **features**: +>The module tries to recognize some text from an image and import the result as a freetext attribute, there is then no special feature asked to users to make it work. +- **input**: +>Image +- **output**: +>freetext MISP attribute + +----- + +#### [openiocimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) + +Module to import OpenIOC packages. +- **features**: +>The module imports MISP Attributes from OpenIOC packages, there is then no special feature for users to make it work. +- **input**: +>OpenIOC packages +- **output**: +>MISP Event attributes +- **references**: +>https://www.fireeye.com/blog/threat-research/2013/10/openioc-basics.html +- **requirements**: +>PyMISP + +----- + +#### [threatanalyzer_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) + +Module to import ThreatAnalyzer archive.zip / analysis.json files. +- **features**: +>The module imports MISP Attributes from a ThreatAnalyzer format file. This file can be either ZIP, or JSON format. +>There is by the way no special feature for users to make the module work. +- **input**: +>ThreatAnalyzer format file +- **output**: +>MISP Event attributes +- **references**: +>https://www.threattrack.com/malware-analysis.aspx + +----- + +#### [vmray_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) + + + +Module to import VMRay (VTI) results. +- **features**: +>The module imports MISP Attributes from VMRay format, using the VMRay api. +>Users should then provide as the module configuration the API Key as well as the server url in order to fetch their data to import. +- **input**: +>VMRay format +- **output**: +>MISP Event attributes +- **references**: +>https://www.vmray.com/ +- **requirements**: +>vmray_rest_api + +----- diff --git a/doc/documentation.md b/doc/documentation.md deleted file mode 100644 index 31f09ed..0000000 --- a/doc/documentation.md +++ /dev/null @@ -1,1243 +0,0 @@ -# MISP modules documentation - -## Expansion Modules - -#### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) - -Query BGP Ranking (https://bgpranking-ng.circl.lu/). -- **features**: ->The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking. -> -> -- **input**: ->Autonomous system number. -- **output**: ->Text containing a description of the ASN, its history, and the position in BGP Ranking. -- **references**: ->https://github.com/D4-project/BGP-Ranking/ -- **requirements**: ->pybgpranking python library - ------ - -#### [btc](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc.py) - - - -An expansion hover module to get a blockchain balance from a BTC address in MISP. -- **input**: ->btc address attribute. -- **output**: ->Text to describe the blockchain balance and the transactions related to the btc address in input. - ------ - -#### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) - - - -Module to access CIRCL Passive DNS. -- **features**: ->This module takes a hostname, domain or ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive DNS REST API to get and display information about this input. -> ->To make it work a username and a password are thus required to authenticate to the CIRCL Passive DNS API. -- **input**: ->Hostname, domain, or ip-address attribute. -- **ouput**: ->Text describing passive DNS information related to the input attribute. -- **references**: ->https://www.circl.lu/services/passive-dns/, https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/ -- **requirements**: ->pypdns: Passive DNS python library, A CIRCL passive DNS account with username & password - ------ - -#### [circl_passivessl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) - - - -Modules to access CIRCL Passive SSL. -- **features**: ->This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive SSL REST API to get and display information about this input. -> ->To make it work a username and a password are thus required to authenticate to the CIRCL Passive SSL API. -- **input**: ->Ip-address attribute. -- **output**: ->Text describing passive SSL information related to the input attribute. -- **references**: ->https://www.circl.lu/services/passive-ssl/ -- **requirements**: ->pypssl: Passive SSL python library, A CIRCL passive SSL account with username & password - ------ - -#### [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) - -Module to expand country codes. -- **features**: ->The module takes a domain or a hostname as input, and returns the country it belongs to. -> ->For non country domains, a list of the most common possible extensions is used. -- **input**: ->Hostname or domain attribute. -- **output**: ->Text with the country code the input belongs to. - ------ - -#### [crowdstrike_falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) - - - -Module to query Crowdstrike Falcon. -- **features**: ->This module takes a MISP attribute as input to query a CrowdStrike Falcon API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. -> ->Please note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported. -- **input**: ->A MISP attribute included in the following list: ->- domain ->- email-attachment ->- email-dst ->- email-reply-to ->- email-src ->- email-subject ->- filename ->- hostname ->- ip-src ->- ip-dst ->- md5 ->- mutex ->- regkey ->- sha1 ->- sha256 ->- uri ->- url ->- user-agent ->- whois-registrant-email ->- x509-fingerprint-md5 -- **output**: ->MISP attributes mapped after the CrowdStrike API has been queried, included in the following list: ->- hostname ->- email-src ->- email-subject ->- filename ->- md5 ->- sha1 ->- sha256 ->- ip-dst ->- ip-dst ->- mutex ->- regkey ->- url ->- user-agent ->- x509-fingerprint-md5 -- **references**: ->https://www.crowdstrike.com/products/crowdstrike-falcon-faq/ -- **requirements**: ->A CrowdStrike API access (API id & key) - ------ - -#### [cve](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) - - - -An expansion hover module to expand information about CVE id. -- **features**: ->The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to get information about the vulnerability as it is described in the list of CVEs. -- **input**: ->Vulnerability attribute. -- **output**: ->Text giving information about the CVE related to the Vulnerability. -- **references**: ->https://cve.circl.lu/, https://cve.mitre.org/ - ------ - -#### [dbl_spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) - - - -Module to check Spamhaus DBL for a domain name. -- **features**: ->This modules takes a domain or a hostname in input and queries the Domain Block List provided by Spamhaus to determine what kind of domain it is. -> ->DBL then returns a response code corresponding to a certain classification of the domain we display. If the queried domain is not in the list, it is also mentionned. -> ->Please note that composite MISP attributes containing domain or hostname are supported as well. -- **input**: ->Domain or hostname attribute. -- **output**: ->Information about the nature of the input. -- **references**: ->https://www.spamhaus.org/faq/section/Spamhaus%20DBL -- **requirements**: ->dnspython3: DNS python3 library - ------ - -#### [dns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) - -A simple DNS expansion service to resolve IP address from domain MISP attributes. -- **features**: ->The module takes a domain of hostname attribute as input, and tries to resolve it. If no error is encountered, the IP address that resolves the domain is returned, otherwise the origin of the error is displayed. -> ->The address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8). -> ->Please note that composite MISP attributes containing domain or hostname are supported as well. -- **input**: ->Domain or hostname attribute. -- **output**: ->IP address resolving the input. -- **requirements**: ->dnspython3: DNS python3 library - ------ - -#### [domaintools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) - - - -DomainTools MISP expansion module. -- **features**: ->This module takes a MISP attribute as input to query the Domaintools API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. -> ->Please note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported. -- **input**: ->A MISP attribute included in the following list: ->- domain ->- hostname ->- email-src ->- email-dst ->- target-email ->- whois-registrant-email ->- whois-registrant-name ->- whois-registrant-phone ->- ip-src ->- ip-dst -- **output**: ->MISP attributes mapped after the Domaintools API has been queried, included in the following list: ->- whois-registrant-email ->- whois-registrant-phone ->- whois-registrant-name ->- whois-registrar ->- whois-creation-date ->- text ->- domain -- **references**: ->https://www.domaintools.com/ -- **requirements**: ->Domaintools python library, A Domaintools API access (username & apikey) - ------ - -#### [eupi](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) - - - -A module to query the Phishing Initiative service (https://phishing-initiative.lu). -- **features**: ->This module takes a domain, hostname or url MISP attribute as input to query the Phishing Initiative API. The API returns then the result of the query with some information about the value queried. -> ->Please note that composite attributes containing domain or hostname are also supported. -- **input**: ->A domain, hostname or url MISP attribute. -- **output**: ->Text containing information about the input, resulting from the query on Phishing Initiative. -- **references**: ->https://phishing-initiative.eu/?lang=en -- **requirements**: ->pyeupi: eupi python library, An access to the Phishing Initiative API (apikey & url) - ------ - -#### [farsight_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) - - - -Module to access Farsight DNSDB Passive DNS. -- **features**: ->This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. The API returns then the result of the query with some information about the value queried. -- **input**: ->A domain, hostname or IP address MISP attribute. -- **output**: ->Text containing information about the input, resulting from the query on the Farsight Passive DNS API. -- **references**: ->https://www.farsightsecurity.com/ -- **requirements**: ->An access to the Farsight Passive DNS API (apikey) - ------ - -#### [geoip_country](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) - - - -Module to query a local copy of Maxmind's Geolite database. -- **features**: ->This module takes an IP address MISP attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the location of this IP address. -> ->Please note that composite attributes domain|ip are also supported. -- **input**: ->An IP address MISP Attribute. -- **output**: ->Text containing information about the location of the IP address. -- **references**: ->https://www.maxmind.com/en/home -- **requirements**: ->A local copy of Maxmind's Geolite database - ------ - -#### [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) - -A hover module to check hashes against hashdd.com including NSLR dataset. -- **features**: ->This module takes a hash attribute as input to check its known level, using the hashdd API. This information is then displayed. -- **input**: ->A hash MISP attribute (md5). -- **output**: ->Text describing the known level of the hash in the hashdd databases. -- **references**: ->https://hashdd.com/ - ------ - -#### [intelmq_eventdb](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intelmq_eventdb.py) - - - -Module to access intelmqs eventdb. -- **features**: ->/!\ EXPERIMENTAL MODULE, some features may not work /!\ -> ->This module takes a domain, hostname, IP address or Autonomous system MISP attribute as input to query the IntelMQ database. The result of the query gives then additional information about the input. -- **input**: ->A hostname, domain, IP address or AS attribute. -- **output**: ->Text giving information about the input using IntelMQ database. -- **references**: ->https://github.com/certtools/intelmq, https://intelmq.readthedocs.io/en/latest/Developers-Guide/ -- **requirements**: ->psycopg2: Python library to support PostgreSQL, An access to the IntelMQ database (username, password, hostname and database reference) - ------ - -#### [ipasn](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) - -Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History). -- **features**: ->This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input. -- **input**: ->An IP address MISP attribute. -- **output**: ->Text describing additional information about the input after a query on the IPASN-history database. -- **references**: ->https://github.com/D4-project/IPASN-History -- **requirements**: ->pyipasnhistory: Python library to access IPASN-history instance - ------ - -#### [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) - -Module to query IPRep data for IP addresses. -- **features**: ->This module takes an IP address attribute as input and queries the database from packetmail.net to get some information about the reputation of the IP. -- **input**: ->An IP address MISP attribute. -- **output**: ->Text describing additional information about the input after a query on the IPRep API. -- **references**: ->https://github.com/mahesh557/packetmail -- **requirements**: ->An access to the packetmail API (apikey) - ------ - -#### [macaddress_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) - - - -MISP hover module for macaddress.io -- **features**: ->This module takes a MAC address attribute as input and queries macaddress.io for additional information. -> ->This information contains data about: ->- MAC address details ->- Vendor details ->- Block details -- **input**: ->MAC address MISP attribute. -- **output**: ->Text containing information on the MAC address fetched from a query on macaddress.io. -- **references**: ->https://macaddress.io/, https://github.com/CodeLineFi/maclookup-python -- **requirements**: ->maclookup: macaddress.io python library, An access to the macaddress.io API (apikey) - ------ - -#### [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) - - - -Module to process a query on Onyphe. -- **features**: ->This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted. -- **input**: ->A domain, hostname or IP address MISP attribute. -- **output**: ->MISP attributes fetched from the Onyphe query. -- **references**: ->https://www.onyphe.io/, https://github.com/sebdraven/pyonyphe -- **requirements**: ->onyphe python library, An access to the Onyphe API (apikey) - ------ - -#### [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) - - - -Module to process a full query on Onyphe. -- **features**: ->This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted. -> ->The parsing is here more advanced than the one on onyphe module, and is returning more attributes, since more fields of the query result are watched and parsed. -- **input**: ->A domain, hostname or IP address MISP attribute. -- **output**: ->MISP attributes fetched from the Onyphe query. -- **references**: ->https://www.onyphe.io/, https://github.com/sebdraven/pyonyphe -- **requirements**: ->onyphe python library, An access to the Onyphe API (apikey) - ------ - -#### [otx](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) - - - -Module to get information from AlienVault OTX. -- **features**: ->This module takes a MISP attribute as input to query the OTX Alienvault API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. -- **input**: ->A MISP attribute included in the following list: ->- hostname ->- domain ->- ip-src ->- ip-dst ->- md5 ->- sha1 ->- sha256 ->- sha512 -- **output**: ->MISP attributes mapped from the result of the query on OTX, included in the following list: ->- domain ->- ip-src ->- ip-dst ->- text ->- md5 ->- sha1 ->- sha256 ->- sha512 ->- email -- **references**: ->https://www.alienvault.com/open-threat-exchange -- **requirements**: ->An access to the OTX API (apikey) - ------ - -#### [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) - - - - -- **features**: ->The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register -- **input**: ->A MISP attribute included in the following list: ->- hostname ->- domain ->- ip-src ->- ip-dst ->- x509-fingerprint-sha1 ->- email-src ->- email-dst ->- target-email ->- whois-registrant-email ->- whois-registrant-phone ->- text ->- whois-registrant-name ->- whois-registrar ->- whois-creation-date -- **output**: ->MISP attributes mapped from the result of the query on PassiveTotal, included in the following list: ->- hostname ->- domain ->- ip-src ->- ip-dst ->- x509-fingerprint-sha1 ->- email-src ->- email-dst ->- target-email ->- whois-registrant-email ->- whois-registrant-phone ->- text ->- whois-registrant-name ->- whois-registrar ->- whois-creation-date ->- md5 ->- sha1 ->- sha256 ->- link -- **references**: ->https://www.passivetotal.org/register -- **requirements**: ->Passivetotal python library, An access to the PassiveTotal API (apikey) - ------ - -#### [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) - -Module to check an IPv4 address against known RBLs. -- **features**: ->This module takes an IP address attribute as input and queries multiple know Real-time Blackhost Lists to check if they have already seen this IP address. -> ->We display then all the information we get from those different sources. -- **input**: ->IP address attribute. -- **output**: ->Text with additional data from Real-time Blackhost Lists about the IP address. -- **references**: ->[RBLs list](https://github.com/MISP/misp-modules/blob/8817de476572a10a9c9d03258ec81ca70f3d926d/misp_modules/modules/expansion/rbl.py#L20) -- **requirements**: ->dnspython3: DNS python3 library - ------ - -#### [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) - -Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. -- **features**: ->The module takes an IP address as input and tries to find the hostname this IP address is resolved into. -> ->The address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8). -> ->Please note that composite MISP attributes containing IP addresses are supported as well. -- **input**: ->An IP address attribute. -- **output**: ->Hostname attribute the input is resolved into. -- **requirements**: ->DNS python library - ------ - -#### [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) - - - -An expansion modules for SecurityTrails. -- **features**: ->The module takes a domain, hostname or IP address attribute as input and queries the SecurityTrails API with it. -> ->Multiple parsing operations are then processed on the result of the query to extract a much information as possible. -> ->From this data extracted are then mapped MISP attributes. -- **input**: ->A domain, hostname or IP address attribute. -- **output**: ->MISP attributes resulting from the query on SecurityTrails API, included in the following list: ->- hostname ->- domain ->- ip-src ->- ip-dst ->- dns-soa-email ->- whois-registrant-email ->- whois-registrant-phone ->- whois-registrant-name ->- whois-registrar ->- whois-creation-date ->- domain -- **references**: ->https://securitytrails.com/ -- **requirements**: ->dnstrails python library, An access to the SecurityTrails API (apikey) - ------ - -#### [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) - - - -Module to query on Shodan. -- **features**: ->The module takes an IP address as input and queries the Shodan API to get some additional data about it. -- **input**: ->An IP address MISP attribute. -- **output**: ->Text with additional data about the input, resulting from the query on Shodan. -- **references**: ->https://www.shodan.io/ -- **requirements**: ->shodan python library, An access to the Shodan API (apikey) - ------ - -#### [sigma_queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) - - - -An expansion hover module to display the result of sigma queries. -- **features**: ->This module takes a Sigma rule attribute as input and tries all the different queries available to convert it into different formats recognized by SIEMs. -- **input**: ->A Sigma attribute. -- **output**: ->Text displaying results of queries on the Sigma attribute. -- **references**: ->https://github.com/Neo23x0/sigma/wiki -- **requirements**: ->Sigma python library - ------ - -#### [sigma_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) - - - -An expansion hover module to perform a syntax check on sigma rules. -- **features**: ->This module takes a Sigma rule attribute as input and performs a syntax check on it. -> ->It displays then that the rule is valid if it is the case, and the error related to the rule otherwise. -- **input**: ->A Sigma attribute. -- **output**: ->Text describing the validity of the Sigma rule. -- **references**: ->https://github.com/Neo23x0/sigma/wiki -- **requirements**: ->Sigma python library, Yaml python library - ------ - -#### [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) - -Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page. -- **features**: ->This module takes a link or url attribute as input and caches the related web page. It returns then a link of the cached page. -- **input**: ->A link or url attribute. -- **output**: ->A malware-sample attribute describing the cached page. -- **references**: ->https://github.com/adulau/url_archiver -- **requirements**: ->urlarchiver: python library to fetch and archive URL on the file-system - ------ - -#### [stix2_pattern_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - - - -An expansion hover module to perform a syntax check on stix2 patterns. -- **features**: ->This module takes a STIX2 pattern attribute as input and performs a syntax check on it. -> ->It displays then that the rule is valid if it is the case, and the error related to the rule otherwise. -- **input**: ->A STIX2 pattern attribute. -- **output**: ->Text describing the validity of the STIX2 pattern. -- **references**: ->[STIX2.0 patterning specifications](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html) -- **requirements**: ->stix2patterns python library - ------ - -#### [threatcrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) - - - -Module to get information from ThreatCrowd. -- **features**: ->This module takes a MISP attribute as input and queries ThreatCrowd with it. -> ->The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. -- **input**: ->A MISP attribute included in the following list: ->- hostname ->- domain ->- ip-src ->- ip-dst ->- md5 ->- sha1 ->- sha256 ->- sha512 ->- whois-registrant-email -- **output**: ->MISP attributes mapped from the result of the query on ThreatCrowd, included in the following list: ->- domain ->- ip-src ->- ip-dst ->- text ->- md5 ->- sha1 ->- sha256 ->- sha512 ->- hostname ->- whois-registrant-email -- **references**: ->https://www.threatcrowd.org/ - ------ - -#### [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) - - - -Module to get information from ThreatMiner. -- **features**: ->This module takes a MISP attribute as input and queries ThreatMiner with it. -> ->The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. -- **input**: ->A MISP attribute included in the following list: ->- hostname ->- domain ->- ip-src ->- ip-dst ->- md5 ->- sha1 ->- sha256 ->- sha512 -- **output**: ->MISP attributes mapped from the result of the query on ThreatMiner, included in the following list: ->- domain ->- ip-src ->- ip-dst ->- text ->- md5 ->- sha1 ->- sha256 ->- sha512 ->- ssdeep ->- authentihash ->- filename ->- whois-registrant-email ->- url ->- link -- **references**: ->https://www.threatminer.org/ - ------ - -#### [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) - - - -An expansion module to query urlscan.io. -- **features**: ->This module takes a MISP attribute as input and queries urlscan.io with it. -> ->The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. -- **input**: ->A domain, hostname or url attribute. -- **output**: ->MISP attributes mapped from the result of the query on urlscan.io. -- **references**: ->https://urlscan.io/ -- **requirements**: ->An access to the urlscan.io API - ------ - -#### [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) - - - -Module to get information from virustotal. -- **features**: ->This module takes a MISP attribute as input and queries the VirusTotal API with it, in order to get additional data on the input attribute. -> ->Multiple recursive requests on the API can then be processed on some attributes found in the first request. A limit can be set to restrict the number of values to query again, and at the same time the number of request submitted to the API. -> ->This limit is important because the default user VirusTotal apikey only allows to process a certain nunmber of queries per minute. As a consequence it is recommended to have a larger number of requests or a private apikey. -> ->Data is then mapped into MISP attributes. -- **input**: ->A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute. -- **output**: ->MISP attributes mapped from the rersult of the query on VirusTotal API. -- **references**: ->https://www.virustotal.com/ -- **requirements**: ->An access to the VirusTotal API (apikey) - ------ - -#### [vmray_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) - - - -Module to submit a sample to VMRay. -- **features**: ->This module takes an attachment or malware-sample attribute as input to query the VMRay API. -> ->The sample contained within the attribute in then enriched with data from VMRay mapped into MISP attributes. -- **input**: ->An attachment or malware-sample attribute. -- **output**: ->MISP attributes mapped from the result of the query on VMRay API, included in the following list: ->- text ->- sha1 ->- sha256 ->- md5 ->- link -- **references**: ->https://www.vmray.com/ -- **requirements**: ->An access to the VMRay API (apikey & url) - ------ - -#### [vulndb](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) - - - -Module to query VulnDB (RiskBasedSecurity.com). -- **features**: ->This module takes a vulnerability attribute as input and queries VulnDB in order to get some additional data about it. -> ->The API gives the result of the query which can be displayed in the screen, and/or mapped into MISP attributes to add in the event. -- **input**: ->A vulnerability attribute. -- **output**: ->Additional data enriching the CVE input, fetched from VulnDB. -- **references**: ->https://vulndb.cyberriskanalytics.com/ -- **requirements**: ->An access to the VulnDB API (apikey, apisecret) - ------ - -#### [vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) - - - -An expansion hover module to expand information about CVE id using Vulners API. -- **features**: ->This module takes a vulnerability attribute as input and queries the Vulners API in order to get some additional data about it. -> ->The API then returns details about the vulnerability. -- **input**: ->A vulnerability attribute. -- **output**: ->Text giving additional information about the CVE in input. -- **references**: ->https://vulners.com/ -- **requirements**: ->Vulners python library, An access to the Vulners API - ------ - -#### [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) - -Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd). -- **features**: ->This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server). -- **input**: ->A domain or IP address attribute. -- **output**: ->Text describing the result of a whois request for the input value. -- **references**: ->https://github.com/rafiot/uwhoisd -- **requirements**: ->uwhois: A whois python library - ------ - -#### [wiki](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) - - - -An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis. -- **features**: ->This module takes a text attribute as input and queries the Wikidata API. If the text attribute is clear enough to define a specific term, the API returns a wikidata link in response. -- **input**: ->Text attribute. -- **output**: ->Text attribute. -- **references**: ->https://www.wikidata.org -- **requirements**: ->SPARQLWrapper python library - ------ - -#### [xforceexchange](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) - - - -An expansion module for IBM X-Force Exchange. -- **features**: ->This module takes a MISP attribute as input to query the X-Force API. The API returns then additional information known in their threats data, that is mapped into MISP attributes. -- **input**: ->A MISP attribute included in the following list: ->- ip-src ->- ip-dst ->- vulnerability ->- md5 ->- sha1 ->- sha256 -- **output**: ->MISP attributes mapped from the result of the query on X-Force Exchange. -- **references**: ->https://exchange.xforce.ibmcloud.com/ -- **requirements**: ->An access to the X-Force API (apikey) - ------ - -#### [yara_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) - - - -An expansion & hover module to translate any hash attribute into a yara rule. -- **features**: ->The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it. This YARA rule is also validated using the same method as in 'yara_syntax_validator' module. ->Both hover and expansion functionalities are supported with this module, where the hover part is displaying the resulting YARA rule and the expansion part allows you to add the rule as a new attribute, as usual with expansion modules. -- **input**: ->MISP Hash attribute (md5, sha1, sha256, imphash, or any of the composite attribute with filename and one of the previous hash type). -- **output**: ->YARA rule. -- **references**: ->https://virustotal.github.io/yara/, https://github.com/virustotal/yara-python -- **requirements**: ->yara-python python library - ------ - -#### [yara_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) - - - -An expansion hover module to perform a syntax check on if yara rules are valid or not. -- **features**: ->This modules simply takes a YARA rule as input, and checks its syntax. It returns then a confirmation if the syntax is valid, otherwise the syntax error is displayed. -- **input**: ->YARA rule attribute. -- **output**: ->Text to inform users if their rule is valid. -- **references**: ->http://virustotal.github.io/yara/ -- **requirements**: ->yara_python python library - ------ - -## Export Modules - -#### [cef_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) - -Module to export a MISP event in CEF format. -- **features**: ->The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in Common Event Format. ->Thus, there is no particular feature concerning MISP Events since any event can be exported. However, 4 configuration parameters recognized by CEF format are required and should be provided by users before exporting data: the device vendor, product and version, as well as the default severity of data. -- **input**: ->MISP Event attributes -- **output**: ->Common Event Format file -- **references**: ->https://community.softwaregrp.com/t5/ArcSight-Connectors/ArcSight-Common-Event-Format-CEF-Guide/ta-p/1589306?attachment-id=65537 - ------ - -#### [goamlexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) - - - -This module is used to export MISP events containing transaction objects into GoAML format. -- **features**: ->The module works as long as there is at least one transaction object in the Event. -> ->Then in order to have a valid GoAML document, please follow these guidelines: ->- For each transaction object, use either a bank-account, person, or legal-entity object to describe the origin of the transaction, and again one of them to describe the target of the transaction. ->- Create an object reference for both origin and target objects of the transaction. ->- A bank-account object needs a signatory, which is a person object, put as object reference of the bank-account. ->- A person can have an address, which is a geolocation object, put as object reference of the person. -> ->Supported relation types for object references that are recommended for each object are the folowing: ->- transaction: -> - 'from', 'from_my_client': Origin of the transaction - at least one of them is required. -> - 'to', 'to_my_client': Target of the transaction - at least one of them is required. -> - 'address': Location of the transaction - optional. ->- bank-account: -> - 'signatory': Signatory of a bank-account - the reference from bank-account to a signatory is required, but the relation-type is optional at the moment since this reference will always describe a signatory. -> - 'entity': Entity owning the bank account - optional. ->- person: -> - 'address': Address of a person - optional. -- **input**: ->MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target. -- **output**: ->GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities). -- **references**: ->http://goaml.unodc.org/ -- **requirements**: ->PyMISP, MISP objects - ------ - -#### [liteexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) - -Lite export of a MISP event. -- **features**: ->This module is simply producing a json MISP event format file, but exporting only Attributes from the Event. Thus, MISP Events exported with this module should have attributes that are not internal references, otherwise the resulting event would be empty. -- **input**: ->MISP Event attributes -- **output**: ->Lite MISP Event - ------ - -#### [nexthinkexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) - - - -Nexthink NXQL query export module -- **features**: ->This module export an event as Nexthink NXQL queries that can then be used in your own python3 tool or from wget/powershell -- **input**: ->MISP Event attributes -- **output**: ->Nexthink NXQL queries -- **references**: ->https://doc.nexthink.com/Documentation/Nexthink/latest/APIAndIntegrations/IntroducingtheWebAPIV2 - ------ - -#### [osqueryexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) - - - -OSQuery export of a MISP event. -- **features**: ->This module export an event as osquery queries that can be used in packs or in fleet management solution like Kolide. -- **input**: ->MISP Event attributes -- **output**: ->osquery SQL queries - ------ - -#### [pdfexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) - -Simple export of a MISP event to PDF. -- **features**: ->The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event. -- **input**: ->MISP Event -- **output**: ->MISP Event in a PDF file. -- **references**: ->https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html -- **requirements**: ->PyMISP, asciidoctor - ------ - -#### [testexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/testexport.py) - -Skeleton export module. - ------ - -#### [threatStream_misp_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) - - - -Module to export a structured CSV file for uploading to threatStream. -- **features**: ->The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatStream. -- **input**: ->MISP Event attributes -- **output**: ->ThreatStream CSV format file -- **references**: ->https://www.anomali.com/platform/threatstream, https://github.com/threatstream -- **requirements**: ->csv - ------ - -#### [threat_connect_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) - - - -Module to export a structured CSV file for uploading to ThreatConnect. -- **features**: ->The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatConnect. ->Users should then provide, as module configuration, the source of data they export, because it is required by the output format. -- **input**: ->MISP Event attributes -- **output**: ->ThreatConnect CSV format file -- **references**: ->https://www.threatconnect.com -- **requirements**: ->csv - ------ - -## Import Modules - -#### [csvimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) - -Module to import MISP attributes from a csv file. -- **features**: ->In order to parse data from a csv file, a header is required to let the module know which column is matching with known attribute fields / MISP types. ->This header is part of the configuration of the module and should be filled out in MISP plugin settings, each field separated by COMMAS. Fields that do not match with any type known in MISP can be ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment, '). ->There is also one type that is confused and can be either a MISP attribute type or an attribute field: 'comment'. In this case, using 'attrComment' specifies that the attribute field 'comment' should be considered, otherwise it will be considered as the MISP attribute type. -> ->For each MISP attribute type, an attribute is created. ->Attribute fields that are imported are the following: value, type, category, to-ids, distribution, comment, tag. -- **input**: ->CSV format file. -- **output**: ->MISP Event attributes -- **references**: ->https://tools.ietf.org/html/rfc4180, https://tools.ietf.org/html/rfc7111 -- **requirements**: ->PyMISP - ------ - -#### [cuckooimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) - - - -Module to import Cuckoo JSON. -- **features**: ->The module simply imports MISP Attributes from a Cuckoo JSON format file. There is thus no special feature to make it work. -- **input**: ->Cuckoo JSON file -- **output**: ->MISP Event attributes -- **references**: ->https://cuckoosandbox.org/, https://github.com/cuckoosandbox/cuckoo - ------ - -#### [email_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) - -Module to import emails in MISP. -- **features**: ->This module can be used to import e-mail text as well as attachments and urls. ->3 configuration parameters are then used to unzip attachments, guess zip attachment passwords, and extract urls: set each one of them to True or False to process or not the respective corresponding actions. -- **input**: ->E-mail file -- **output**: ->MISP Event attributes - ------ - -#### [goamlimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) - - - -Module to import MISP objects about financial transactions from GoAML files. -- **features**: ->Unlike the GoAML export module, there is here no special feature to import data from GoAML external files, since the module will import MISP Objects with their References on its own, as it is required for the export module to rebuild a valid GoAML document. -- **input**: ->GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities). -- **output**: ->MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target. -- **references**: ->http://goaml.unodc.org/ -- **requirements**: ->PyMISP - ------ - -#### [mispjson](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/mispjson.py) - -Module to import MISP JSON format for merging MISP events. -- **features**: ->The module simply imports MISP Attributes from an other MISP Event in order to merge events together. There is thus no special feature to make it work. -- **input**: ->MISP Event -- **output**: ->MISP Event attributes - ------ - -#### [ocr](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) - -Optical Character Recognition (OCR) module for MISP. -- **features**: ->The module tries to recognize some text from an image and import the result as a freetext attribute, there is then no special feature asked to users to make it work. -- **input**: ->Image -- **output**: ->freetext MISP attribute - ------ - -#### [openiocimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) - -Module to import OpenIOC packages. -- **features**: ->The module imports MISP Attributes from OpenIOC packages, there is then no special feature for users to make it work. -- **input**: ->OpenIOC packages -- **output**: ->MISP Event attributes -- **references**: ->https://www.fireeye.com/blog/threat-research/2013/10/openioc-basics.html -- **requirements**: ->PyMISP - ------ - -#### [threatanalyzer_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) - -Module to import ThreatAnalyzer archive.zip / analysis.json files. -- **features**: ->The module imports MISP Attributes from a ThreatAnalyzer format file. This file can be either ZIP, or JSON format. ->There is by the way no special feature for users to make the module work. -- **input**: ->ThreatAnalyzer format file -- **output**: ->MISP Event attributes -- **references**: ->https://www.threattrack.com/malware-analysis.aspx - ------ - -#### [vmray_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) - - - -Module to import VMRay (VTI) results. -- **features**: ->The module imports MISP Attributes from VMRay format, using the VMRay api. ->Users should then provide as the module configuration the API Key as well as the server url in order to fetch their data to import. -- **input**: ->VMRay format -- **output**: ->MISP Event attributes -- **references**: ->https://www.vmray.com/ -- **requirements**: ->vmray_rest_api - ------ diff --git a/doc/expansion/btc_scam_check.json b/doc/expansion/btc_scam_check.json new file mode 100644 index 0000000..44fce03 --- /dev/null +++ b/doc/expansion/btc_scam_check.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.", + "requirements": ["dnspython3: dns python library"], + "features": "The module queries a dns blacklist directly with the bitcoin address and get a response if the address has been abused.", + "logo": "logos/bitcoin.png", + "input": "btc address attribute.", + "output" : "Text to indicate if the BTC address has been abused.", + "references": ["https://btcblack.it/"] +} diff --git a/doc/expansion/btc.json b/doc/expansion/btc_steroids.json similarity index 100% rename from doc/expansion/btc.json rename to doc/expansion/btc_steroids.json diff --git a/doc/generate_documentation.py b/doc/generate_documentation.py index 980ddf6..caef84e 100644 --- a/doc/generate_documentation.py +++ b/doc/generate_documentation.py @@ -30,7 +30,7 @@ def generate_doc(root_path): value = ', '.join(value) if isinstance(value, list) else '{}'.format(value.replace('\n', '\n>')) markdown.append('- **{}**:\n>{}\n'.format(field, value)) markdown.append('\n-----\n') - with open('documentation.md', 'w') as w: + with open('README.md', 'w') as w: w.write(''.join(markdown)) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 559e5aa..2507226 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,4 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471'] + 'intel471', 'btc_scam_check'] diff --git a/misp_modules/modules/expansion/btc_scam_check.py b/misp_modules/modules/expansion/btc_scam_check.py new file mode 100644 index 0000000..b49414d --- /dev/null +++ b/misp_modules/modules/expansion/btc_scam_check.py @@ -0,0 +1,43 @@ +import json +import sys + +try: + from dns.resolver import Resolver, NXDOMAIN + from dns.name import LabelTooLong + resolver = Resolver() + resolver.timeout = 1 + resolver.lifetime = 1 +except ImportError: + sys.exit("dnspython3 in missing. use 'pip install dnspython3' to install it.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['btc'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'Checks if a BTC address is referenced as a scam.', + 'module-type': ['hover']} +moduleconfig = [] + +url = 'bl.btcblack.it' + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + btc = request['btc'] + query = f"{btc}.{url}" + try: + result = ' - '.join([str(r) for r in resolver.query(query, 'TXT')])[1:-1] + except NXDOMAIN: + result = f"{btc} is not known as a scam address." + except LabelTooLong: + result = f"{btc} is probably not a valid BTC address." + return {'results': [{'types': mispattributes['output'], 'values': result}]} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 08fe0cbe09e73e866f75fb348ef20f6c5e363d7f Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Tue, 5 Feb 2019 14:54:22 +0100 Subject: [PATCH 05/46] fix: Description fixed --- misp_modules/modules/expansion/btc_scam_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/btc_scam_check.py b/misp_modules/modules/expansion/btc_scam_check.py index b49414d..9f9a7d6 100644 --- a/misp_modules/modules/expansion/btc_scam_check.py +++ b/misp_modules/modules/expansion/btc_scam_check.py @@ -13,7 +13,7 @@ except ImportError: misperrors = {'error': 'Error'} mispattributes = {'input': ['btc'], 'output': ['text']} moduleinfo = {'version': '0.1', 'author': 'Christian Studer', - 'description': 'Checks if a BTC address is referenced as a scam.', + 'description': 'Checks if a BTC address has been abused.', 'module-type': ['hover']} moduleconfig = [] From e4c14689683c5b069e7707f578e80499de8b0997 Mon Sep 17 00:00:00 2001 From: 9b Date: Fri, 8 Feb 2019 12:27:20 -0500 Subject: [PATCH 06/46] Stubbed module --- README.md | 1 + REQUIREMENTS | 1 + misp_modules/modules/expansion/__init__.py | 2 +- .../modules/expansion/backscatter_io.py | 77 +++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/backscatter_io.py diff --git a/README.md b/README.md index 368ef6f..59f2346 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Expansion modules +* [Backscatter.io](misp_modules/modules/expansion/backscatter_io) - a hover and expansion module to expand an IP address with mass-scanning observations. * [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. * [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. diff --git a/REQUIREMENTS b/REQUIREMENTS index c3c16e6..0720e90 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -11,6 +11,7 @@ aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 attrs==18.2.0 +backscatter==0.2.3 beautifulsoup4==4.7.1 blockchain==1.4.4 certifi==2018.11.29 diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 559e5aa..b6bc74d 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,4 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471'] + 'intel471', 'backscatter_io'] diff --git a/misp_modules/modules/expansion/backscatter_io.py b/misp_modules/modules/expansion/backscatter_io.py new file mode 100644 index 0000000..2af073e --- /dev/null +++ b/misp_modules/modules/expansion/backscatter_io.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +"""Backscatter.io Module.""" +import json +try: + from backscatter import Backscatter +except ImportError: + print("Backscatter.io library not installed.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} +moduleinfo = {'version': '1', 'author': 'brandon@backscatter.io', + 'description': 'Backscatter.io module to bring mass-scanning observations into MISP.', + 'module-type': ['expansion', 'hover']} +moduleconfig = ['api_key'] +query_playbook = [ + {'inputs': ['ip-src', 'ip-dst'], + 'services': ['observations', 'enrichment'], + 'name': 'generic'} +] + + +def check_query(request): + """Check the incoming request for a valid configuration.""" + output = {'success': False} + config = request.get('config', None) + if not config: + misperrors['error'] = "Configuration is missing from the request." + return output + for item in moduleconfig: + if config.get(item, None): + continue + misperrors['error'] = "Backscatter.io authentication is missing." + return output + if not request.get('ip-src') and request.get('ip-dst'): + misperrors['error'] = "Unsupported attributes type." + return output + profile = {'success': True, 'config': config, 'playbook': 'generic'} + if 'ip-src' in request: + profile.update({'value': request.get('ip-src')}) + else: + profile.update({'value': request.get('ip-dst')}) + return profile + + +def handler(q=False): + """Handle gathering data.""" + if not q: + return q + request = json.loads(q) + checks = check_query(request) + if not checks['success']: + return misperrors + + output = {'results': list()} + + try: + bs = Backscatter(checks['config']['api_key']) + response = bs.get_observations(query=output['value'], query_type='ip') + if not response['success']: + misperrors['error'] = '%s: %s' % (response['error'], response['message']) + return misperrors + r = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]} + except Exception, e: + misperrors['error'] = str(e) + return misperrors + + return output + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + From c8b410161a19f0edbaedefd80d860ddc0cab326d Mon Sep 17 00:00:00 2001 From: 9b Date: Fri, 8 Feb 2019 12:29:43 -0500 Subject: [PATCH 07/46] Use the write var on return --- misp_modules/modules/expansion/backscatter_io.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/backscatter_io.py b/misp_modules/modules/expansion/backscatter_io.py index 2af073e..dab07a7 100644 --- a/misp_modules/modules/expansion/backscatter_io.py +++ b/misp_modules/modules/expansion/backscatter_io.py @@ -51,15 +51,13 @@ def handler(q=False): if not checks['success']: return misperrors - output = {'results': list()} - try: bs = Backscatter(checks['config']['api_key']) response = bs.get_observations(query=output['value'], query_type='ip') if not response['success']: misperrors['error'] = '%s: %s' % (response['error'], response['message']) return misperrors - r = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]} + output = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]} except Exception, e: misperrors['error'] = str(e) return misperrors From acc35e3a02362b2cddfdec31d48a0c740e4ecbac Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 10 Feb 2019 16:33:09 +0100 Subject: [PATCH 08/46] chg: [backscatter.io] Exception handler fixed for recent version of Python --- misp_modules/modules/expansion/backscatter_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/backscatter_io.py b/misp_modules/modules/expansion/backscatter_io.py index dab07a7..5a8a9cd 100644 --- a/misp_modules/modules/expansion/backscatter_io.py +++ b/misp_modules/modules/expansion/backscatter_io.py @@ -58,7 +58,7 @@ def handler(q=False): misperrors['error'] = '%s: %s' % (response['error'], response['message']) return misperrors output = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]} - except Exception, e: + except Exception as e: misperrors['error'] = str(e) return misperrors From 7b1a837b109f0ff1a8c9240bb31a84753739f438 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 10 Feb 2019 16:40:06 +0100 Subject: [PATCH 09/46] chg: [backscatter.io] remove blank line at the end of the file --- misp_modules/modules/expansion/backscatter_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/backscatter_io.py b/misp_modules/modules/expansion/backscatter_io.py index 5a8a9cd..bfa04f6 100644 --- a/misp_modules/modules/expansion/backscatter_io.py +++ b/misp_modules/modules/expansion/backscatter_io.py @@ -72,4 +72,3 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo - From 30753d57aa3096ce85c3ed54eed6e0f2dc3d054c Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 10 Feb 2019 16:46:43 +0100 Subject: [PATCH 10/46] chg: [doc] backscatter.io documentation added --- doc/expansion/backscatter_io.json | 8 ++++++++ doc/logos/backscatter_io.png | Bin 0 -> 25590 bytes 2 files changed, 8 insertions(+) create mode 100644 doc/expansion/backscatter_io.json create mode 100644 doc/logos/backscatter_io.png diff --git a/doc/expansion/backscatter_io.json b/doc/expansion/backscatter_io.json new file mode 100644 index 0000000..22123a5 --- /dev/null +++ b/doc/expansion/backscatter_io.json @@ -0,0 +1,8 @@ +{ + "description": "Query backscatter.io (https://backscatter.io/).", + "requirements": ["backscatter python library"], + "features": "The module takes a source or destination IP address as input and displays the information known by backscatter.io.\n\n", + "references": ["https://pypi.org/project/backscatter/"], + "input": "IP addresses.", + "output": "Text containing a history of the IP addresses especially on scanning based on backscatter.io information ." +} diff --git a/doc/logos/backscatter_io.png b/doc/logos/backscatter_io.png new file mode 100644 index 0000000000000000000000000000000000000000..0973112835e7f4ce67375c3c2b51e2fc23628175 GIT binary patch literal 25590 zcmXt8by(Bi_aBVz?iNH!8b)^sf`W97l192?69h@=P5}|A(WNj%QW}NPlLqOA5x@EQ zKEFSB#`E0sex11Yp1LRYg{}rM0RsU503g=VRMiInFwy^F0`PFqKWBlzZqPsYo|>jU z001H7zdsB>Zay6Vzy{D#ReBYezgOVcW@$JHS;<=lLZeavC6~Nx-?Yigc-SnHpMHFA z0w7l-!%!oCiG}?K$n){cDt$QwmmLcepEDV-GxTlcL;FC!#hBdIUc>8|Wyj^M9PXFi zhq*b;7xRtAInKRa`682q+rYE^^xf`&yOl5q3HY&4)#-SS_}=$?6!hGiiJP4T07@bTLd&b{eY3hroAvq0 zSOIVRhO?hWL@=5@fiD)i8}4G*maqQx$OE-d3*9RlT&=jZ zTwh@DmAzlF^pik_Y}?B8H7a~GMMX$3pbCQ#nCE-no?fZ0D>o0Nh2lbSr5~8=DwEfK zB;W#)L_c7_Z$?7q-e_KvtTi*NWh<}+(qy}SNm)~Uiwn@@YXrc!U_2+^tr@L{1!amb zKCtz`!!*H2NwWORA3#7whLeibiHz?sj-sAHB9rSU%u7HkMk|LT8B+z<5dh93(+hD= zp}qHqbF}P&Qcig(@D_1yakFizlGdVn(cil@Vf^I~yxmWc3wsz2m=YqslB6J8jI)sQ z&7VSIKn!q#G0g!X7@e5C&zrr5hh5r{ae0_VmT|Ho zk{9V#qOTs+@oQpzKM}!gS8roUS_5I1DtJ<*QuPIk_<8X(EaL7ZyqT58 zI8xeJAHqW}z*FGLXi(jpKU({o56r_fV~)<5Rc z-}!MZqFMKRye2dCpw<9FB#7sj8o*7gjiI1advY#(v8bld$fU@uTGGR?+W>x_cj^twXH#+Bw{9^R4kN%@sjB{sg9zy@T-{b|$1)9~LqTUXGb*UOCL0Wt`s?-1(}*M-oNT8V>92(O zIJelQJ6`$Yf6zxN9s_=vl6I$(;KxIm2H8U2Jjt8~wH23lHtIqVU>H9P2gb|dwmD=D zAJRN7Ny2%UaTZ2zS4RWs5g$0cnFIT^#1X+sLtW-2e+Dm}`v@V<7YF*#uHo;c@BpYr zHMQ4j59^bP?;V;fCyCY6Jap60ffW1;({;56g08(>Ng(uh3K;yntmOxTqMmj+fjX`y z_K^xWzy4{`nkw8D&ZQ`G5s(+pMw$FkeE`E+b(ld$f(WMzD<5<5oke6L3#(f4c(Uu) zJIatnuYCOpS`bE&*k3If{V(UroLfEKm9qOzlawa7Qcb~J-ehCJM1A~r5V>j4yjT9} ziQzg~?l~4!X@Z84>c#iJ^y$#@l)qJyY}Q>fYkcbMr1<$v13bc2n3uK+e|~WKKyZ_> zH2xyDY=?DN!LCR*q}twqm5vB)8{W_c5~LfuO_)FYbzQLL=^>Ez(a$2n*GE+bZkd=Q zxV62l-+52iVQ*+Eq?0H~86Tk#GnDboU*B20xy8KCI1DY6hx==;=6`^(*3Ebm!t5V^ zkWRAe2O~+FE_Uwg;dRh+(VHn}fncBvW`qibnCS;QzH2HFju!PkNLLAgABc;>|DoJ3 zELa4M*pV^aA>4awKjr${{cTLjsS4VRBY5jKY3ETT0+cDglSYr~sJWT$QW(W%0irP0 z%Ulu4D|##JUtS;A6**_Wo~xE{?Zf5Z%Y(+bYfcShp8PcdU7K98=AYM8du`$F2t6V* z5f+|+U9F&e69nB5voJMkOGRZ9a|;P=UCVAD^mZQzR-VZA^uYEXFgTPx7NG(W1Tvr; z)l~|6$8kVO(m)6ytO%g|6J)>QRT;Xt33929mo0S|mj^LxecJ;iVOlGG88ub7V&5=D z0&ZPbc;pE=#3h#p?%N4?S^@Fv>!Jf*9TUJS@{O_009=^66^T#&`)weTPP+k_9-u2f ztCwkjT>a%T(|sdEJ7xYL$-sX;Icr@Gp%7BCE?aE2OZ2uZZVkArS>*Dcuqi z+&0YY#Jkh%*01kz&y&(}!!@nE|Kdu$$-%oNqf0HSVyc+nkL7I0jlDa~sTctyjZNw$ zO;aS`6#lr!7&DMpe|A7JHvT-xfHzb5*-56m8d8Ka{^J^m_=@5r!|iTjldA}zfJ-wt zt(9b8o2g=z>r)>QewykCaX9f!T}zLZSnqgY%5pOf5lv37Oj_e~q(-_78%&-)0>@a$ zt%JD0LPvUgtP`^K(npMCLMdyYIaA6u7L=n5X%1n2g7K~|6Ts*bQ*78K2P!Lm>o3cI zm$WHq8!5_BjUitlJz;NYbS7IIPi=jEmBuaruNM3!!QG)05~Z zxtT-v@eN1JkdX@J)bikHV!bAHzhg?BRz+YF1`4zLV_S^ZA<$o?mPlHc2SJnlHY}<@ zT^^XE_Jv%!&p{N)YmfS56%m1#Pg-qX`Y8>l0#$gq=ngeLDpS(c0!6KUy7zD0T)y;y#viNds=XFP8t!eSh@D_G)cgUxDF8E4)1Y~NE8h&(Zx%$M1;yN)l2mZ8US4OH4 z=ZNm&R9ioXkck$~nnW38LnUfHYXTks+1d|wWgsV@pi9acy-=S!kw*2Z`Pen{pl zm}n*bLo^V@qB-|F*>1DWWd_9Vtm1Jw0uN_-#v%F0l#H31!ALQ4qJ}kz)0_ET+*sm* z{kAP9o(kqY+`|ATM2u@_t+RyoO~pq2h8`mns78lhA&=M!J*$JL3WR+NGnJAE?jS36 z3X)tYF-Ih!S^oe%Zem6GP<7rB_M9cL%2bE3&CuuOpET6@K0fT;y9m1XMR{Hog^p; zhTAvv_dz?fs3l{fYxYi?xsl0^rWj<7dAR17pRc&5DtzJ`@iu+h*DPKPI|3c6a!ZIo z?`&VuGQ}8E_-w$H}gm_KMj6G(!(u1)#+cG1Z*|5r6or$1b1HcbkCT*8E;ite3c z4*0vBy`ee?Kzi?Ob*Z25>*tSpQ8W@9v0p^C6^6dIKK6;)m}Cr~JRWu^VAlrN!C0FT zahe3(r7Wc&M|O40CeMGeRghIJ5j@r12d=_kMli{=?U4 z`RR3{0jD}e7nPq;JIOZnFoteD{aYr6dX}>Db+L8b+BM$+^}(JAvb94Xm|V^v|E(}x z8#XGaq0oY1B)9^0raZN2Mmlz2FL5*qo{@1_GdW?6Y)lTNOEUOSI7me@9nKDlrr|A?N!$fMD$SM}gE4V-J4-L7x_awEw`KKO8x%HU1K= z(~#Ih3UzOy4tUlx`(85gOP!O{Ui_Wnp01UO2Pa6EW(L5*xkUGsLW=G$$a#+9iVAhz zl&OPbMC{l>DVmpd_xDM<><7KCx?OG`ei1r}ZB_uX0IqpXJLxIJv67{`bY_9I?!PHF z_%KK@2+jEPBe|jb5UrUwP`N!DOgMGN+pCyWh`N2?jz3k(y(PzA*VL3z%IO=dnL|8~ z;+ierrRF_lAvrt`p;6epjJG8ffFS z6w-ZfYr*UU1@V{ZZ^D?hRTy*`N(}T@81OQSn&hqu;i6L^P0S@sr@{3(U1l62CHe|Wijg!)kS1w)#6*yrvqM0ke6_C#Q0Ck z4N~?*|L;D+7m=T<7ULJyJ(&i)q8uPzi1BgD4S2G;%pVX1s%P4e1q0`u*t{20MR9dP zdVWh`MU{heA$6mtn_{SQcoF;9NF+Mvu7}Ya316K0HGjn>B6My_V~qzK+A27G*>KCv=eJfVmGEau0{#pQL`V1+ktH0nYsPA5L6wD=0)eQYE)KmvJWHe7 z*s=+CY@Y|$;E-Y(8T775@R1pNwTkxU{`DOghaKL=C1B7HyVRhS2f}!S&>waK+Zoyn zO?(P!-Be{SRTMCch+S*{*`}`xF@8rKgsVqg-A~5ZS?E_`Y=4QC94(VL2p4tT%`uDB z@_T2UV^5}&!x5`R9P8V$_m8ACsv?Ab_aWmivzQv#;dXtFB92cd|J z8hUsYd;^XF2R~v)8)b^Z;JB6am>=4+ttV@&xcvAa$wTGiq_NZTCv;&$*=mXAXE~P~ zINWW`f|$wN(Akq|1zvkwNy2JO>4+Egf>Dn#LY(>b(e*Yo+XZKj3{7Omyef76FbZ#Z zZDW9ZbW-2=ev6T5z}fO?&;8v5w3>{Rz}4mN1mRGd{Zb_}ya>3E`|f%2S$fbsD#VZw zZPTgg4}r_ng%tAK;qSv7{Q(~+-pmhhL1Lro7a{=zUUA=@RtKp-vm(^lt5bT$H4!yuM8Gh4Pt8ctbp2%>E{4|><(0r3 z%7;eZ<+s=HcSFQ@8ZK^Y16wG$(?E;pL1{Mm+$RL7Jt^jDDfhLWH+Q8O67+`-8rD;M z3`b5x_vN9Ui0sHPQCaD z>5CsnkJaOGF*ddsd{HB=K#+r4hyI8sZ;|ii2X?*$XH#n0Is`Pm*1dgLmhm4Pux4gMu=9MZqU!s-70uiK+2NNihe(QYl*K z$f~1U=se&o-W9pDcuAw*(B|nx%Uzin@dQE3_XyezXmnb(KEx2@yt<@8%O%nky*In{ z2SxhCD^7pFSM#1j9e%#=8zdQ`PjnAUaI#5X3qrad1P$lTd4b5hIR15ORVGzn|jb=~KzpPIR;wRdxYK zbsUGu{D>1axL3Z>nv&aQW2sm|RUg|r+uN3&eNF1GM##r|*4ZmsC9VI7OR)lf{S@`# zt-JVKrxx~Uri@rZJY*CW@v`gvqM~Nxe>4O@$k6t4oTl-AjD|;q+?GFlaHhlcgrzE3ZUhpq zC0w(&2*S0Z^!We$v!25yjoW6egIGeN^j&v)v{y7gUTDJp&qKzN#JwN?K@39?Bj!DC zfM@-WFCQM25XVD`Y*AC4qF-ZA|A%1zM2PZE>Tk`c@{2oxGm7 zMX<>d?WXAdg8%}dmD}SlHMZO}-+<$w>;voWn)bJZowTMSj(q=_p2)-vxR~Ecp2tWx ziNT-MiiY46!tKmLJpS{1BKRHBbX1B(Nw6EX6av+h9#e`l*>{*%6 z@*%P$|FhOBeu{0mysQ&z;TQ6O&s6MK;G1gVF{?C&|DAUZ+Z5%9r}}AZqTM1d?q_7F zKBRBJLbNQ)!Pc7p>ELvRBd-)8KQB+ZA<$*?GxY9|7V)iE;ss1lj0LOAkxu{Q+7{`B zlqBtN+AT|<(AGJh>_In8h3O%H&87alL67`%bY7q%h;%HgdwwtMncOjx zWFU{G)K%Mr{JMoJ?ub}FbTZ{-P13CnUl-A`VEK+?!oMwTXS{mKv8IYHy@veY=M7In zBcb0r9bSTLt51{z00TS-$9K3GMUGF7?4N~ord6c~@S{_BuctSs3z|@pd*q8|umpTL31n4E6xw z6*d-Ffn;oGr{ zU7`8~Uyhw}CYAQkooy93bQ*kGE-X}QZo?&>|Apj3EY@_<2Tm7T@;${O(ZC5yV0w;V zLw3TEkZx9cha*?$0$Lv1<0S3#rxEmbnhqOcaWsHS)n6;?>RZHX3f@MnpELc&^MVK&9x&1k@5pJfC)8(^ug5pO_%48Fgq-qos8trle zJcN_DfO$TEAcrCD&uGDi0hPH($XpQ56ekNzyq9|-z`kh7twv>=$sWNflh-P&_V@}_9||!t4{i$_ z2rzLz8o&8iZSJ2`+Gn*r&W$Tkif9~umiT7MMs1D#Oy7rJboyYI770XX- zNPVt?;WmdYP_D_V1-Z@?{cn(*;b%*eU&jf072Pf#&=0n{ITA?elgr;A@bQ?$Fa=h1 zQvMie3sfuR*16UI4+m|jOvH$~5lhDgnZrA2rxSbIo;Kz%g?rUL5z7aoA!B73yJ~;rgWD{nr2#2iMTiUQmEYh!sydZXDs%k!=YCC>yL2`!Tb>W zm|sUr%a-#`RmflxN8!5razOrh)x4xNv2ls*ac1*h2HOwm89@R$-2^?}+X1@+VkYxE zY%wKU-G5!1tfSe$R7Zyi`xein#xp@l;*TbaBuTzhOhgV3ryt|M<TFujwrMZQzvOAVSOOYeIe{a3lEk(JTIGJFy$V zWrxnmav~+bAi<Bg%^)rf&?m z-HnxLP_OeLZQ}9|jo%rMHrb0&Fc`Z`bFe|No2nZ<7~!@EJ+SbK2#BjLuGYjoI;SFu zFVLGf|6|YG_$r9{uD;CM9hrup=!94$q>-cXB_*BtDYfsS6;?d;l*Atu5 zIPTH*5vBeE;>U0PMHX(orWL5%=V5GcJEjnGcJbnC^xOZS`Wv^Fcb^{3o#yuZ3&JP1 z2}*gTd=Zr-fXr7$wh7)L3?^6rhj~l{g$bf~2?dqHzyJ0xkgh}fc(1mHt-k;vYUvqt zrHh3)ReY_Ilv~aTvjh%H#vE^0w0EIuOnC8vtP6UUp_}9HFaUw6+4284IXI0=VK?w-q-}D=kAj zpjZbN&?@BQBkSp?+x_z&%cY7%ahG~==?7s*E*txP!tzv9Y5c=P;RJK%B!O|AVSAG z^r?=&==6LtLAZwr@kiQiNT`TDYyT#K4Pazw%9&k`2>34n6LqQsG-33@p8~RVxbui%4IPS|;6Vd;{e76-3wSf@=hFcyKsP4?E9t=GMsodeY!|wio2P2pNG}*N| zmbbt|b86Us5OkT?lu3NhVuw8`x#d4fu?_jJu3+Be0}%2`TD4FE;sM~VN_p%ieZ|ic zHy&#L6GY%Ora{Jn1(~!SCgxo_<0Um@FP#ej(|m9x2SBhe0JLo8to2^LTSY4pzdxEJ zU>YeCRG$IlmBzNdK5O6gT^|L24cBkpM1UoZSdKV0WZ0I8C{vnd2sm^m`pD9(4{`Yxj^BsPR0IX==KX=Gn*277$;9BkY%ZhcJeNt^1r8vLC#*oxqIX;ahm zq%Vcm6_+CYE4D2DBF19WNc=nz<^}GBdSR*6(rxD!#1+yFION+Fr#N1s(NvpF(S%Gv zf8o;uW#26_b)>j8t8}QyRA{g4K6ugER%hNyb3e+&daC^R?dyd?d<$j6=j%)$9wk(e z=KLyoB-9|8)+qJjfOFCEZh>Mq)i4!HR}LM}W+_^ZF=3?V)<-cMf)dXc(%x_S!=ne| zd1AvUp+zMj;r3GNK#od5YqhuU$f|*hiVgK*^jFlu=H1_*Ysob$oZ)#G`H`XdQDAYT z87Iiqs6tQ1kHKo~(&0}fmh?Ki!WWNQ`HgMP z&qjza-2MC>xo?JP6&AHF$T4Gp5+$G_`vs+>o0>OX@hmm6&g_ zbX7W>Sd`lHG7MVf^J_!8!TL`pn?_h}iVVeW0Os{KCo9!V#?#ft{4*D$r|!xXf{#+d zS@y%C_<#CqRN-J~1ID+?>|{J`+2oY0Xo9X!y&5X4Sd}iaT$=NhviCnXC#P=&ZG+N6 z-)7iof(y4!CnNW6xit!MQ>MQ#Mwqo9vsU=(zd>dKZWgs#e8`M(CMpXp(iB^Zu$>}^ zuI=TSm=s&pz4d_w{#dU!7B|Iqjb+@L*}$9)+y$+L6`R%EgIW5^!SYFa!WA|)->Y*A z^7M-iOp?~k6BF$vVD2~t(G;7Om$O9DhMFDNnTBJv9=B>6{U5Y<$`r7E?R-p^R?03e zE4(e&`gTG5xp&&WTdu)E@vb9HZg;zp_m2^${KxQ~&xlCK%FLu3eppJcQ_0zgkEy@L z_wykxp0gb-gkWFkP)zuGYsa*qy!u!;S2?j1h+#OuS;2*Gk?KXOnTR#~GTM918D~C= z`fOg{ofUxe$+nTemRQL&#o*3La(O)amE@Ces>w z=9qHo8(W76U3(^t{wy=a?%5i8yiIYe%YLH?9=x2ZT=52X##v%cKi8Vkq`THz$#2ar zgHeIM#3#6vvMhmp$p~_RKUO4OGEGeoJo0!!EvrC6YF}s(f20IB~m-(rjQ=c2oCoo>yr$&9`rDm>QnJ_cg<7S7$J*z@^;& zmC?<&Z7h)WO0sZPDe?6NJZfyt`emyM#V*dD%+8LqHOB2m*Y&)T*TsF^{i%_&&u)AH)=v5rW>NPy zb(*P)8&q5c(w`${Dp<0f>>{98Ob;^OJ{RCQ8GVl^+APn{6mvfxvy8tP@i=g+1j><9ox zFQ#o-oG;CrjdfjW0B4HbEJ~^Jal&}p8Ye@`AGZzW9oavaV`ZfZxy*!#AiW=shCwnAvHW{VD;9U068y<-6f zXUXnQldEVF>8H?8eE}uj7mm(#PfklNT()(xX#v@L3SwHhcj?ufOWt^vJ5_w=q54=6 zrZ4}hz4;@kdC-V>3n|;O=kM3d?b?Ti2c4-yJzo|&?>E|yj&!Qg2YkYS4Be}w>EM4- zy9`Y|S{YdH9&#}oc~sfW+qCsX&CX>>`ZK5Yl)d!OSkJ5XJb0QNF1E)^6+tvjZ$2rm z(dP7t7JTZql_4VD+4{s^c3d#@{40kW#;s&F&2=;VG{G>P0N0m5(T#CuRL#t~JY%4- z>4#R4TQ`0s#i^SO=K)QcsjE2-fRot6YkEOvt!ZoFf_GnDL|x)GA!rm+owkO+X1?Q; z=-{by&qx?rjNVeMJ3|}wS~IavokMqCh5}_Ho;0?2nEjl$m-FMtF=d3&0+^5;;``2v ztNdsxrf1I*Fd!Fgc|h5`)h(UcRaysZ=#Dy%210 zmjEQY3+#Fvq0}^3SCicW0Nj82_|B7+Lae-`Gvu1RQoP4nWlakt_z57ndDdC|77~z> z-mgW6yGW^+bZe)M~_N%ODFejVa9S_Qa*GM~QM%TVKNY5+vx<7qRf zz_)qajQSLt` zQz&JboaZv5*Z@*?KEo2J!n4`jW+3FX0znb(J%fIVZn)~r2c>Ch2qF{a>LoEps`q=% zc?y?)K4vagOYHa!k`%))Te;;UI~%k>V#xu7BIChliX$nTc=g#;%S;J=UAA(9!Gp~r zGI8^0u7#mVlt%aCHSNKq=Elqm3k-{_KU>X|7+j`GxAI5+Pt+icn%%dg>W%}{ z?@d2zN|WyfzMcma`L@^bSuiM81W)Z52e5LI4R+p{YRX?#pQ-AOy4=)G>hIQJHizhY zUzTy$^Zb#7C`oN!aoH1X)T~4d-gEf{0KV-Go#LM_;LRxyn}6UQw5rysD)U$CN_)}W zoRf9=NG}MwH}`0TsMuKU+Q@Ism{&493`axYgT>*Hc2>ZUmF!kN)48*HYIPJ5#|j%5 z2WI~Eg7MfYsYCn``qj2MB_iTpHqlOmi~}zlv%0V}N#>EG$H~;;%DN9k@CUBCT)X(Y zB=bkOPN!3Ugik6Z=vFt(^tw`%2NEG4ynp?XN_Fh3e%Eri$rtj#z0uD)xLr*?XqZtQ zycqwgYj{{oYCBam_3_R&Fkky3>wBJN`?_q9SgEeoM)TuyX`=SAwDKA~r`qKZJQAhM z%+sSe`|q#H%xO&x2fwTYtvALlyXJUrsrPvP$-A6P}f*7|lsXcKypQBL}q>!tfE| zDhrmT{Aj)oU0a)-!--o#S5=ZQ+9SPP1w4b~Mv{=UhuEKTyG@MG8W1pSPxA9D8Apzg zFDpu4X5XzgNlQnimI*dxBsXFR&RiGl50(YEu%`AU_iAYw13$5cO^+m|$(oRX zaOizR|8Ac@74dH+D(;o{nM?`#+&~wUU&X~6%TsB%i23a=N7iT^ET5FR zz1fLzx9lit^3H&Y@NcNC6vG_CNj)CR63GUg&L<*zzHEG0Z&D9(JsNNE+qFuD=Ha4J z_gz+SZ}{rS>5}8%ZPq~OB(qG~zMC;aCdWczdhMC&+sl-}-DkKIR0-^E-;xEA_nn^8 zl)&5ENg5{zw#%B4l%`92*}_`a0`-xz6XpMZ1*G6(G9~Fmf5e`|=Yt;0VA*NXpREj%{1U!g4iBgqjEL62)7KgpB(a}Z}N2D?j`=u_~GJ>@^-&Ia`xZ(2K zPI%;CLYj%^Lwk!y$@q2TsU^~0hyn&hOZ7z)jL^*<8f|?C0zi`}&)bBEDqA^SCX;GR zKfonj6JwlT`a+%(Vk9&-LBi>D|Gy55@bELDj~2KX1E~=%YwGDdfGf!puxn$tQl2go zk(xfj<&=Lzy04ZQ@Kr4+HPEy=>fzAtw+H~@U4uEI>_XzC{|Nw=roeJ)BuGs{ILlpN z+|W0LR4L_^&_&;ya;`j#_dgf`f(`GAXtWX92DP0q!D5sUe~b>8#(Ipsjk;31Wc0&e z>sQB-Gv(TEHPIGJyNOuVhitLVRxJk4GYa9CiSthl0L6Fbr235|{OE>9f=1!x_$2Gy zTdaYtH$^-ZrH8$OpG`jXHf*EnfO8$DmOb*?UGLN1y1$YMXF6g$qwI8QRFJUSf3*2< z#4D|!xDta~g(er+zNz&%h{Ycs&M*39p3*3JL)t58$UnVCeinwnxV7vB0W&c?INB9n z10%3#CXA&(?r?2HLZbo?qGBjx8wJy0l@l!e+sU`fQ!018oca%g0gYk(597%{j1e`L ziLBVLWLs_1+Y^$8;1`eihwF^&A7P`!f`~02Mte3F)t`JS`P)5>hF~7eaPO|C)i+R@ zj;Z@%@@YWmEX*WD1lwRBt;qV5osYyLDm@#~uHmKtVj;RY`RTJi+UR~I?8%MYQL+nm z$EP<;X;ndqrqSN}F{2n(FFky)!EN4o?ALg0bD#I`ChA;h-l{-EeQv|UFZVCNJ&O zUb8#DjsNa+7AneecXMcp0Csx)Qi|-H#opjYH^*b6ZJzOe=z5T+ZJP~#-%MJch(+_t zLlk{6PaKE<{H-y;K-z$NQ>>PkZm#5GQc|KLGj6jWD8G&#MKqNsr3hvRqr46zRR*)l4n@{%OzXPd}A0=Rd9{^L#C2TXaIywrLvQ~maDFNgQb|8Rm`?U=)D7Q^vc#GOt> z%MfO!ur`b%sfQo{S@dOn`-wrnA*9S3gkf6pJ?W~VZzHgZntwxkdm`oek8kosXg8=&pZo-Xt z-bZ6M#kTa__aR`PhX;>H2^C?oJC{6hg}u%Y#iq2mYTd;&SB3*oCm+JIZY=ROvMabr z3^8n!WA=Tnco*=by)IUBJMxu()8Zv=8|&4FdJ=XM+<)!m&i{-?u#tOsc7O@kcZb{$ z!g(nJ=$htO5ktjG6FM3$zsyesIP)wLoZh4*7B9?;TX;mB=p^$2p=^m zb(SE06pyMNp7l<2mtK7%>hBwl-$osqp`T^w6Q~HLE*u4N>OLBAGR4-C+Ex#FldX0KcGF*EW|U5SY$dH)7n44#vwN4lNeffBDsoys@p&hAXLHvyeEE#f@6&R-Qc zx_Ao}dcrMU**C&l%<9=C?q&6RL0t36(%iZCsD!938=Q~UTS1`|*o1G@Iz~vqti)?X z+p+xEW>-yBI^12cRzD>pW<)+~kqm^(zWVVcwHzk`-d{oE;Gy669wzNCTmSyqtM2NbRZmN=(eBmuF9aYIf#kBS1n0e>Uq#2M4+%c8TyB zu14saC@~O59{E9hAP20ikY{Nsdh|Ls_ylwjgsXf)>UP^v8SNEkQe9oa>(UYp{V4e9 zrI>eS|7H)5!SbG2C?#X}Od2arb6Zfj>V3;-|6`zqaWFe-ZzjxAzz6E5!r(rFIr!yc zi?xF4*j@TN@@U18-;zDA0UFRd{&X-P~-Z;g<27)cX-n)A6AL3C3J{U<8P@E zsd-jtU+g(Y5(Zh4fNLl!V59h@$Qf58Qt=V%I)}p@aLTgAKhmq8ZvwEq_oq7aI-kjG z+=44#i%H-LRB+5r{#2dc5{kDUmToJDes|!#HW{@;=;#acFGZITw=w?%YKK=M4((dl zW8)8skK=~K*2^An9_NmYgE{NE{S^GU9iv6%7^+Fk@BfKJqok0W3$;+w8wE57vVTh9 zYIwrTGhlPdmQl~9WgPveeMjJSA>o$ML?rzf-<0f%ph3XzsrTuoB)lGO^rO+XiNzat zvE#5KE#|!-qwO*g#wYOFO5GE5)Wq0E31!7K(OB2UY?K7r^Fx;kHnJ&NqijW2Wl3M8 zR1Rvlx)}&iodOV66SU*lX4=*O~Yr}Uj3rLuK23^(J&e2OhyPYQ z)_Y=&h3DVr_L_Xargq|tZTS>nxk0e!Iy3N{0e*+L)DopoF?Aun=p7rV|BV=oQzI|; z2mg^xxBjJgO)`9)A%+=5Z+>;kj~+K;yEp~>KEe0I*%)PAc~1@Pw(%Kzk|?VzrJ&!aZ~Ym>KqlQCD8Ta3{{D2{BtV3CXii8ZN|eL? zInrfydFzn(5UBmmQ$5Npv7jj86Szp?L+RmwMVf(WJ}^*T56EF_zQy}Sj=!DB?H$GhH8wqEG$^nx*c#wZJZ~J65c<*t` zArE>YPYzNAI}N~d*3fl%z(L^O0Sn^+AwXKpH=Mt|;d$GJ*YXk>;ik(#rm6R)Upu;m zw!7fjWr~YzCX@B6!gUdHZmchEzipJAVH|uP+bSt%ri)xx0?~p9<|GEa8;iM577yQJ zCDO80mJeuv-)KrnMvlh~;~=g>NYL}DZWc*hf&-6V3hjjboP}&q-s>10a3pAJB2~y9Ui~C9+gc8J_wS8%s41hZ zC?R7_#8co%9Gw_VvR+RNp9@)0-qO=|D_t>5O#V4~fkS^Pu=nvm3qFA!hSjh}vLddR zxU8fyX6%jK?z&zEQvR+VEM9<~IS92#itN5mR;L#MD*R3$QRcXqjMTLzjSomVuO|n@{30 zQfIgnuG75>1zj?G;8=UN3OG_uW)c_Q`->DNgbtoOTI;H@$tS0^ks7Q zSU#Dh@48_bg$-XLNaE)&>KsTul$SBGe(texO@U7U+oN4AnT^_h2TRV4LA3S8pTrqt zEpruEB;Eh6rmNSvnTAezH)G&}*T&P(^MznT`|^Zx4!-L6ec&~DL&x;)nexdaliIuf!F`l@V0efid2KhSo9rkwJcel!Uz(SDA{Y>keQ) zrsQ9{hjt6BU#zR*s-qGWiqJ!Ye68iLB&zm4qm{3+d)R2#_T-E2)!gF<&I(Ol0(4s zs#L)|hZ%O5WuVOHtjfA)M3iL&JM`gy|9od~E?trvFes90Sc#0YAX^+>t>^KcnhD1A zbLI~ywr=F~$9eS5mLIhN#B}^3YOm)OdXpXPwd7?W`RrXLrC5UKkKmkIW`ykj1)3pc z-nf;;qZe=-l0e3?e_U$IILet>E)UMfzJJP`%f+hF=y6$;PFfAHEBgbNX8av#coRus z+hNeQ2FoD%xh*TPy#=g=RFHG#XC!bAl0e9@XAbat;HOA%R&;tGbHMV*^6sL@itE76 z|JIIdaeRyfJ#Qli9V=h>UD+H-6PtCu&%%F^d?&LFbqK{Wz#HiMw*$Ev|BiCI|6>St zX>7Mqu+Nh7u#ea+g1aKIEmG3ix}REE9-#Oq{Cgvbs7?d^1AIn0Ura{6M1KNohy)A$ zf!9<1zC5MO^%hRY#-|Tz7}p4^t7}(j7-z^r$lZK@aO0*ga;<3^_I=e#@*#4_&+@A_ z3vEapg1;dz?hrou0C*(O?@l6*Q`~-PL@(^FRendozJ^%UJ)Q{+SFq2L?_mG%ehe%b z+RzfPC~^%b?bn-2b21*uB3IPIWP^e4A={YNzSNSfa`9Z8?1qg`XAw!_w%*uPlsYB6 zd8YJ2_V%nF;Qt`YiB4k{0)CDJPS*WaCK(GHjVx&wk%7R8#IdTUDc9!Vyoj;N6B3nQ ze}#<`KA^Z0LbJqhcejJ`-@M41A;6E3F>2j+MV8lrZ(@5OHY@u9?_uvtZJA2BR}a_6 z&bQku*w-}6W9Nt?6zsF+7VID9Ih1=TFBAiit(B!AZ1PY zTj6SK{5pmRS7h(S#u={?p)+!3C2VRhbd?W)B?8-8R(d08mj8*IU$O4X7-b6Z7v$Pn z(GQL7jo4%DQ!9BLIUrkBmc}k&+sO*{C1xM&2HUO*_F3~A>>uh5B=c06A8ZYV0Y?$X z32kecgRF4vO*yvndLcOpto>>&U(V{t$T=6g=Vg?FeTnG{Ov1(~4=C7Y%^)PdK#;1|fUo^`)!$}`Am_(d~&Fdl}`+DrZwxgo0~BWFSEZu{J%U|(u}i`}5x zK*2t1zKs3DT@kOXo-b{{X24m<{7JPgAfjGnNiIPr<$z*dP0cy&|t8 zL5ZVFUtmpOUu0sq8VL+SCm@=D9A5lCWI6w)0A#6RXn^0{h?z*0z;_MW*5)=~J8LC6c?j4VnFJhr1_O(C$_OMtTcGp* zAB>!~?}IESwjq zwS6a2e5??hgh*~ zyFJQTNCvJ};9%?@{%ScpthtU81&a{RIUH-52CS0JorPdu?0ozJ1^l?W3_FMYRl&Yk zxCZ-4pjv(s)?CLi=O5TVXR*l98GTs__Xa5#28{tT%bQfMqQWVK3mK#*mkC$XOxs$~vxPuz)%9>{gs&5)}_8&UrBty!`D z5{yaSrTi6GL2_+aZm<46youpC5IYC918avjF&b9Hu1H09G8?HB>DV#|`GUP2$=X+| zuO=baId`Z0*SeBoS?x4N**%*(3&FRs^Y0u=lG(O55;0mivQ-YMRM->y$)T&v#ft}7 z(HA&@B^(OE53wDrv1H8C;%w|Eh_2EOd?%#Q z&gWQ_?3D#?L5kh9GOJ+!{KP2BWpQfcoQAD$zZl*`J%%D>BCVYH60NnQ*%P=2`w60! z6#7(af!Lb-p1C)HWp&k$ieF&=+{7rSWbkRsj0UD-=hp{#A>!<%T&kFj;~9pO!kj-J3x*!sPmOrT_^UXA!CI{o!yErTK; z>n)13G9FouZiY>;yR!B|lRC!EP$a#vh%PaT|v zVd$0MenrV9YkDJzthAqniAYkfBrK23ygHk5oaOlqw)0<0<*PXkTW7yUIR-$D>E;8{t$V8!o%ldP3!f*p{w zqdQqE5;3!s+spc*Dgr%~+nb)#@0pgtM-gz%Q+pXoBHL2DI}j?iWW$CX~EA3$%l(yb>%el+*8Nmf6bfWo0Om9L|CltsjhLWe#u_z8X+r1N3}90~nxVmMOcVKdukU zsPiEPI)6Q*&YEnF{o&_V@LVUK<7{j@rGs+iwN671;0o;fA&I;JY#qWG1)~S>E;>HH zPRS-~G=r*&yh{3ti@lK5F>ReUPpPjq{g5PZ)^pa5p$#n}Ly@m=?;KltB8M}r`yr|P z6|XFKGCICao2jm7498|HpOR8vt^V&SOQqCTn_pu;Pu&{a$^sKZ>CFlFm9sYDt{~W* zjHk8h2`oVPYMYO!SrZ2n(I3a{Bu}ohA2wA1r&5euDqq5W>3=q`a%e+~Ob_5`Y&)p~ zxp!WKK^+?H^s;%$L%@0g{80oBCY)bq0qf|Q5(Rf*Gltud=daNjmww1C9;=tzqfL3y zmW6Hv-jZD>LGDfWGlM%s}x9gF4ek@&faFh2dhrV)|wD{RN{ zJ4mjMHJEocpcj%p_5|#+ss28tUp4s`_Vdw1e2cPawDX$RlIgT)j^-MCAN&4DE1x3u z>$9d<)b>RCp)$B+d4^$R;136&TDi7qhfKN#0tG9ve^;I@u5R#c~GWKJ% zC~{vaPwUVdcpQ5l%>}*~*w#|g6L=E)4(y`;P2D>aDlRvT-vE0dB{>U-FW7kbL?>@^q3D9m^nJWuIa{=87y=Cw06M zBZw!cYnAP@pv02Cv4dh%&C{e8aTM0bQ>Gwksav%~bXnOH_$P6E8J0mdPUe0UdIA5& z-Zve<|AjTRwb%mtEKU{KEv%`9`o zbb=x=r26duJ_HsHY-`IA%a`USz>0xwEh3u$?_=LTA><9c}e2d)SYi zGRjuXcZC{zVeglUv~oJ~U<@Um)8{WC*M^+=Sqyjp`~C?fvyo-w9q{uYu`vSpE^(w1 zFSFdzY=5XKKv+keJG66UrNnPC|7tc9K>5F8d zyNNh%3zc1F65s0QP~tfw_9?JrSaY2i(gauJ2Z3!ZGW*gx+0@=cf}V}>^8%q6gq*|q zE6wLMaLGr0+C=|o(gs|Qy0RzK>f`o*=$l87E=ymx|nt2mfkpPdv8*K13!cr#Ag)jV*y%G835? zjz@war#X5c35kA5>jjW8$|u0`b$d*o9f;>j=OkccSaY4O#Pm{m0ojXkx@QOCS-G*2 zcYw=*qkuK>vjY_d0owp4Bgv&aAAda34(#52*O{;;>FZmBkl6gw67(akuYZ|NgO&UIUQa4~usGIf{8N`IU*gR=iW6i-ALsEr1y4g{&UNl3#x|a_i@DYelk9 zh5(=qxE0vWT9GRDL-u-|`m_Ou1HZzH@MPgLU}C3C>XhogPep==S~3S&LFYbB~U5h*$BM8KlRX?*Mbpo+{yN>Vz_F%q~A z`+jqAX=P~#en@=$5PRQL%YX0#ywR~U_5-6@J|e!XS+TPRepA53#X>#?b|OCZB%UkZ z6+bNz)M6(!f5d)(RLg&n9OX{)EQp*9i0PqsaVfLhkKD|Njg5in*!!ni_6lmTlZk%7 zQ`irZYIz!1Af(YHWnJJw?EBBfC9A}AXT4fJ1CAxmOfZ4{!YA0*{cxOnXV3G4(}7vXeU3m zBc3#o6}cC`_nwvkz$w5q?EBHhrIEah)Pm^B2;g_fAz=%>+L7Wp>(tOK7M3NRvhNic zkEi-A94BHwaJtGZcodn1C6O-|>wa``sU`0t*IIj0pHaXGNT8sh*JLE?;>r#5vStY+ zzlC)_R?BRpqmaD=rIC1Eop z{jw)tax|C&T#1ai*8;ZF16Ul%ytp7ThxP1ie;RU1emw9Va^|Us3_(g}uWPMH7hVFs zh}@KL3eH~GN0YVWHzAE~LAJrK4!VSsPm!6kvsbb~^_A2K#|k zOU|P;R$?^S82A_Q-2`zdrc6M};f#ivllT-^aD&DttAz^o%Cnf-^TDrCFUY^aBn?vU*wfwM$$w z{r}s$_b9EZGLGY42ADzRrlP1QFrlI(7;1{k8=BY5YbK?ptx}h0dCBZcdqb@ zS-`odUPs(^1m@xYeRmYE16#&->Jp;k2zQa#3~YOh#IF}{8dl@m`+HX;6N|CI-|k2f zMqpbNy}$QjePO5gPF+HF0sbAZSUihHrAZqOT!oc!_Wu4A(MA(670ZmLX}08yCcMHv zg;j0soFzEM@~k;yZQe#9qL)g!|f3(RDNfPXj+7+{$-Zb}-?U^ljTW#>IE) zk`^`Co^kIESuEZI_KfZ9r7PXA)T-ALUX?P%#<2$br9U3aq9;C1$5Iq~_lIKf{rFB@ z(q>aEU9fkjEEb;t$HsQ{QmPv4mEo1x;<0A@`-Z{_-%P@O(P{0BzsvFO6P?8^v7Nc3 z*4DsV_;=1?(FB|y+u2Jc1_EbbC5o5e-$(KkZ)1gc4nz4fLQxCcj(^|iET)jILP?UK zVTAvh_Xd=pBt$h>b;8SmM+kSDX-XOMq_g-p_O@4~s9gzn z)oLN`M;$qIJ%GJ|i-3CwH~CbE(TMGWxehpy=*sO%JnDg0@b5RB#WQGnNJ=!Ca96Lk z?a%|`Y5$^bq@#nRNBCjxV^DrN#6aavuFUeh^qn8 zOnHxs+ZULIm8=ZaJm3W2Iqw6JiVXoqwTqp|W^t(bSRVM4nv-sBigf`*%H7+$cmJW7}32w8XpE z4Y=^2|Jd))BtkuaYq069kj(`S0N!yvj$mm3(6ytnJOQDvG`<+5wK4@%^+gVB-}E>?PC(k7a$^)8H%O6_P&pm5=*cGULvtO1^bol-RVn-+31y+ zbgTz9KvZT)Y40S`liqqEFfyKI5b4V?MBREG;3@g(HBK_+JF2#&onxAde0L9;Nht(ui&K(v3A= z_6AuP`>lb&hK^;lvl z{g{y;?2T?*!qts`pSc7F=>3q94bl%^Idr5i_I znV8^tNPM;g{)qp*r`&iHRf?3r4I#Q$uSrG&HkBdib`|mVlM|9V6^X&ASgKs_`%SrV zFJV_56#2jY*j6j=i>BOY0=LgypZtHE+REXV)8Qf@Q?-;BOVM3QqE{uffEaR)YEDB-Ka za-k>DFJ>fkPEwr_z!Ui2M=FU=fMcR>5|QMcPI|HiwvRSGDp;Uw25n$f^8WVOH* zr1zfl9&+VBkaiFRn^s3bxBZ!gLwQ_J_|Wb6QfS!>}Yd-uH`2<0Wjpf=K4< zO7#1)lZ>_4f}H+QH;hzm63O-jm75OBj=%%>-y^Dthk$-jH;hO~1_Dpxf3Z~)9{}f| zmAN7L2g&wuTJK) zq=*{e8(1k^@B2VC(Fj};aodQb!ui;1b?=L?+PDX(kGO55JbD9=ANvEd@W1Y>jYYsA z5x0#Q?Du?hB4;H4gMgb!|2%&oZbgNqL^jY9xDo%0GpcwWxCrPKWg|&ZN0Quk z+YAgy=CpEcAaFhQcFp^Gjv`iL>lQ?$h*NxWs z0l+n6>pn0Kl^z!<$td7W{4d(5ZvbZjJtAx;+0gGv=DXCdxOnu(7FQ-UkV`72qc=t( zW$6!0#{c4tI^G1%Km&m|e2VyYmAsASZ===^xRi8f3hgMGvF)PWqiZUWeEAmHnFDP9 zAh{Ur8i?98#QVLSl-#Lg-e4@VX4Zc3ug9kT4vnswH8cCP$Ku#6g|Kbl`}n8Z4*kf!B!U`eO*0Gx{|gjK2^)ny1=2mBN*;;O_DtnIQsSZ~4> z`l*nQNW#VgEAhVx#2qVu+kpKdX|?=06nLESZtC{}pY}M|6zd6m5txd#q4#wYcgzP4 zj-=&8q$0Z#-=eP&V>XsyP>YPp_Zh5S!whWkW!rXLh!qt2F81qHO1Bg6OVVGNubj9W z*gT5H6Orol1b#?1w+j}F)!6IpgRuoxQoc>GX#(x*FJB#S3bv>(s|#~GiUrtP4iO2= z&cLJi-z1Wb*MXm)MMx5_Zdmos-vCSS@ApYYGcW;5P$nXQ>IR(6#;c!WBhr!GfvbQw@b9nQ@eE4JC6WpKfLqDeVx`XF1#Iu~ zL8vHEp7jBa2X4WtReJZ!5Ml*ZqeHnLL^9?e;6?mzG9krBz&*g3z@W;TR#IX&;1b|r z;;)>Tw)h*CK~h9AsylEVuo(XvPKeQf?dcwivYTZ=Uu+s|0{LCU*HyfXQg4go%I3h0 zWVa$&6tM!^=Y1tsgFz{Y!#4;x8MpzM1vKK{k3x;5zy)X;OeD8PU>V@NZ$@c|ChQHt z?|{?L?gDQ$z$h#w_Fd$A#eO-_2;7XaY380Xs7b*wz^|~hrc$~Ez)WnHK?AnH@B`N) z%BF7EfNK=CI(^@k&|8`EJPcfjtp$>t(*Qu)ss+vmF2&|fr84hhgMk-YJ{z!k)-|37 zl_GtCk=Vn&5WAs1S&g@V?*R9CABcoQ0|4pB*1*qzucP_Fs5D|3)f#}8up9DRU=CJ9 zCU3f9IT?mxE6;~v5BiZoe=UQg?TzVNv{8eGO{IdDDjYitnG-~*lj00V?cL_t&_vJnjcq!haV zmjUBYpYhOyt=L|IZQtzda~bwaZ*9wG+rOJi{QvdDri1F*d=_F0U;ws&th3L4sIPcf zi4`ll7V8fpMd<>Sa*PD708a2e5UEHr8xIa@*ziGLi|z|)v>Mx;ay78X`B+3G!}h?k zhIwyiQYi8sYk-@u;$I>ni5UY-!~Z6zP~}ruY}|1@t3LL{hE>+d4QC{~N195o$C7e+KqKBmo*-O47z)Qv)Yq zZwN&4?tS1kY#;wT-$Ri+(Evca^v6o-o`vOr7s-=mU^=#y?*VLEpGcxK0FV&ugAD|} z3~c6nEFy7Q089k_fYo{s5fPvda5nG^{`0X4MO4uUOa_icMYSX#dIKP#8jZabI0Y*# zBa$|6W54t#0`E8)KfXA^n03t!s06@}YQ{ZUe zXCQa%j; zBn$ch$6SR-IJ8l06?;)5AZo`-M~@64!(yYDfa>}1(=G> z>Fc2{nV|uIWZei_2LlHHwZ4ZVfm?@o=EO!03gz8Ft9(Z z1A>uQB|}NC)mUc9!UO*aYz4na?r8uZQe0oGdSPKeFa{+e3f&^?VPA;<0Bc>3NF-1i z0Em>XJ65@Hbj#-`>@C3n&x6Xbw}FN>F$;LZ^@v1LL<0bk>TOHwfS?e&pdANMSq{9! zh7bD}v3>a>QXvfhMB>pC8w3;v1NAL26dN$;+S^bgFdvxHCT3#~`#C75zDVSB6B0>Y zFJK6*1A_W?e`j$@p&8p#KL?mg>qEZq_d6(^u1KOZ01ydJZ(ti>OKdPu%4gE)k5>VU zfh8q=zK0g$i3CLh0Fi9ygY8hL#R? Date: Sun, 10 Feb 2019 16:47:42 +0100 Subject: [PATCH 11/46] chg: [doc] backscatter.io updated --- doc/documentation.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/documentation.md b/doc/documentation.md index 31f09ed..23b4bce 100644 --- a/doc/documentation.md +++ b/doc/documentation.md @@ -2,6 +2,24 @@ ## Expansion Modules +#### [backscatter_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) + +Query backscatter.io (https://backscatter.io/). +- **features**: +>The module takes a source or destination IP address as input and displays the information known by backscatter.io. +> +> +- **input**: +>IP addresses. +- **output**: +>Text containing a history of the IP addresses especially on scanning based on backscatter.io information . +- **references**: +>https://pypi.org/project/backscatter/ +- **requirements**: +>backscatter python library + +----- + #### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) Query BGP Ranking (https://bgpranking-ng.circl.lu/). From f0ccfd2027f19378c681ab4d5f79191bae3c43cb Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Sun, 10 Feb 2019 16:56:01 +0100 Subject: [PATCH 12/46] chg: [backscatter.io] blind fix regarding undefined value --- misp_modules/modules/expansion/backscatter_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/backscatter_io.py b/misp_modules/modules/expansion/backscatter_io.py index bfa04f6..0796917 100644 --- a/misp_modules/modules/expansion/backscatter_io.py +++ b/misp_modules/modules/expansion/backscatter_io.py @@ -53,7 +53,7 @@ def handler(q=False): try: bs = Backscatter(checks['config']['api_key']) - response = bs.get_observations(query=output['value'], query_type='ip') + response = bs.get_observations(query=checks['value'], query_type='ip') if not response['success']: misperrors['error'] = '%s: %s' % (response['error'], response['message']) return misperrors From 61c0274f78907170d50dfef5c2659b6e4b35c3ae Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 11 Feb 2019 09:32:53 +0100 Subject: [PATCH 13/46] fix: Regenerated documentation --- doc/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/README.md b/doc/README.md index e47470d..44bb5b0 100644 --- a/doc/README.md +++ b/doc/README.md @@ -2,6 +2,24 @@ ## Expansion Modules +#### [backscatter_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) + +Query backscatter.io (https://backscatter.io/). +- **features**: +>The module takes a source or destination IP address as input and displays the information known by backscatter.io. +> +> +- **input**: +>IP addresses. +- **output**: +>Text containing a history of the IP addresses especially on scanning based on backscatter.io information . +- **references**: +>https://pypi.org/project/backscatter/ +- **requirements**: +>backscatter python library + +----- + #### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) Query BGP Ranking (https://bgpranking-ng.circl.lu/). From e940070956432cd7aaba0c4a8fad9c49839792e2 Mon Sep 17 00:00:00 2001 From: chrisr3d Date: Mon, 11 Feb 2019 09:43:25 +0100 Subject: [PATCH 14/46] add: [doc] Added backscatter.io logo + regenerated documentation --- doc/README.md | 2 ++ doc/expansion/backscatter_io.json | 1 + 2 files changed, 3 insertions(+) diff --git a/doc/README.md b/doc/README.md index 44bb5b0..c32a8c4 100644 --- a/doc/README.md +++ b/doc/README.md @@ -4,6 +4,8 @@ #### [backscatter_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) + + Query backscatter.io (https://backscatter.io/). - **features**: >The module takes a source or destination IP address as input and displays the information known by backscatter.io. diff --git a/doc/expansion/backscatter_io.json b/doc/expansion/backscatter_io.json index 22123a5..a8475c5 100644 --- a/doc/expansion/backscatter_io.json +++ b/doc/expansion/backscatter_io.json @@ -2,6 +2,7 @@ "description": "Query backscatter.io (https://backscatter.io/).", "requirements": ["backscatter python library"], "features": "The module takes a source or destination IP address as input and displays the information known by backscatter.io.\n\n", + "logo": "logos/backscatter_io.png", "references": ["https://pypi.org/project/backscatter/"], "input": "IP addresses.", "output": "Text containing a history of the IP addresses especially on scanning based on backscatter.io information ." From 0bf27c1b69b12ea4bbefad830a89791d558eca07 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 11 Feb 2019 14:23:18 +0100 Subject: [PATCH 15/46] chg: [btc_scam_check] fix spacing for making flake 8 happy --- misp_modules/modules/expansion/btc_scam_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_modules/modules/expansion/btc_scam_check.py b/misp_modules/modules/expansion/btc_scam_check.py index 9f9a7d6..f551926 100644 --- a/misp_modules/modules/expansion/btc_scam_check.py +++ b/misp_modules/modules/expansion/btc_scam_check.py @@ -19,6 +19,7 @@ moduleconfig = [] url = 'bl.btcblack.it' + def handler(q=False): if q is False: return False From 9abc3a4b0a323cdd2617be673847422ad8d39fb5 Mon Sep 17 00:00:00 2001 From: iwitz Date: Fri, 15 Feb 2019 10:16:52 +0100 Subject: [PATCH 16/46] add: rhel installation instructions --- README.md | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b60accd..ee4f2f8 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127. /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ -## How to install and start MISP modules? +## How to install and start MISP modules on Debian-based distributions ? ~~~~bash sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick @@ -115,6 +115,42 @@ sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127. /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ +## How to install and start MISP modules on RHEL-based distributions ? +As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository. +~~~~bash +yum install rh-ruby22 +cd /var/www/MISP +git clone https://github.com/MISP/misp-modules.git +cd misp-modules +scl enable rh-python36 ‘python3 –m pip install cryptography’ +scl enable rh-python36 ‘python3 –m pip install -I -r REQUIREMENTS’ +scl enable rh-python36 ‘python3 –m pip install –I .’ +scl enable rh-ruby22 ‘gem install asciidoctor-pdf –pre’ +~~~~ +Create the service file /etc/systemd/system/misp-workers.service : +~~~~ +[Unit] +Description=MISP's modules +After=misp-workers.service + +[Service] +Type=simple +User=apache +Group=apache +ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 ‘/opt/rh/rh-python36/root/bin/misp-modules –l 127.0.0.1 –s’ +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +~~~~ +The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service. +Then, enable the misp-modules service and start it ; +~~~~bash +systemctl daemon-reload +systemctl enable --now misp-modules +~~~~ + ## How to add your own MISP modules? Create your module in [misp_modules/modules/expansion/](misp_modules/modules/expansion/), [misp_modules/modules/export_mod/](misp_modules/modules/export_mod/), or [misp_modules/modules/import_mod/](misp_modules/modules/import_mod/). The module should have at minimum three functions: From 2753f354ab3291985b8beb9aa745fe30d1e6b2a8 Mon Sep 17 00:00:00 2001 From: Vincent-CIRCL Date: Mon, 18 Feb 2019 14:27:16 +0100 Subject: [PATCH 17/46] test update --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index df7f879..1b4c731 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -12,7 +12,7 @@ from pymisp import MISPEvent misperrors = {'error': 'Error'} -moduleinfo = {'version': '1', +moduleinfo = {'version': '42', 'author': 'Raphaël Vinot', 'description': 'Simple export to PDF', 'module-type': ['export'], From be01d547791f7f9950dad170b3908155e15ff3c8 Mon Sep 17 00:00:00 2001 From: Vincent-CIRCL Date: Mon, 18 Feb 2019 15:23:57 +0100 Subject: [PATCH 18/46] print values --- misp_modules/modules/export_mod/pdfexport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 1b4c731..074e473 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -144,6 +144,8 @@ def handler(q=False): return False for evt in request['data']: + print(request['data']) + report = ReportGenerator() report.report_headers() report.from_event(evt) From 2d29ce11bbf8dadd82fe886330476488df32c140 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Thu, 21 Feb 2019 15:42:18 +0100 Subject: [PATCH 19/46] Test 1 - PDF call --- misp_modules/modules/export_mod/pdfexport.py | 68 +++++++------------- 1 file changed, 22 insertions(+), 46 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 074e473..23d0edd 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -7,61 +7,26 @@ import shlex import subprocess import base64 -from pymisp import MISPEvent +from pymisp import MISPEvent, reportlab_generator misperrors = {'error': 'Error'} -moduleinfo = {'version': '42', - 'author': 'Raphaël Vinot', +moduleinfo = {'version': '2', + 'author': 'Vincent Falconieri (prev. Raphaël Vinot)', 'description': 'Simple export to PDF', 'module-type': ['export'], 'require_standard_format': True} moduleconfig = [] - mispattributes = {} + outputFileExtension = "pdf" responseType = "application/pdf" types_to_attach = ['ip-dst', 'url', 'domain'] objects_to_attach = ['domain-ip'] -headers = """ -:toc: right -:toclevels: 1 -:toc-title: Daily Report -:icons: font -:sectanchors: -:sectlinks: -= Daily report by {org_name} -{date} - -:icons: font - -""" - -event_level_tags = """ -IMPORTANT: This event is classified TLP:{value}. - -{expanded} - -""" - -attributes = """ -=== Indicator(s) of compromise - -{list_attributes} - -""" - -title = """ -== ({internal_id}) {title} - -{summary} - -""" - class ReportGenerator(): def __init__(self): @@ -79,6 +44,9 @@ class ReportGenerator(): self.misp_event = MISPEvent() self.misp_event.load(event) + ''' + + def attributes(self): if not self.misp_event.attributes: return '' @@ -132,7 +100,7 @@ class ReportGenerator(): self.report += self.title() self.report += self.event_level_tags() self.report += self.attributes() - + ''' def handler(q=False): if q is False: @@ -144,19 +112,27 @@ def handler(q=False): return False for evt in request['data']: + + ''' + print(" DATA ") print(request['data']) + + reportlab_generator. report = ReportGenerator() report.report_headers() report.from_event(evt) report.asciidoc() - command_line = 'asciidoctor-pdf -' - args = shlex.split(command_line) - with subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) as process: - cmd_out, cmd_err = process.communicate( - input=report.report.encode('utf-8')) - return {'response': [], 'data': str(base64.b64encode(cmd_out), 'utf-8')} + print(" REPORT : ") + print(report) + ''' + misp_event = MISPEvent() + misp_event.load(request['data']) + + pdf = reportlab_generator.get_base64_from_buffer(reportlab_generator.convert_event_in_pdf_buffer(misp_event)) + + return {'response': [], 'data': str(pdf, 'utf-8')} def introspection(): From a93b34208f358c76184b725f656f43f39dbd8e18 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 22 Feb 2019 10:14:22 +0100 Subject: [PATCH 20/46] fix: [pdfexport] Bugfix on PyMisp exportpdf call --- misp_modules/modules/export_mod/pdfexport.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 23d0edd..cb4e297 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -7,8 +7,10 @@ import shlex import subprocess import base64 -from pymisp import MISPEvent, reportlab_generator +print("test PDF pdf export (reportlab generator import)") +from pymisp import MISPEvent +from pymisp.tools import reportlab_generator misperrors = {'error': 'Error'} @@ -45,8 +47,6 @@ class ReportGenerator(): self.misp_event.load(event) ''' - - def attributes(self): if not self.misp_event.attributes: return '' @@ -127,12 +127,13 @@ def handler(q=False): print(" REPORT : ") print(report) ''' - misp_event = MISPEvent() - misp_event.load(request['data']) - pdf = reportlab_generator.get_base64_from_buffer(reportlab_generator.convert_event_in_pdf_buffer(misp_event)) + misp_event = MISPEvent() + misp_event.load(evt) - return {'response': [], 'data': str(pdf, 'utf-8')} + pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event)) + + return {'response': [], 'data': str(pdf, 'utf-8')} def introspection(): @@ -164,3 +165,8 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo + +import pprint + +if __name__ == "__main__": + pprint.pprint("test") \ No newline at end of file From 40cd32f1b8c073caf20d133ca0e9780ec2f602cc Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 22 Feb 2019 10:25:12 +0100 Subject: [PATCH 21/46] tidy: Remove old dead export code --- misp_modules/modules/export_mod/pdfexport.py | 78 -------------------- 1 file changed, 78 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index cb4e297..ef3d775 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -7,8 +7,6 @@ import shlex import subprocess import base64 -print("test PDF pdf export (reportlab generator import)") - from pymisp import MISPEvent from pymisp.tools import reportlab_generator @@ -46,62 +44,6 @@ class ReportGenerator(): self.misp_event = MISPEvent() self.misp_event.load(event) - ''' - def attributes(self): - if not self.misp_event.attributes: - return '' - list_attributes = [] - for attribute in self.misp_event.attributes: - if attribute.type in types_to_attach: - list_attributes.append("* {}".format(attribute.value)) - for obj in self.misp_event.Object: - if obj.name in objects_to_attach: - for attribute in obj.Attribute: - if attribute.type in types_to_attach: - list_attributes.append("* {}".format(attribute.value)) - return attributes.format(list_attributes="\n".join(list_attributes)) - - def _get_tag_info(self, machinetag): - return self.taxonomies.revert_machinetag(machinetag) - - def report_headers(self): - content = {'org_name': 'name', - 'date': date.today().isoformat()} - self.report += headers.format(**content) - - def event_level_tags(self): - if not self.misp_event.Tag: - return '' - for tag in self.misp_event.Tag: - # Only look for TLP for now - if tag['name'].startswith('tlp'): - tax, predicate = self._get_tag_info(tag['name']) - return self.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) - - def title(self): - internal_id = '' - summary = '' - # Get internal refs for report - if not hasattr(self.misp_event, 'Object'): - return '' - for obj in self.misp_event.Object: - if obj.name != 'report': - continue - for a in obj.Attribute: - if a.object_relation == 'case-number': - internal_id = a.value - if a.object_relation == 'summary': - summary = a.value - - return title.format(internal_id=internal_id, title=self.misp_event.info, - summary=summary) - - def asciidoc(self, lang='en'): - self.report += self.title() - self.report += self.event_level_tags() - self.report += self.attributes() - ''' - def handler(q=False): if q is False: return False @@ -113,21 +55,6 @@ def handler(q=False): for evt in request['data']: - ''' - print(" DATA ") - print(request['data']) - - reportlab_generator. - - report = ReportGenerator() - report.report_headers() - report.from_event(evt) - report.asciidoc() - - print(" REPORT : ") - print(report) - ''' - misp_event = MISPEvent() misp_event.load(evt) @@ -165,8 +92,3 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo - -import pprint - -if __name__ == "__main__": - pprint.pprint("test") \ No newline at end of file From 9f0f6e71e87f658ebb4518f9c9142db1cf0efe1e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 22 Feb 2019 12:15:28 +0100 Subject: [PATCH 22/46] chg: PyMISP requirement --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 0720e90..6f3d1b2 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@2c877f2aec11b7f5d2f23dfc5ce7398b2ce33b48#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@ccd7565d3ce4693b96ea2352792099b40c53e494#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe From 66ee78e7af41f7062e3239e7fe676c80dc8a378d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 24 Feb 2019 16:02:13 -0800 Subject: [PATCH 23/46] new: Add systemd launcher --- etc/systemd/system/misp-modules.service | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 etc/systemd/system/misp-modules.service diff --git a/etc/systemd/system/misp-modules.service b/etc/systemd/system/misp-modules.service new file mode 100644 index 0000000..3ff05ae --- /dev/null +++ b/etc/systemd/system/misp-modules.service @@ -0,0 +1,14 @@ +[Unit] +Description=System-wide instance of the MISP Modules +After=network.target + +[Service] +User=www-data +Group=www-data +WorkingDirectory=/usr/local/src/misp-modules +Environment="PATH=/var/www/MISP/venv/bin" +ExecStart=misp-modules -l 127.0.0.1 -s + +[Install] +WantedBy=multi-user.target + From 43d2ae6203a484e4614166a610a3d6bc73c12b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 24 Feb 2019 18:20:28 -0800 Subject: [PATCH 24/46] fix: systemd service --- etc/systemd/system/misp-modules.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/systemd/system/misp-modules.service b/etc/systemd/system/misp-modules.service index 3ff05ae..99cd102 100644 --- a/etc/systemd/system/misp-modules.service +++ b/etc/systemd/system/misp-modules.service @@ -7,7 +7,7 @@ User=www-data Group=www-data WorkingDirectory=/usr/local/src/misp-modules Environment="PATH=/var/www/MISP/venv/bin" -ExecStart=misp-modules -l 127.0.0.1 -s +ExecStart=/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s [Install] WantedBy=multi-user.target From a3a871f2faa6b27c79e8c55bd0b128b9edbc7cf3 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Mon, 25 Feb 2019 15:51:33 +0100 Subject: [PATCH 25/46] fix [exportpdf] update parameters for links generation --- misp_modules/modules/export_mod/pdfexport.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index ef3d775..977ee87 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -18,7 +18,9 @@ moduleinfo = {'version': '2', 'module-type': ['export'], 'require_standard_format': True} -moduleconfig = [] +# config fields that your code expects from the site admin +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata"] + mispattributes = {} outputFileExtension = "pdf" @@ -53,12 +55,19 @@ def handler(q=False): if 'data' not in request: return False + config = {} + + # Construct config object for reportlab_generator + for config_item in moduleconfig : + if (request.get('config')) and (request['config'].get(config_item) is not None): + config[config_item] = request['config'].get(config_item) + for evt in request['data']: misp_event = MISPEvent() misp_event.load(evt) - pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event)) + pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event, config)) return {'response': [], 'data': str(pdf, 'utf-8')} From 0d8ead483e204045eaff9af35bc61836488c30fc Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 16:18:41 +0100 Subject: [PATCH 26/46] chg: [PyMISP] dep updated to the latest version --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 6f3d1b2..e42481b 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@ccd7565d3ce4693b96ea2352792099b40c53e494#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@345f055844fed0acdfb34c52d96d1751728bb82c#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe From 9e48b3994a70cc9447c279470a3dce9b23a1d278 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 20:29:04 +0100 Subject: [PATCH 27/46] chg: [requirements] updated --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index e42481b..69b0568 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@345f055844fed0acdfb34c52d96d1751728bb82c#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@634ecc3ac308d01ebf5f5fbb9aace7746a2b8707#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe From bbe7fe51e70ee7ef24fb9a9573d7ad5a85ddaf9f Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 20:34:48 +0100 Subject: [PATCH 28/46] chg: [pipenv] Pipfile.lock updated --- Pipfile.lock | 174 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 106 insertions(+), 68 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 19f32f0..1c08572 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -150,9 +150,9 @@ }, "httplib2": { "hashes": [ - "sha256:f61fb838a94ce3b349aa32c92fd8430f7e3511afdb18bf9640d647e30c90a6d6" + "sha256:4ba6b8fd77d0038769bf3c33c9a96a6f752bc4cdf739701fdcaf210121f399d4" ], - "version": "==0.12.0" + "version": "==0.12.1" }, "idna": { "hashes": [ @@ -177,10 +177,10 @@ }, "jsonschema": { "hashes": [ - "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", - "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02" + "sha256:acc8a90c31d11060516cfd0b414b9f8bcf4bc691b21f0f786ea57dd5255c79db", + "sha256:dd3f8ecb1b52d94d45eedb67cb86cac57b94ded562c5d98f63719e55ce58557b" ], - "version": "==2.6.0" + "version": "==3.0.0" }, "maclookup": { "hashes": [ @@ -281,22 +281,22 @@ }, "psutil": { "hashes": [ - "sha256:04d2071100aaad59f9bcbb801be2125d53b2e03b1517d9fed90b45eea51d297e", - "sha256:1aba93430050270750d046a179c5f3d6e1f5f8b96c20399ba38c596b28fc4d37", - "sha256:3ac48568f5b85fee44cd8002a15a7733deca056a191d313dbf24c11519c0c4a8", - "sha256:96f3fdb4ef7467854d46ad5a7e28eb4c6dc6d455d751ddf9640cd6d52bdb03d7", - "sha256:b755be689d6fc8ebc401e1d5ce5bac867e35788f10229e166338484eead51b12", - "sha256:c8ee08ad1b716911c86f12dc753eb1879006224fd51509f077987bb6493be615", - "sha256:d0c4230d60376aee0757d934020b14899f6020cd70ef8d2cb4f228b6ffc43e8f", - "sha256:d23f7025bac9b3e38adc6bd032cdaac648ac0074d18e36950a04af35458342e8", - "sha256:f0fcb7d3006dd4d9ccf3ccd0595d44c6abbfd433ec31b6ca177300ee3f19e54e" + "sha256:5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a", + "sha256:72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810", + "sha256:77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c", + "sha256:8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d", + "sha256:a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191", + "sha256:ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f", + "sha256:c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a", + "sha256:ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59", + "sha256:ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd" ], - "version": "==5.5.0" + "version": "==5.5.1" }, "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "7e698f87366e6f99b4d0d11852737db28e3ddc62", + "ref": "37c97ae252ec4bf1d67733a49d4895c8cb009cf9", "subdirectory": "client" }, "pydnstrails": { @@ -333,12 +333,12 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "2c877f2aec11b7f5d2f23dfc5ce7398b2ce33b48" + "ref": "634ecc3ac308d01ebf5f5fbb9aace7746a2b8707" }, "pyonyphe": { "editable": true, "git": "https://github.com/sebdraven/pyonyphe", - "ref": "66329baeee7cab844f2203c047c2551828eaf14d" + "ref": "cbb0168d5cb28a9f71f7ab3773164a7039ccdb12" }, "pyparsing": { "hashes": [ @@ -361,6 +361,12 @@ "index": "pypi", "version": "==2.1" }, + "pyrsistent": { + "hashes": [ + "sha256:3ca82748918eb65e2d89f222b702277099aca77e34843c5eb9d52451173970e2" + ], + "version": "==0.14.11" + }, "pytesseract": { "hashes": [ "sha256:11c20321595b6e2e904b594633edf1a717212b13bac7512986a2d807b8849770" @@ -370,10 +376,10 @@ }, "python-dateutil": { "hashes": [ - "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", - "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], - "version": "==2.7.5" + "version": "==2.8.0" }, "pyyaml": { "hashes": [ @@ -400,10 +406,43 @@ }, "redis": { "hashes": [ - "sha256:74c892041cba46078ae1ef845241548baa3bd3634f9a6f0f952f006eb1619c71", - "sha256:7ba8612bbfd966dea8c62322543fed0095da2834dbd5a7c124afbc617a156aa7" + "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54", + "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775" ], - "version": "==3.1.0" + "version": "==3.2.0" + }, + "reportlab": { + "hashes": [ + "sha256:069f684cd0aaa518a27dc9124aed29cee8998e21ddf19604e53214ec8462bdd7", + "sha256:09b68ec01d86b4b120456b3f3202570ec96f57624e3a4fc36f3829323391daa4", + "sha256:0c32be9a406172c29ea20ff55a709ccac1e7fb09f15aba67cb7b455fd1d3dbe0", + "sha256:233196cf25e97cfe7c452524ea29d9a4909f1cb66599299233be1efaaaa7a7a3", + "sha256:2b5e4533f3e5b962835a5ce44467e66d1ecc822761d1b508077b5087a06be338", + "sha256:2e860bcdace5a558356802a92ae8658d7e5fdaa00ded82e83a3f2987c562cb66", + "sha256:3546029e63a9a9dc24ee38959eb417678c2425b96cd27b31e09e216dafc94666", + "sha256:4452b93f9c73b6b70311e7d69082d64da81b38e91bfb4766397630092e6da6fd", + "sha256:528c74a1c6527d1859c2c7a64a94a1cba485b00175162ea23699ae58a1e94939", + "sha256:6116e750f98018febc08dfee6df20446cf954adbcfa378d2c703d56c8864aff3", + "sha256:6b2b3580c647d75ef129172cb3da648cdb24566987b0b59c5ebb80ab770748d6", + "sha256:727b5f2bed08552d143fc99649b1863c773729f580a416844f9d9967bb0a1ae8", + "sha256:74c24a3ec0a3d4f8acb13a07192f45bdb54a1cc3c2286241677e7e8bcd5011fa", + "sha256:98ccd2f8b4f8636db05f3f14db0b471ad6bb4b66ae0dc9052c4822b3bd5d6a7d", + "sha256:a5905aa567946bc938b489a7249c7890c3fd3c9b7b5680dece5bc551c2ddbe0d", + "sha256:acbb7f676b8586b770719e9683eda951fdb38eb7970d46fcbf3cdda88d912a64", + "sha256:b5e30f865add48cf880f1c363eb505b97f2f7baaa88c155f87a335a76515a3e5", + "sha256:be2a7c33a2c28bbd3f453ffe4f0e5200b88c803a097f4cf52d69c6b53fad7a8f", + "sha256:c356bb600f59ac64955813d6497a08bfd5d0c451cb5829b61e3913d0ac084e26", + "sha256:c7ec4ae2393beab584921b1287a04e94fd98c28315e348362d89b85f4b464546", + "sha256:d476edc831bb3e9ebd04d1403abaf3ea57b3e4c2276c91a54fdfb6efbd3f9d97", + "sha256:db059e1a0691c872784062421ec51848539eb4f5210142682e61059a5ca7cc55", + "sha256:dd423a6753509ab14a0ac1b5be39d219c8f8d3781cce3deb4f45eda31969b5e8", + "sha256:ed9b7c0d71ce6fe2b31c6cde530ad8238632b876a5d599218739bda142a77f7c", + "sha256:f0a2465af4006f97b05e1f1546d67d3a3213d414894bf28be7f87f550a7f4a55", + "sha256:f20bfe26e57e8e1f575a9e0325be04dd3562db9f247ffdd73b5d4df6dec53bc2", + "sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c", + "sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9" + ], + "version": "==3.5.13" }, "requests": { "hashes": [ @@ -422,10 +461,10 @@ }, "shodan": { "hashes": [ - "sha256:c40abb6ff2fd66bdee9f773746fb961eefdfaa8e720a07cb12fb70def136268d" + "sha256:f93b7199e89eecf5c84647f66316c2c044c3aebfc1fe4d9caa43dfda07f74c4e" ], "index": "pypi", - "version": "==1.10.4" + "version": "==1.11.1" }, "sigmatools": { "hashes": [ @@ -443,10 +482,10 @@ }, "soupsieve": { "hashes": [ - "sha256:466910df7561796a60748826781ebe9a888f7a1668a636ae86783f44d10aae73", - "sha256:87db12ae79194f0ff9808d2b1641c4f031ae39ffa3cab6b907ea7c1e5e5ed445" + "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b", + "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26" ], - "version": "==1.7.3" + "version": "==1.8" }, "sparqlwrapper": { "hashes": [ @@ -500,49 +539,48 @@ "uwhois": { "editable": true, "git": "https://github.com/Rafiot/uwhoisd.git", - "ref": "f6f035e52213c8abc20f2084d28cfffb399457cb", + "ref": "411572840eba4c72dc321c549b36a54ed5cea9de", "subdirectory": "client" }, "vulners": { "hashes": [ - "sha256:5f05404041cfaa8e5367bf884fc9ee319ebf34bedc495d7f84c433fa121cdb49", - "sha256:919b24df64ea55b6a8ba13e2a0530578f8a4be6a9cee257bf2214046e81c6f35", - "sha256:d45ecb13f5111947056a2dcc071b3e3fd45f6ad654eda06526245bba3850325e" + "sha256:40041bcf893fa1bfaf29c650369d9a249991911f28b4d8795f7bc06508013e14", + "sha256:6d00709300dcc7e2727499d8a60f51eaced1dc6b63cc19cb8a4b065b658c51aa", + "sha256:de8cef247c9852c39bd54434e63026b46bdb2bd4ca22813bf66626b7d359b0f3" ], "index": "pypi", - "version": "==1.4.0" + "version": "==1.4.4" }, "wand": { "hashes": [ - "sha256:3e59e4bda9ef9d643d90e881cc950c8eee1508ec2cde1c150a1cbd5a12c1c007", - "sha256:52763dbf65d00cf98d7bc910b49329eea15896249c5555d47e169f2b6efbe166" + "sha256:7d6b8dc9d4eaccc430b9c86e6b749013220c994970a3f39e902b397e2fa732c3", + "sha256:cc0b5c9cd50fecd10dc8888b739dd5984c6f8085d2954f34903b83ca39a91236" ], "index": "pypi", - "version": "==0.5.0" + "version": "==0.5.1" }, "xlsxwriter": { "hashes": [ - "sha256:7cc07619760641b67112dbe0df938399d4d915d9b9924bb58eb5c17384d29cc6", - "sha256:ae22658a0fc5b9e875fa97c213d1ffd617d86dc49bf08be99ebdac814db7bf36" + "sha256:de9ef46088489915eaaee00c7088cff93cf613e9990b46b933c98eb46f21b47f", + "sha256:df96eafc3136d9e790e35d6725b473e46ada6f585c1f6519da69b27f5c8873f7" ], - "version": "==1.1.2" + "version": "==1.1.5" }, "yara-python": { "hashes": [ - "sha256:03e5c5e333c8572e7994b0b11964d515d61a393f23c5e272f8d0e4229f368c58", - "sha256:0423e08bd618752a028ac0405ff8e0103f3a8fd607dde7618a64a4c010c3757b", - "sha256:0a0dd632dcdb347d1a9a8b1f6a83b3a77d5e63f691357ea4021fb1cf1d7ff0a4", - "sha256:728b99627a8072a877eaaa4dafb4eff39d1b14ff4fd70d39f18899ce81e29625", - "sha256:7cb0d5724eccfa52e1bcd352a56cb4dc422aa51f5f6d0945d4f830783927513b", - "sha256:8c76531e89806c0309586dd4863a972d12f1d5d63261c6d4b9331a99859fd1d8", - "sha256:9472676583e212bc4e17c2236634e02273d53c872b350f0571b48e06183de233", - "sha256:9735b680a7d95c1d3f255c351bb067edc62cdb3c0999f7064278cb2c85245405", - "sha256:997f104590167220a9af5564c042ec4d6534261e7b8a5b49655d8dffecc6b8a2", - "sha256:a48e071d02a3699363e628ac899b5b7237803bcb4b512c92ebcb4fb9b1488497", - "sha256:b67c0d75a6519ca357b4b85ede9768c96a81fff20fbc169bd805ff009ddee561" + "sha256:0d002170b2f2c56ff75c846ad1e6765f59d4569e81494c76f15243197e4a974c", + "sha256:16be7c7623685b4b2813db33a39553d6faef236ddffa0758c08e2071ab11ed84", + "sha256:2031ac6ac01754dbc82b5a47b69cb91302c6b66ea9d9f2f27cc2eaf771e19c14", + "sha256:228a96efc86c766d968c984bd80f5ebb0bb775afb9045c10fb632e2b7275c9c1", + "sha256:468a9770e6b578f0562a540b6cb5cafd4122bea989404b53440d4eb065d54eda", + "sha256:752d12a795159b806cd74ab7f0fd7c3a14cb6e17c9e4a818511dc7a4932b15df", + "sha256:755406cb5fa944d5e0dd097a4b25c3fcdd5ba244f0367114afed1ba30ccd2a12", + "sha256:7936c10c8802fc279802dcdda8270d3fda5c3d3c8fbe6bb02010934ed30b8929", + "sha256:95c8d39ee5938744dbd8e0153ec6d466f8a4ed11b8ac7b1068f498c26a292b65", + "sha256:cfd00cfb7bcbe862b0793f91b5393bad3fb37da78883af19924059367ba80f51" ], "index": "pypi", - "version": "==3.8.1" + "version": "==3.9.0" }, "yarl": { "hashes": [ @@ -643,11 +681,11 @@ }, "flake8": { "hashes": [ - "sha256:09b9bb539920776da542e67a570a5df96ff933c9a08b62cfae920bcc789e4383", - "sha256:e0f8cd519cfc0072c0ee31add5def09d2b3ef6040b34dc426445c3af9b02163c" + "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", + "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" ], "index": "pypi", - "version": "==3.7.4" + "version": "==3.7.7" }, "idna": { "hashes": [ @@ -665,11 +703,11 @@ }, "more-itertools": { "hashes": [ - "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", - "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", - "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9" + "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", + "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" ], - "version": "==5.0.0" + "markers": "python_version > '2.7'", + "version": "==6.0.0" }, "nose": { "hashes": [ @@ -682,17 +720,17 @@ }, "pluggy": { "hashes": [ - "sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616", - "sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a" + "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", + "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" ], - "version": "==0.8.1" + "version": "==0.9.0" }, "py": { "hashes": [ - "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", - "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" ], - "version": "==1.7.0" + "version": "==1.8.0" }, "pycodestyle": { "hashes": [ @@ -710,11 +748,11 @@ }, "pytest": { "hashes": [ - "sha256:65aeaa77ae87c7fc95de56285282546cfa9c886dc8e5dc78313db1c25e21bc07", - "sha256:6ac6d467d9f053e95aaacd79f831dbecfe730f419c6c7022cb316b365cd9199d" + "sha256:067a1d4bf827ffdd56ad21bd46674703fce77c5957f6c1eef731f6146bfcef1c", + "sha256:9687049d53695ad45cf5fdc7bbd51f0c49f1ea3ecfc4b7f3fde7501b541f17f4" ], "index": "pypi", - "version": "==4.2.0" + "version": "==4.3.0" }, "requests": { "hashes": [ From 637d7f25381b33f30a947cc23c50325f088e21cc Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 20:42:45 +0100 Subject: [PATCH 29/46] chg: [requirements] reportlab added --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index 69b0568..4891c60 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -47,6 +47,7 @@ rdflib==4.2.2 redis==3.1.0 requests-cache==0.4.13 requests==2.21.0 +reportlab shodan==1.10.4 sigmatools==0.7.1 six==1.12.0 From b0ea67e393f91aefbf770123ce6f4cd0699d0e5e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 21:11:24 +0100 Subject: [PATCH 30/46] chg: [pipenv] fix the temporary issue that python-yara is not officially released --- Pipfile | 3 ++- Pipfile.lock | 26 ++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Pipfile b/Pipfile index c086e62..45c05f5 100644 --- a/Pipfile +++ b/Pipfile @@ -25,12 +25,13 @@ pytesseract = "*" pygeoip = "*" beautifulsoup4 = "*" oauth2 = "*" -yara-python = ">=3.8.0" +yara-python = "==3.8.1" sigmatools = "*" stix2-patterns = "*" maclookup = "*" vulners = "*" blockchain = "*" +reportlab = "*" pyintel471 = {editable = true,git = "https://github.com/MISP/PyIntel471.git"} shodan = "*" Pillow = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1c08572..9e6265d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f501a84bdd41ca21a2af020278ce030985cccd5f2f5683cd075797be4523587d" + "sha256": "d0cd64bfe7702365d3ea66d1f51a1ec8592df2490899e7e163fe38f97172561e" }, "pipfile-spec": 6, "requires": { @@ -442,6 +442,7 @@ "sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c", "sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9" ], + "index": "pypi", "version": "==3.5.13" }, "requests": { @@ -568,19 +569,20 @@ }, "yara-python": { "hashes": [ - "sha256:0d002170b2f2c56ff75c846ad1e6765f59d4569e81494c76f15243197e4a974c", - "sha256:16be7c7623685b4b2813db33a39553d6faef236ddffa0758c08e2071ab11ed84", - "sha256:2031ac6ac01754dbc82b5a47b69cb91302c6b66ea9d9f2f27cc2eaf771e19c14", - "sha256:228a96efc86c766d968c984bd80f5ebb0bb775afb9045c10fb632e2b7275c9c1", - "sha256:468a9770e6b578f0562a540b6cb5cafd4122bea989404b53440d4eb065d54eda", - "sha256:752d12a795159b806cd74ab7f0fd7c3a14cb6e17c9e4a818511dc7a4932b15df", - "sha256:755406cb5fa944d5e0dd097a4b25c3fcdd5ba244f0367114afed1ba30ccd2a12", - "sha256:7936c10c8802fc279802dcdda8270d3fda5c3d3c8fbe6bb02010934ed30b8929", - "sha256:95c8d39ee5938744dbd8e0153ec6d466f8a4ed11b8ac7b1068f498c26a292b65", - "sha256:cfd00cfb7bcbe862b0793f91b5393bad3fb37da78883af19924059367ba80f51" + "sha256:03e5c5e333c8572e7994b0b11964d515d61a393f23c5e272f8d0e4229f368c58", + "sha256:0423e08bd618752a028ac0405ff8e0103f3a8fd607dde7618a64a4c010c3757b", + "sha256:0a0dd632dcdb347d1a9a8b1f6a83b3a77d5e63f691357ea4021fb1cf1d7ff0a4", + "sha256:728b99627a8072a877eaaa4dafb4eff39d1b14ff4fd70d39f18899ce81e29625", + "sha256:7cb0d5724eccfa52e1bcd352a56cb4dc422aa51f5f6d0945d4f830783927513b", + "sha256:8c76531e89806c0309586dd4863a972d12f1d5d63261c6d4b9331a99859fd1d8", + "sha256:9472676583e212bc4e17c2236634e02273d53c872b350f0571b48e06183de233", + "sha256:9735b680a7d95c1d3f255c351bb067edc62cdb3c0999f7064278cb2c85245405", + "sha256:997f104590167220a9af5564c042ec4d6534261e7b8a5b49655d8dffecc6b8a2", + "sha256:a48e071d02a3699363e628ac899b5b7237803bcb4b512c92ebcb4fb9b1488497", + "sha256:b67c0d75a6519ca357b4b85ede9768c96a81fff20fbc169bd805ff009ddee561" ], "index": "pypi", - "version": "==3.9.0" + "version": "==3.8.1" }, "yarl": { "hashes": [ From e7fd7e8eb20ed92ab5b09e83d7acf004fa366b6f Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 21:18:26 +0100 Subject: [PATCH 31/46] chg: [pdfexport] make flake8 happy --- misp_modules/modules/export_mod/pdfexport.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 977ee87..6b0c12f 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -1,11 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from datetime import date import json -import shlex -import subprocess -import base64 from pymisp import MISPEvent from pymisp.tools import reportlab_generator @@ -46,6 +42,7 @@ class ReportGenerator(): self.misp_event = MISPEvent() self.misp_event.load(event) + def handler(q=False): if q is False: return False @@ -58,12 +55,11 @@ def handler(q=False): config = {} # Construct config object for reportlab_generator - for config_item in moduleconfig : + for config_item in moduleconfig: if (request.get('config')) and (request['config'].get(config_item) is not None): config[config_item] = request['config'].get(config_item) for evt in request['data']: - misp_event = MISPEvent() misp_event.load(evt) From 2a59c6becc3e24c56febe838adbcc965e929d49b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 21:33:47 +0100 Subject: [PATCH 32/46] chg: [doc] PDF export --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee4f2f8..501e54f 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). * [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). * [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. -* [Simple PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export in PDF (required: asciidoctor-pdf). +* [PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. * [Nexthink query format](misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format. * [osquery](misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format. * [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. From a770bfc5934157f895f211539911f6dcd7481e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 26 Feb 2019 16:43:08 -0800 Subject: [PATCH 33/46] chg: Bump dependencies, add update script --- Pipfile.lock | 2 +- REQUIREMENTS | 32 ++++++++++++++++---------------- setup.py | 1 + tools/update_misp_modules.sh | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 17 deletions(-) create mode 100755 tools/update_misp_modules.sh diff --git a/Pipfile.lock b/Pipfile.lock index 9e6265d..85cd9db 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -333,7 +333,7 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "634ecc3ac308d01ebf5f5fbb9aace7746a2b8707" + "ref": "62e047f3c1972e21aa36a8882bebf4488cdc1f84" }, "pyonyphe": { "editable": true, diff --git a/REQUIREMENTS b/REQUIREMENTS index 4891c60..4709747 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,17 +1,16 @@ -i https://pypi.org/simple -e . --e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client +-e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@634ecc3ac308d01ebf5f5fbb9aace7746a2b8707#egg=pymisp --e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client +-e git+https://github.com/MISP/PyMISP.git@62e047f3c1972e21aa36a8882bebf4488cdc1f84#egg=pymisp +-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails --e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe +-e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 attrs==18.2.0 -backscatter==0.2.3 beautifulsoup4==4.7.1 blockchain==1.4.4 certifi==2018.11.29 @@ -24,42 +23,43 @@ domaintools-api==0.3.3 enum-compat==0.0.2 ez-setup==0.9 future==0.17.1 -httplib2==0.12.0 +httplib2==0.12.1 idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 isodate==0.6.0 -jsonschema==2.6.0 +jsonschema==3.0.0 maclookup==1.0.3 multidict==4.5.2 oauth2==1.9.0.post1 passivetotal==1.0.30 pillow==5.4.1 -psutil==5.5.0 +psutil==5.5.1 pyeupi==1.0 pygeoip==0.3.2 pyparsing==2.3.1 pypdns==1.3 pypssl==2.1 +pyrsistent==0.14.11 pytesseract==0.2.6 -python-dateutil==2.7.5 +python-dateutil==2.8.0 pyyaml==3.13 rdflib==4.2.2 -redis==3.1.0 +redis==3.2.0 +reportlab==3.5.13 requests-cache==0.4.13 requests==2.21.0 -reportlab -shodan==1.10.4 +shodan==1.11.1 sigmatools==0.7.1 six==1.12.0 -soupsieve==1.7.3 +soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 tornado==5.1.1 url-normalize==1.4.1 urlarchiver==0.2 urllib3==1.24.1 -vulners==1.4.0 -wand==0.5.0 -xlsxwriter==1.1.2 +vulners==1.4.4 +wand==0.5.1 +xlsxwriter==1.1.5 yara-python==3.8.1 yarl==1.3.0 diff --git a/setup.py b/setup.py index fc78750..55ed8b7 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ setup( description='MISP modules are autonomous modules that can be used for expansion and other services in MISP', packages=find_packages(), entry_points={'console_scripts': ['misp-modules = misp_modules:main']}, + scripts=['tools/update_misp_modules.sh'], test_suite="tests", classifiers=[ 'License :: OSI Approved :: GNU Affero General Public License v3', diff --git a/tools/update_misp_modules.sh b/tools/update_misp_modules.sh new file mode 100755 index 0000000..e0578fd --- /dev/null +++ b/tools/update_misp_modules.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -e +set -x + +# Updates the MISP Modules while respecting the current permissions +# It aims to support the two following installation methods: +# * Everything is runinng on the same machine following the MISP installation guide. +# * The modules are installed using pipenv on a different machine from the one where MISP is running. + +if [ -d "/var/www/MISP" ] && [ -d "/usr/local/src/misp-modules" ] +then + echo "MISP is installed on the same machine, following the recommanded install script. Using MISP virtualenv." + PATH_TO_MISP="/var/www/MISP" + PATH_TO_MISP_MODULES="/usr/local/src/misp-modules" + + pushd ${PATH_TO_MISP_MODULES} + USER=`stat -c "%U" .` + sudo -H -u ${USER} git pull + sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS + sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e . + + popd +else + if ! [ -x "$(command -v pipenv)" ]; then + echo 'Error: pipenv not available, unable to automatically update.' >&2 + exit 1 + fi + + echo "Standalone mode, use pipenv from the current directory." + git pull + pipenv install +fi + + From 75953c32a7c093401a88e0f05d2447c9f4dbbac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 26 Feb 2019 16:48:11 -0800 Subject: [PATCH 34/46] chr: Restart the modules after update --- tools/update_misp_modules.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/update_misp_modules.sh b/tools/update_misp_modules.sh index e0578fd..372d146 100755 --- a/tools/update_misp_modules.sh +++ b/tools/update_misp_modules.sh @@ -20,6 +20,8 @@ then sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e . + service misp-modules restart + popd else if ! [ -x "$(command -v pipenv)" ]; then From a937b7c85dae6a22f7f3d5373e628e9f36aae384 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Wed, 27 Feb 2019 12:45:22 +0100 Subject: [PATCH 35/46] fix: [reportlab] Textual description parameter --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 6b0c12f..c143b5e 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description"] mispattributes = {} From a2716bc05d0c94a4c4d147591c2cb74a1a685882 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 1 Mar 2019 09:11:34 +0100 Subject: [PATCH 36/46] fix: [exportpdf] add configmodule parameter for galaxy --- misp_modules/modules/export_mod/pdfexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index c143b5e..096992b 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,8 +15,8 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description"] - +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", + "Activate_galaxy_description"] mispattributes = {} outputFileExtension = "pdf" From aef8dbbe2eca411757bc3e56c6dd4040de0daec5 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 1 Mar 2019 09:17:38 +0100 Subject: [PATCH 37/46] fix: [exportpdf] problem on one line --- misp_modules/modules/export_mod/pdfexport.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 096992b..d4e3be5 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,8 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", - "Activate_galaxy_description"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description","Activate_galaxy_description"] mispattributes = {} outputFileExtension = "pdf" From 7d7c90143ef062cbb97e4f2baf88ea5e32632033 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 1 Mar 2019 09:25:02 +0100 Subject: [PATCH 38/46] fix: [exportpdf] mising whitespace --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index d4e3be5..7402e27 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description","Activate_galaxy_description"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description"] mispattributes = {} outputFileExtension = "pdf" From 3b415cb53a2eff78dc1d3d56264764a81de6c3a9 Mon Sep 17 00:00:00 2001 From: cgi1 Date: Fri, 1 Mar 2019 12:13:27 +0100 Subject: [PATCH 39/46] Adding virtualenv to apt-get install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 501e54f..6ef4bf4 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ## How to install and start MISP modules in a Python virtualenv? ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git From a30bcc5dd20f44c19d867e5cb9328113ad8ea980 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Mon, 4 Mar 2019 12:36:18 +0100 Subject: [PATCH 40/46] fix: [exportpdf] add parameters --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 7402e27..d12dece 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts"] mispattributes = {} outputFileExtension = "pdf" From e3ddbe66a62830f69a47c228209dcef6f3a6c14e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 4 Mar 2019 23:08:58 +0100 Subject: [PATCH 41/46] chg: [doc] asciidoctor requirement removed (new PDF module use reportlab) --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index 6ef4bf4..951de64 100644 --- a/README.md +++ b/README.md @@ -94,8 +94,6 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS sudo -u www-data /var/www/MISP/venv/bin/pip install . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -109,8 +107,6 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo pip3 install -I -r REQUIREMENTS sudo pip3 install -I . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -125,7 +121,6 @@ cd misp-modules scl enable rh-python36 ‘python3 –m pip install cryptography’ scl enable rh-python36 ‘python3 –m pip install -I -r REQUIREMENTS’ scl enable rh-python36 ‘python3 –m pip install –I .’ -scl enable rh-ruby22 ‘gem install asciidoctor-pdf –pre’ ~~~~ Create the service file /etc/systemd/system/misp-workers.service : ~~~~ From 32e10ee27333410a1ed4d2c4afe10d8c3b7456be Mon Sep 17 00:00:00 2001 From: Falconieri Date: Tue, 5 Mar 2019 10:39:07 +0100 Subject: [PATCH 42/46] fix: [exportpdf] custom path parameter --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index d12dece..44b3bc9 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -15,7 +15,7 @@ moduleinfo = {'version': '2', 'require_standard_format': True} # config fields that your code expects from the site admin -moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts"] +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"] mispattributes = {} outputFileExtension = "pdf" From 30c08708c62a773d24de1b36308cdb613d0f55a5 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Tue, 5 Mar 2019 12:11:44 +0100 Subject: [PATCH 43/46] fix: [exportpdf] update documentation --- doc/export_mod/pdfexport.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/export_mod/pdfexport.json b/doc/export_mod/pdfexport.json index 9803c77..f1654dc 100644 --- a/doc/export_mod/pdfexport.json +++ b/doc/export_mod/pdfexport.json @@ -1,7 +1,7 @@ { "description": "Simple export of a MISP event to PDF.", - "requirements": ["PyMISP", "asciidoctor"], - "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event.", + "requirements": ["PyMISP", "reportlab"], + "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option ", "references": ["https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html"], "input": "MISP Event", "output": "MISP Event in a PDF file." From 9611c7f2a9a1bd84081ca3bc625f731d8185c842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Mar 2019 06:15:16 +0100 Subject: [PATCH 44/46] chg: Bump Requirements --- Pipfile.lock | 74 ++++++++++++++++++++++++++-------------------------- REQUIREMENTS | 14 +++++----- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 85cd9db..36d7c3c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -59,10 +59,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "beautifulsoup4": { "hashes": [ @@ -177,10 +177,10 @@ }, "jsonschema": { "hashes": [ - "sha256:acc8a90c31d11060516cfd0b414b9f8bcf4bc691b21f0f786ea57dd5255c79db", - "sha256:dd3f8ecb1b52d94d45eedb67cb86cac57b94ded562c5d98f63719e55ce58557b" + "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", + "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" ], - "version": "==3.0.0" + "version": "==3.0.1" }, "maclookup": { "hashes": [ @@ -281,17 +281,17 @@ }, "psutil": { "hashes": [ - "sha256:5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a", - "sha256:72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810", - "sha256:77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c", - "sha256:8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d", - "sha256:a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191", - "sha256:ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f", - "sha256:c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a", - "sha256:ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59", - "sha256:ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd" + "sha256:1020a37214c4138e34962881372b40f390582b5c8245680c04349c2afb785a25", + "sha256:151c9858c268a1523e16fab33e3bc3bae8a0e57b57cf7fcad85fb409cbac6baf", + "sha256:1c8e6444ca1cee9a60a1a35913b8409722f7474616e0e21004e4ffadba59964b", + "sha256:722dc0dcce5272f3c5c41609fdc2c8f0ee3f976550c2d2f2057e26ba760be9c0", + "sha256:86f61a1438c026c980a4c3e2dd88a5774a3a0f00d6d0954d6c5cf8d1921b804e", + "sha256:c4a2f42abee709ed97b4498c21aa608ac31fc1f7cc8aa60ebdcd3c80757a038d", + "sha256:d9cdc2e82aeb82200fff3640f375fac39d88b1bed27ce08377cd7fb0e3621cb7", + "sha256:da6676a484adec2fdd3e1ce1b70799881ffcb958e40208dd4c5beba0011f3589", + "sha256:dca71c08335fbfc6929438fe3a502f169ba96dd20e50b3544053d6be5cb19d82" ], - "version": "==5.5.1" + "version": "==5.6.0" }, "pybgpranking": { "editable": true, @@ -333,7 +333,7 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "62e047f3c1972e21aa36a8882bebf4488cdc1f84" + "ref": "b8759673b91e733c307698abdc0d5ed82fd7e0de" }, "pyonyphe": { "editable": true, @@ -469,10 +469,10 @@ }, "sigmatools": { "hashes": [ - "sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b" + "sha256:3bdbd2ee99c32f245e948d6b882219729ab379685dd7366e4d6149c390e08170" ], "index": "pypi", - "version": "==0.7.1" + "version": "==0.9" }, "six": { "hashes": [ @@ -506,15 +506,15 @@ }, "tornado": { "hashes": [ - "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", - "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", - "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", - "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", - "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", - "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", - "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" + "sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8", + "sha256:33f93243cd46dd398e5d2bbdd75539564d1f13f25d704cfc7541db74066d6695", + "sha256:34e59401afcecf0381a28228daad8ed3275bcb726810654612d5e9c001f421b7", + "sha256:35817031611d2c296c69e5023ea1f9b5720be803e3bb119464bb2a0405d5cd70", + "sha256:666b335cef5cc2759c21b7394cff881f71559aaf7cb8c4458af5bb6cb7275b47", + "sha256:81203efb26debaaef7158187af45bc440796de9fb1df12a75b65fae11600a255", + "sha256:de274c65f45f6656c375cdf1759dbf0bc52902a1e999d12a35eb13020a641a53" ], - "version": "==5.1.1" + "version": "==6.0.1" }, "url-normalize": { "hashes": [ @@ -545,12 +545,12 @@ }, "vulners": { "hashes": [ - "sha256:40041bcf893fa1bfaf29c650369d9a249991911f28b4d8795f7bc06508013e14", - "sha256:6d00709300dcc7e2727499d8a60f51eaced1dc6b63cc19cb8a4b065b658c51aa", - "sha256:de8cef247c9852c39bd54434e63026b46bdb2bd4ca22813bf66626b7d359b0f3" + "sha256:08a7ccb2b210d45143354c6161c73fe209dc14fae8692e8b793b36b79330ad11", + "sha256:bfe2478cc11c69ba7e436d7a5df925e227565782c0bd603929fb3d612c73d78d", + "sha256:d035f6a883625878a1dc377830d17d9702ef138ca31569ac01cb8686874f89cd" ], "index": "pypi", - "version": "==1.4.4" + "version": "==1.4.5" }, "wand": { "hashes": [ @@ -611,10 +611,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "certifi": { "hashes": [ @@ -743,10 +743,10 @@ }, "pyflakes": { "hashes": [ - "sha256:5e8c00e30c464c99e0b501dc160b13a14af7f27d4dffb529c556e30a159e231d", - "sha256:f277f9ca3e55de669fba45b7393a1449009cff5a37d1af10ebb76c52765269cd" + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], - "version": "==2.1.0" + "version": "==2.1.1" }, "pytest": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index 4709747..d672411 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,14 +3,14 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@62e047f3c1972e21aa36a8882bebf4488cdc1f84#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@b8759673b91e733c307698abdc0d5ed82fd7e0de#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 -attrs==18.2.0 +attrs==19.1.0 beautifulsoup4==4.7.1 blockchain==1.4.4 certifi==2018.11.29 @@ -27,13 +27,13 @@ httplib2==0.12.1 idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 isodate==0.6.0 -jsonschema==3.0.0 +jsonschema==3.0.1 maclookup==1.0.3 multidict==4.5.2 oauth2==1.9.0.post1 passivetotal==1.0.30 pillow==5.4.1 -psutil==5.5.1 +psutil==5.6.0 pyeupi==1.0 pygeoip==0.3.2 pyparsing==2.3.1 @@ -49,16 +49,16 @@ reportlab==3.5.13 requests-cache==0.4.13 requests==2.21.0 shodan==1.11.1 -sigmatools==0.7.1 +sigmatools==0.9 six==1.12.0 soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 -tornado==5.1.1 +tornado==6.0.1 url-normalize==1.4.1 urlarchiver==0.2 urllib3==1.24.1 -vulners==1.4.4 +vulners==1.4.5 wand==0.5.1 xlsxwriter==1.1.5 yara-python==3.8.1 From c4ced9dfbf49ec605f230b35564e17a94bcea6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sat, 9 Mar 2019 06:40:23 +0100 Subject: [PATCH 45/46] fix: Tornado expects a KILL now. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b574d4c..e8fea8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,14 +19,14 @@ script: - pid=$! - sleep 5 - pipenv run nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - kill -s KILL $pid - pushd ~/ - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & - pid=$! - popd - sleep 5 - pipenv run nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - kill -s KILL $pid - pipenv run flake8 --ignore=E501,W503 misp_modules after_success: From 4b77cb5055e0a114c5bf361cadefc4aacbe0ae39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 10 Mar 2019 21:17:30 +0100 Subject: [PATCH 46/46] new: Add missing dependency (backscatter) --- Pipfile | 1 + Pipfile.lock | 92 ++++++++++++++++++++++++++++++---------------------- REQUIREMENTS | 4 ++- 3 files changed, 57 insertions(+), 40 deletions(-) diff --git a/Pipfile b/Pipfile index 45c05f5..2f2d172 100644 --- a/Pipfile +++ b/Pipfile @@ -41,6 +41,7 @@ domaintools_api = "*" misp-modules = {editable = true,path = "."} pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} +backscatter = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 36d7c3c..3c902e7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d0cd64bfe7702365d3ea66d1f51a1ec8592df2490899e7e163fe38f97172561e" + "sha256": "23dec0fa6400c828e294ea9981b433903c17358ca61d7abdaec8df5a1c89f08c" }, "pipfile-spec": 6, "requires": { @@ -64,6 +64,14 @@ ], "version": "==19.1.0" }, + "backscatter": { + "hashes": [ + "sha256:7a0d1aa3661635de81e2a09b15d53e35cbe399a111cc58a70925f80e6874abd3", + "sha256:afb0efcf5d2551dac953ec4c38fb710b274b8e811775650e02c1ef42cafb14c8" + ], + "index": "pypi", + "version": "==0.2.4" + }, "beautifulsoup4": { "hashes": [ "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", @@ -82,10 +90,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -504,6 +512,12 @@ "index": "pypi", "version": "==1.1.0" }, + "tabulate": { + "hashes": [ + "sha256:8af07a39377cee1103a5c8b3330a421c2d99b9141e9cc5ddd2e3263fea416943" + ], + "version": "==0.8.3" + }, "tornado": { "hashes": [ "sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8", @@ -618,10 +632,10 @@ }, "certifi": { "hashes": [ - "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", - "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", + "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" ], - "version": "==2018.11.29" + "version": "==2019.3.9" }, "chardet": { "hashes": [ @@ -640,39 +654,39 @@ }, "coverage": { "hashes": [ - "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", - "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", - "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", - "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", - "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", - "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", - "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", - "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", - "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", - "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", - "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", - "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", - "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", - "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", - "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", - "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", - "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", - "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", - "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", - "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", - "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", - "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", - "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", - "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", - "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", - "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", - "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", - "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", - "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", - "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", - "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" ], - "version": "==4.5.2" + "version": "==4.5.3" }, "entrypoints": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index d672411..99e1c02 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -11,9 +11,10 @@ aiohttp==3.4.4 antlr4-python3-runtime==4.7.2 ; python_version >= '3' async-timeout==3.0.1 attrs==19.1.0 +backscatter==0.2.4 beautifulsoup4==4.7.1 blockchain==1.4.4 -certifi==2018.11.29 +certifi==2019.3.9 chardet==3.0.4 click-plugins==1.0.4 click==7.0 @@ -54,6 +55,7 @@ six==1.12.0 soupsieve==1.8 sparqlwrapper==1.8.2 stix2-patterns==1.1.0 +tabulate==0.8.3 tornado==6.0.1 url-normalize==1.4.1 urlarchiver==0.2