From d24a6e2e249f2d36a92634197c4e4c21a1b3b9b6 Mon Sep 17 00:00:00 2001 From: iceone23 Date: Mon, 15 Apr 2019 06:17:27 -0700 Subject: [PATCH 01/27] Create cisco_firesight_manager_ACL_rule_export.py Cisco Firesight Manager ACL Rule Export module --- ...cisco_firesight_manager_ACL_rule_export.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py diff --git a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py new file mode 100644 index 00000000..dcb61786 --- /dev/null +++ b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py @@ -0,0 +1,135 @@ +###################################################### +# # +# Author: Stanislav Klevtsov, Ukraine; Feb 2019. # +# # +# # +# Script was tested on the following configuration: # +# MISP v2.4.90 # +# Cisco Firesight Manager Console v6.2.3 (bld 84) # +# # +###################################################### + +import json +import base64 +import csv +from urllib.parse import quote + +misperrors = {'error': 'Error'} + +moduleinfo = {'version': '1', 'author': 'Stanislav Klevtsov', + 'description': 'Export malicious network activity attributes of the MISP event to Cisco firesight manager block rules', + 'module-type': ['export']} + + +moduleconfig = ['fmc_ip_addr', 'fmc_login', 'fmc_pass', 'domain_id', 'acpolicy_id'] + +fsmapping = {"ip-dst":"dst", "url":"request"} + +mispattributes = {'input':list(fsmapping.keys())} + +# options: event, attribute, event-collection, attribute-collection +inputSource = ['event'] + +outputFileExtension = 'sh' +responseType = 'application/txt' + +# .sh file templates +SH_FILE_HEADER = """#!/bin/sh\n\n""" + +BLOCK_JSON_TMPL = """ +BLOCK_RULE='{{ "action": "BLOCK", "enabled": true, "type": "AccessRule", "name": "{rule_name}", "destinationNetworks": {{ "literals": [ {dst_networks} ] }}, "urls": {{ "literals": [ {urls} ] }}, "newComments": [ "{event_info_comment}" ] }}'\n +""" + +BLOCK_DST_JSON_TMPL = """{{ "type": "Host", "value": "{ipdst}" }} """ +BLOCK_URL_JSON_TMPL = """{{ "type": "Url", "url": "{url}" }} """ + +CURL_ADD_RULE_TMPL = """ +curl -X POST -v -k -H 'Content-Type: application/json' -H \"Authorization: Basic $LOGINPASS_BASE64\" -H \"X-auth-access-token: $ACC_TOKEN\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_config/v1/domain/$DOMAIN_ID/policy/accesspolicies/$ACPOLICY_ID/accessrules\" --data \"$BLOCK_RULE\" """ + + +def handler(q=False): + if q is False: + return False + + r = {'results': []} + request = json.loads(q) + + if "config" in request: + config = request["config"] + + #check if config is empty + if not config['fmc_ip_addr']: config['fmc_ip_addr'] = "0.0.0.0" + if not config['fmc_login']: config['fmc_login'] = "login" + if not config['fmc_pass']: config['fmc_pass'] = "password" + if not config['domain_id']: config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID" + if not config['acpolicy_id']: config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID" + + data = request["data"] + output = "" + ipdst = [] + urls = [] + + #populate the ACL rule with attributes + for ev in data: + + event = ev["Attribute"] + event_id = ev["Event"]["id"] + event_info = ev["Event"]["info"] + + for index, attr in enumerate(event): + if attr["to_ids"] is True: + + if attr["type"] in fsmapping: + if attr["type"] in "ip-dst": + ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"])) + else: + urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*'))) + + + #building the .sh file + output += SH_FILE_HEADER + output += "FIRESIGHT_IP_ADDR='" + config['fmc_ip_addr'] + "'\n" + + output += "LOGINPASS_BASE64=`echo -n '" + config['fmc_login'] + ":" + config['fmc_pass'] + "' | base64`\n" + output += "DOMAIN_ID='" + config['domain_id'] + "'\n" + output += "ACPOLICY_ID='" + config['acpolicy_id'] + "'\n\n" + + output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`" + "\n" + + output += BLOCK_JSON_TMPL.format(rule_name="misp_event_"+event_id,dst_networks=', '.join(ipdst), urls=', '.join(urls), event_info_comment=event_info) + "\n" + + output += CURL_ADD_RULE_TMPL + # END building the .sh file + + r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')} + return r + + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From f5167c2f230c7dea0725bdb585c6ce8d61496942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 16 Apr 2019 11:25:39 +0200 Subject: [PATCH 02/27] fix: Make flake8 happy. --- ...cisco_firesight_manager_ACL_rule_export.py | 147 +++++++++--------- 1 file changed, 76 insertions(+), 71 deletions(-) diff --git a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py index dcb61786..ab79692d 100644 --- a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py +++ b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py @@ -11,21 +11,20 @@ import json import base64 -import csv from urllib.parse import quote misperrors = {'error': 'Error'} moduleinfo = {'version': '1', 'author': 'Stanislav Klevtsov', - 'description': 'Export malicious network activity attributes of the MISP event to Cisco firesight manager block rules', - 'module-type': ['export']} + 'description': 'Export malicious network activity attributes of the MISP event to Cisco firesight manager block rules', + 'module-type': ['export']} moduleconfig = ['fmc_ip_addr', 'fmc_login', 'fmc_pass', 'domain_id', 'acpolicy_id'] -fsmapping = {"ip-dst":"dst", "url":"request"} +fsmapping = {"ip-dst": "dst", "url": "request"} -mispattributes = {'input':list(fsmapping.keys())} +mispattributes = {'input': list(fsmapping.keys())} # options: event, attribute, event-collection, attribute-collection inputSource = ['event'] @@ -48,88 +47,94 @@ curl -X POST -v -k -H 'Content-Type: application/json' -H \"Authorization: Basic def handler(q=False): - if q is False: - return False - - r = {'results': []} - request = json.loads(q) + if q is False: + return False - if "config" in request: - config = request["config"] + r = {'results': []} + request = json.loads(q) - #check if config is empty - if not config['fmc_ip_addr']: config['fmc_ip_addr'] = "0.0.0.0" - if not config['fmc_login']: config['fmc_login'] = "login" - if not config['fmc_pass']: config['fmc_pass'] = "password" - if not config['domain_id']: config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID" - if not config['acpolicy_id']: config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID" + if "config" in request: + config = request["config"] - data = request["data"] - output = "" - ipdst = [] - urls = [] + # check if config is empty + if not config['fmc_ip_addr']: + config['fmc_ip_addr'] = "0.0.0.0" + if not config['fmc_login']: + config['fmc_login'] = "login" + if not config['fmc_pass']: + config['fmc_pass'] = "password" + if not config['domain_id']: + config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID" + if not config['acpolicy_id']: + config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID" - #populate the ACL rule with attributes - for ev in data: + data = request["data"] + output = "" + ipdst = [] + urls = [] - event = ev["Attribute"] - event_id = ev["Event"]["id"] - event_info = ev["Event"]["info"] + # populate the ACL rule with attributes + for ev in data: - for index, attr in enumerate(event): - if attr["to_ids"] is True: + event = ev["Attribute"] + event_id = ev["Event"]["id"] + event_info = ev["Event"]["info"] - if attr["type"] in fsmapping: - if attr["type"] in "ip-dst": - ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"])) - else: - urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*'))) + for index, attr in enumerate(event): + if attr["to_ids"] is True: + if attr["type"] in fsmapping: + if attr["type"] == "ip-dst": + ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"])) + else: + urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*'))) + # building the .sh file + output += SH_FILE_HEADER + output += "FIRESIGHT_IP_ADDR='{}'\n".format(config['fmc_ip_addr']) - #building the .sh file - output += SH_FILE_HEADER - output += "FIRESIGHT_IP_ADDR='" + config['fmc_ip_addr'] + "'\n" + output += "LOGINPASS_BASE64=`echo -n '{}:{}' | base64`\n".format(config['fmc_login'], config['fmc_pass']) + output += "DOMAIN_ID='{}'\n".format(config['domain_id']) + output += "ACPOLICY_ID='{}'\n\n".format(config['acpolicy_id']) - output += "LOGINPASS_BASE64=`echo -n '" + config['fmc_login'] + ":" + config['fmc_pass'] + "' | base64`\n" - output += "DOMAIN_ID='" + config['domain_id'] + "'\n" - output += "ACPOLICY_ID='" + config['acpolicy_id'] + "'\n\n" - - output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`" + "\n" + output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`\n" - output += BLOCK_JSON_TMPL.format(rule_name="misp_event_"+event_id,dst_networks=', '.join(ipdst), urls=', '.join(urls), event_info_comment=event_info) + "\n" + output += BLOCK_JSON_TMPL.format(rule_name="misp_event_{}".format(event_id), + dst_networks=', '.join(ipdst), + urls=', '.join(urls), + event_info_comment=event_info) + "\n" - output += CURL_ADD_RULE_TMPL - # END building the .sh file + output += CURL_ADD_RULE_TMPL + # END building the .sh file - r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')} - return r + r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')} + return r def introspection(): - modulesetup = {} - try: - responseType - modulesetup['responseType'] = responseType - except NameError: - pass - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo From 639534f152f9e94f529bc0c8a55847ddcd1d00f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 16 Apr 2019 11:25:53 +0200 Subject: [PATCH 03/27] chg: Bump Dependencies. --- Pipfile.lock | 106 +++++++++++++++++++++++++-------------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index b5faca17..428ae111 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -111,10 +111,10 @@ }, "click-plugins": { "hashes": [ - "sha256:b1ee1ccc9421c73007fe290680d97984eb6eaf5f4512b7620c6aa46031d6cb6b", - "sha256:dfed74b5063546a137de99baaaf742b4de4337ad2b3e1df5ec7c8a256adc0847" + "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", + "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" ], - "version": "==1.0.4" + "version": "==1.1.1" }, "colorama": { "hashes": [ @@ -300,7 +300,7 @@ "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "019ef1c40aad1e5bb5c5072c9a998c6a8f0271f3", + "ref": "86180c080ba5f83a5160a8295ebc263709aa4eba", "subdirectory": "client" }, "pydnstrails": { @@ -331,13 +331,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "0c4f11792061417b77ca6e22d2ece18109d74c75", + "ref": "1009124c39e9cc3b362f0f9ce4d272817b9a5186", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "64bcaad0e578129543cdffad532a232722615f6c" + "ref": "13445b51748524195bb15aec2c1c6176bcb6bd95" }, "pyonyphe": { "editable": true, @@ -346,10 +346,10 @@ }, "pyparsing": { "hashes": [ - "sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a", - "sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3" + "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", + "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" ], - "version": "==2.3.1" + "version": "==2.4.0" }, "pypdns": { "hashes": [ @@ -417,37 +417,37 @@ }, "reportlab": { "hashes": [ - "sha256:0135bc54a463db5315c93bba4182fb83dc088fefaa7da18784ecd2a0c4a9c068", - "sha256:09e167e01458ea1e0cf3acff634ae9ecc1f1757e7585060d039c90b762859cfd", - "sha256:0dfcea18ba3ca1fac55cb273d056a8a43a48bd04d419299b3267e1994c72455a", - "sha256:1a61e56593ea1a8a38135eedfb40f79dcad13164fff034313ebf2a30e200ca79", - "sha256:1bdd871c2087d3853a0e9a3a573b1a7535500f3341944b1e34e68f3213cd28b8", - "sha256:26878a4b9c45f046c635b5695681188c19806f08b04129ea01c9ed51c7754039", - "sha256:27c62264c758aa30113df105da816223d149e4e87ee778ad49469725b79be2eb", - "sha256:29a9dd3954465b9e4efb129ffda9ab3e6a4f06488e8aa2efd5aff8ad332f13c2", - "sha256:5740e3218ca98c1bc86bd2d2e2a8c1d23e7c97d949d6377ac30aaf449f01c363", - "sha256:605892bb3f822a1e7342ce2b461d645ab8e4d13875127c0ae5377f76853db422", - "sha256:6dacc72552bc0dd50286e856f09a5e646a007d9345598bf6f75b117a200bfd9d", - "sha256:7021b7c8ba6d8e69e4c68c9473067482aaa40b9094270b45dbf798fcb0e09bd4", - "sha256:8acd950dad5b20a417579d1253c1065222dde48f9412e71533b052ab3dd98632", - "sha256:8b8fb3b0dd1e2124aba24544a02c95bff1fffa966b0581f30abf4fb28e414005", - "sha256:920c61c942eb1cc446e1647a04978f4afe31993ed403b74576a018c3ca526394", - "sha256:928e8d99befe064e28e9a29a4fd9afcf2066dcd758b0903280e67e221527422a", - "sha256:a04787eee401a74c80b65e539b5fe9226fdeabe25caa3d216c21dc990b2f8a01", - "sha256:a5bb6bd7753cba854425fcf7ecf04627a17de78d47ef9e8fac615887c5658da3", - "sha256:a70d970619014dc83b4406bcfed7e2f9d5aaf5f521aad808f5560d90ea896fb4", - "sha256:ae468fe82c8af3d1987113f03c1f87d01daa5b4c85c1f10da126be84423a744d", - "sha256:b278d83a7f76410bd310b368309e6e4b19664ffa686abfa9f0696130b09c17d3", - "sha256:b6623e9a96db3edc4b384e036e67c7bc87bbd7e5dc2d72ce66efa0043f9383b0", - "sha256:dc15cfa577bb25f0a598d483cf6dcc5ecad576ba723fe9bec63b6ec720dab2a3", - "sha256:dffdb4f6b34ce791e67365f3f96ab3c45b4cdd2c70d212fac98fb146dc75ac80", - "sha256:e84020e3482856da733e1359cb7b84e6bac09179bd3af860e70468a9c3cb43e3", - "sha256:edda09668e8474d5acb1a37fb64599557b43a714f1469bd49a058e95b5b410ff", - "sha256:f77e9835873931d25f836a3c107e53e0f7d3c0b4906b13063815308cf5ca1fac", - "sha256:f91d16ff07d5d3c92303f64c6864d74d3b6a491dde186bfef90c58088f932998" + "sha256:1c228a3ac2c405f7fc16eac43ba92aec448bc25438902f30590ad021e8828097", + "sha256:2210fafd3bb06308a84876fe6d19172b645373edce2b6d7501378cb9c768f825", + "sha256:232fb2037b7c3df259685f1c5ecb7826f55742dc81f0713837b84a152307483e", + "sha256:2c4f25e63fa75f3064871cf435696a4e19b7bd4901d922b766ae58a447b5b6da", + "sha256:47951166d897b60e9e7ca349db82a2b689e6478ac6078e2c7c88ca8becbb0c7d", + "sha256:526ab1193ea8e97c4838135917890e66de5f777d04283008007229b139f3c094", + "sha256:5a9cc8470623ec5b76c7e59f56b7d1fcf0254896cd61842dbdbd278934cc50f4", + "sha256:5ddc1a4a74f225e35a7f60e2eae10de6878dddc9960dad2d9cadc49092f8850d", + "sha256:6b594f6d7d71bc5778e19adb1c699a598c69b9a7bcf97fa638d8762279f9d80a", + "sha256:6e8c89b46cfaf9ae40b7db87e9f29c9e5d32d18d25f9cd10d423a5241e8ec453", + "sha256:71f4f3e3975b91ddbfc1b36a537b46d07533ca7f31945e990a75db5f9bd7a0ba", + "sha256:763654dc346eeb66fa726a88d27f911339950d20a25303dfc098f3b59ba26614", + "sha256:7bae4b33363f44343e0fac5004c8e44576c3ed00885be4eee1f2260802c116c3", + "sha256:8a4b8a0fd0547f3b436b548284aa604ba183bfac26f41a7ffb23d0ff5db8c658", + "sha256:8b08d68e4cb498eabf85411beda5c32e591ef8d0a6d18c948c3f80ed5d2c6e31", + "sha256:9840f27948b54aefa3c6386e5ed0f124d641eb54fa2f2bc9aebcb270598487fc", + "sha256:9ae8f822370e47486ba1880f7580669058a41e64bdaa41019f4617317489f884", + "sha256:9db49197080646a113059eba1c0758161164de1bc57315e7422bbf8c86e03dcf", + "sha256:a08d23fa3f23f13a1cc6dca3b3c431d08ae48e52384e6bf47bbefb22fde58e61", + "sha256:ac111bc47733dbfa3e34d61282c91b69b1f66800b0c72b7b86dc2534faa09bef", + "sha256:bc3c69707c0bf9308193612d34ca87249d6fc91a35ce0873102321395d39024a", + "sha256:c375759a763c1c93d5b4f36620390440d9fa6dec6fcf88bce8234701d88b339c", + "sha256:c8a5988d73ec93a54f22660b64c5f3d2018163dd9ca4a5cdde8022a7e4fcb345", + "sha256:eba2bc7c28a3b2b0a3c24caff33e4d8708db008f480b03a6ea39c28661663746", + "sha256:ee187977d587b9b81929e08022f385eb11274efd75795d59d99eb23b3fa9b055", + "sha256:f3ef7616ffc27c150ffec61ac820739495f6a9ca5d8532047102756ebb27e8d1", + "sha256:f46f223fcae09c8bf2746b4eb2f351294faae04b262429cc480d34c69b133fd9", + "sha256:fd9f6429a68a246fb466696d97d1240752c889b5bfdc219fea15ae787cf366a6" ], "index": "pypi", - "version": "==3.5.17" + "version": "==3.5.19" }, "requests": { "hashes": [ @@ -466,10 +466,10 @@ }, "shodan": { "hashes": [ - "sha256:f93b7199e89eecf5c84647f66316c2c044c3aebfc1fe4d9caa43dfda07f74c4e" + "sha256:c30baebce853ad67677bf002dde96a1ca1a9729bdd300fbb3c5e5d889547a639" ], "index": "pypi", - "version": "==1.11.1" + "version": "==1.12.1" }, "sigmatools": { "hashes": [ @@ -487,10 +487,10 @@ }, "soupsieve": { "hashes": [ - "sha256:3aef141566afd07201b525c17bfaadd07580a8066f82b57f7c9417f26adbd0a3", - "sha256:e41a65e99bd125972d84221022beb1e4b5cfc68fa12c170c39834ce32d1b294c" + "sha256:6898e82ecb03772a0d82bd0d0a10c0d6dcc342f77e0701d0ec4a8271be465ece", + "sha256:b20eff5e564529711544066d7dc0f7661df41232ae263619dede5059799cdfca" ], - "version": "==1.9" + "version": "==1.9.1" }, "sparqlwrapper": { "hashes": [ @@ -555,12 +555,12 @@ }, "vulners": { "hashes": [ - "sha256:6617d5904b5369507bc34105071d312e9e1c38d73654505e7b15b9a3f1325915", - "sha256:8b05d12a9dd7cbc07198a13281299a6e014ec348522e214b1efd097e194b7568", - "sha256:a19b02e0a112d70951e10c5abc1993f7f029234212828e1b617ab35f4e460a24" + "sha256:6506f3ad45bf3fd72f9cd1ebd5fbc13f814e7cf62faed6897e33db949fe4584a", + "sha256:a0e86015343ecf1c3313f6101567749988b5eb5299a672f19fd2974121817444", + "sha256:f243fe025a84b85bd6f37e45d6cf693e24697d30661695fe5d29b652dff6a5a1" ], "index": "pypi", - "version": "==1.4.7" + "version": "==1.4.9" }, "wand": { "hashes": [ @@ -572,10 +572,10 @@ }, "xlsxwriter": { "hashes": [ - "sha256:de9ef46088489915eaaee00c7088cff93cf613e9990b46b933c98eb46f21b47f", - "sha256:df96eafc3136d9e790e35d6725b473e46ada6f585c1f6519da69b27f5c8873f7" + "sha256:3a4e4a24a6753f046dc5a5e5bc5f443fce6a18988486885a258db6963eb54163", + "sha256:92a2ba339ca939815f0e125fcde728e94ccdb3e97e1acd3275ecf25a3cacfdc6" ], - "version": "==1.1.5" + "version": "==1.1.6" }, "yara-python": { "hashes": [ @@ -760,11 +760,11 @@ }, "pytest": { "hashes": [ - "sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", - "sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a" + "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", + "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" ], "index": "pypi", - "version": "==4.4.0" + "version": "==4.4.1" }, "requests": { "hashes": [ From eefa35c65db9915af7e7a47a6ca4ba80ac29742a Mon Sep 17 00:00:00 2001 From: Evert0x <35029353+Evert0x@users.noreply.github.com> Date: Thu, 18 Apr 2019 00:23:38 +0200 Subject: [PATCH 04/27] Create cuckoo_submit.py --- .../modules/expansion/cuckoo_submit.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 misp_modules/modules/expansion/cuckoo_submit.py diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py new file mode 100644 index 00000000..d0cb2364 --- /dev/null +++ b/misp_modules/modules/expansion/cuckoo_submit.py @@ -0,0 +1,62 @@ +import json +import requests + +misperrors = {'error': 'Error'} +mispattributes = { + 'input': ['url'], + 'output': ['text'] +} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '1', 'author': 'Evert Kors', + 'description': 'MODULE_DESCRIPTION', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ['cuckoo_api'] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + config = request.get('config') + if config is None: + misperrors['error'] = 'config is missing' + return misperrors + + cuck = config.get('cuckoo_api') + if cuck is None: + misperrors['error'] = 'cuckoo api url is missing' + return misperrors + + # The url to submit + url = request.get('url') + + HEADERS = {"Authorization": "Bearer S4MPL3"} + + urls = [ + url + ] + + try: + r = requests.post( + "%s/tasks/create/submit" % (cuck), + headers=HEADERS, + data={"strings": "\n".join(urls)} + ) + except Exception as e: + misperrors['error'] = str(e) + return misperrors + + r = {'results': [{'types': "text", 'values': "cool"}]} + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From e243edb50341899a4feedbd90ec3bd4e5df5fd00 Mon Sep 17 00:00:00 2001 From: Evert0x <35029353+Evert0x@users.noreply.github.com> Date: Thu, 18 Apr 2019 14:25:05 +0200 Subject: [PATCH 05/27] Update __init__.py --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 42f74a3e..df83ca99 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,6 +1,6 @@ from . import _vmray # noqa -__all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', +__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', From 49acb5374538a4fe1bafbb4af44d422f8ca0287d Mon Sep 17 00:00:00 2001 From: Ricardo van Zutphen Date: Fri, 19 Apr 2019 14:06:35 +0200 Subject: [PATCH 06/27] Update Cuckoo module to support files and URLs --- .../modules/expansion/cuckoo_submit.py | 151 ++++++++++++++---- 1 file changed, 119 insertions(+), 32 deletions(-) diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py index d0cb2364..e368fbde 100644 --- a/misp_modules/modules/expansion/cuckoo_submit.py +++ b/misp_modules/modules/expansion/cuckoo_submit.py @@ -1,56 +1,143 @@ +import base64 +import io import json +import logging import requests +import sys +import urllib.parse +import zipfile -misperrors = {'error': 'Error'} +from requests.exceptions import RequestException + +log = logging.getLogger('cuckoo_submit') +log.setLevel(logging.DEBUG) +sh = logging.StreamHandler(sys.stdout) +sh.setLevel(logging.DEBUG) +fmt = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +sh.setFormatter(fmt) +log.addHandler(sh) + +moduleinfo = { + "version": "0.1", 'author': "Evert Kors", + "description": "Submit files and URLs to Cuckoo Sandbox", + "module-type": ["expansion", "hover"] +} +misperrors = {"error": "Error"} +moduleconfig = ["cuckoo_api", "api_key"] mispattributes = { - 'input': ['url'], - 'output': ['text'] + "input": ["attachment', 'malware-sample", "url", "domain"], + "output": ["text"] } -# possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '1', 'author': 'Evert Kors', - 'description': 'MODULE_DESCRIPTION', - 'module-type': ['expansion', 'hover']} -# config fields that your code expects from the site admin -moduleconfig = ['cuckoo_api'] +class APIKeyError(RequestException): + """Raised if the Cuckoo API returns a 401. This means no or an invalid + bearer token was supplied.""" + pass + + +class CuckooAPI(object): + + def __init__(self, api_url, api_key=""): + self.api_key = api_key + if not api_url.startswith("http"): + api_url = "https://{}".format(api_url) + + self.api_url = api_url + + def _post_api(self, endpoint, files=None, data={}): + data.update({ + "owner": "MISP" + }) + + try: + response = requests.post( + urllib.parse.urljoin(self.api_url, endpoint), + files=files, data=data, + headers={"Authorization: Bearer {}".format(self.api_key)} + ) + except RequestException as e: + log.error("Failed to submit sample to Cuckoo Sandbox. %s", e) + return None + + if response.status_code == 401: + raise APIKeyError("Invalid or no Cuckoo Sandbox API key provided") + + return response.json() + + def create_task(self, filename, fp): + response = self._post_api( + "/tasks/create/file", files={"file": (filename, fp)} + ) + if not response: + return False + + return response["task_id"] + + def create_url(self, url): + response = self._post_api( + "/tasks/create/url", data={"url": url} + ) + if not response: + return False + + return response["task_id"] def handler(q=False): if q is False: return False + request = json.loads(q) - config = request.get('config') - if config is None: - misperrors['error'] = 'config is missing' + + # See if the API URL was provided. The API key is optional, as it can + # be disabled in the Cuckoo API settings. + api_url = request["config"].get("api_url") + api_key = request["config"].get("api_key", "") + if not api_url: + misperrors["error"] = "No Cuckoo API URL provided" return misperrors - cuck = config.get('cuckoo_api') - if cuck is None: - misperrors['error'] = 'cuckoo api url is missing' - return misperrors + url = request.get("url") or request.get("domain") + data = request.get("data") + filename = None + if data: + data = base64.b64decode(data) - # The url to submit - url = request.get('url') + if "malware-sample" in request: + filename = request.get("malware-sample").split("|", 1)[0] + with zipfile.ZipFile(io.BytesIO(data)) as zipf: + data = zipf.read(zipf.namelist()[0], pwd=b"infected") - HEADERS = {"Authorization": "Bearer S4MPL3"} - - urls = [ - url - ] + elif "attachment" in request: + filename = request.get("attachment") + cuckoo_api = CuckooAPI(api_url=api_url, api_key=api_key) + task_id = None try: - r = requests.post( - "%s/tasks/create/submit" % (cuck), - headers=HEADERS, - data={"strings": "\n".join(urls)} - ) - except Exception as e: - misperrors['error'] = str(e) + if url: + log.debug("Submitting URL to Cuckoo Sandbox %s", api_url) + task_id = cuckoo_api.create_url(url) + elif data and filename: + log.debug("Submitting file to Cuckoo Sandbox %s", api_url) + task_id = cuckoo_api.create_task( + filename=filename, fp=io.BytesIO(data) + ) + except APIKeyError as e: + misperrors["error"] = "Failed to submit to Cuckoo: {}".format(e) return misperrors - r = {'results': [{'types': "text", 'values': "cool"}]} - return r + if not task_id: + misperrors["error"] = "File or URL submission failed" + return misperrors + + return { + "results": [ + {"types": "text", "values": "Cuckoo task id: {}".format(task_id)} + ] + } def introspection(): From e6326185d568575a200147a2866aa8cd9769ac28 Mon Sep 17 00:00:00 2001 From: Ricardo van Zutphen Date: Fri, 19 Apr 2019 16:24:30 +0200 Subject: [PATCH 07/27] Use double quotes and provide headers correctly --- .../modules/expansion/cuckoo_submit.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py index e368fbde..c1ded90c 100644 --- a/misp_modules/modules/expansion/cuckoo_submit.py +++ b/misp_modules/modules/expansion/cuckoo_submit.py @@ -9,25 +9,25 @@ import zipfile from requests.exceptions import RequestException -log = logging.getLogger('cuckoo_submit') +log = logging.getLogger("cuckoo_submit") log.setLevel(logging.DEBUG) sh = logging.StreamHandler(sys.stdout) sh.setLevel(logging.DEBUG) fmt = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) sh.setFormatter(fmt) log.addHandler(sh) moduleinfo = { - "version": "0.1", 'author': "Evert Kors", + "version": "0.1", "author": "Evert Kors", "description": "Submit files and URLs to Cuckoo Sandbox", "module-type": ["expansion", "hover"] } misperrors = {"error": "Error"} -moduleconfig = ["cuckoo_api", "api_key"] +moduleconfig = ["api_url", "api_key"] mispattributes = { - "input": ["attachment', 'malware-sample", "url", "domain"], + "input": ["attachment", "malware-sample", "url", "domain"], "output": ["text"] } @@ -56,7 +56,7 @@ class CuckooAPI(object): response = requests.post( urllib.parse.urljoin(self.api_url, endpoint), files=files, data=data, - headers={"Authorization: Bearer {}".format(self.api_key)} + headers={"Authorization": "Bearer {}".format(self.api_key)} ) except RequestException as e: log.error("Failed to submit sample to Cuckoo Sandbox. %s", e) @@ -65,6 +65,10 @@ class CuckooAPI(object): if response.status_code == 401: raise APIKeyError("Invalid or no Cuckoo Sandbox API key provided") + if response.status_code != 200: + log.error("Invalid Cuckoo API response") + return None + return response.json() def create_task(self, filename, fp): @@ -145,5 +149,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo From 7fefbd2a4c074c43b73d803f6c87491aa564996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 19 Apr 2019 22:41:06 +0200 Subject: [PATCH 08/27] chg: Bump dependencies Fix CVE-2019-11324 (urllib3) --- Pipfile.lock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 428ae111..4a5c092b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -300,7 +300,7 @@ "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "86180c080ba5f83a5160a8295ebc263709aa4eba", + "ref": "4e0741056bcc0077de1120b8724a31330b26033e", "subdirectory": "client" }, "pydnstrails": { @@ -331,13 +331,13 @@ "pyipasnhistory": { "editable": true, "git": "https://github.com/D4-project/IPASN-History.git/", - "ref": "1009124c39e9cc3b362f0f9ce4d272817b9a5186", + "ref": "c0c2bbf8d70811982dad065ea463a7e01593a38d", "subdirectory": "client" }, "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "13445b51748524195bb15aec2c1c6176bcb6bd95" + "ref": "921f414e0e026a3a4b77112012cf930242a33b04" }, "pyonyphe": { "editable": true, @@ -459,10 +459,10 @@ }, "requests-cache": { "hashes": [ - "sha256:e9270030becc739b0a7f7f834234c73a878b2d794122bf76f40055a22419eb67", - "sha256:fe561ca119879bbcfb51f03a35e35b425e18f338248e59fd5cf2166c77f457a2" + "sha256:6822f788c5ee248995c4bfbd725de2002ad710182ba26a666e85b64981866060", + "sha256:73a7211870f7d67af5fd81cad2f67cfe1cd3eb4ee6a85155e07613968cc72dfc" ], - "version": "==0.4.13" + "version": "==0.5.0" }, "shodan": { "hashes": [ @@ -494,12 +494,12 @@ }, "sparqlwrapper": { "hashes": [ - "sha256:2a95fdede2833be660b81092934c4a0054ff85f2693098556762a2759ea486f1", - "sha256:7f4c8d38ea1bfcffbc358c9a05de35a3fd7152cc3e8ea57963ee7a0a242f7a5e", - "sha256:acf6d60f0a3684cb673653b07871acb0c350a974b891f20f8ac94926ff9eb2ff" + "sha256:14ec551f0d60b4a496ffcc31f15337e844c085b8ead8cbe9a7178748a6de3794", + "sha256:21928e7a97f565e772cdeeb0abad428960f4307e3a13dbdd8f6d3da8a6a506c9", + "sha256:abc3e7eadcad32fa69a85c003853e2f6f73bda6cc999853838f401a5a1ea1109" ], "index": "pypi", - "version": "==1.8.2" + "version": "==1.8.4" }, "stix2-patterns": { "hashes": [ @@ -542,10 +542,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], - "version": "==1.24.1" + "version": "==1.24.2" }, "uwhois": { "editable": true, @@ -783,10 +783,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], - "version": "==1.24.1" + "version": "==1.24.2" } } } From 5367bcd409e55eff39ecc498aa28ceab39988f6d Mon Sep 17 00:00:00 2001 From: Ricardo van Zutphen Date: Mon, 22 Apr 2019 22:38:03 +0200 Subject: [PATCH 09/27] Document Cuckoo expansion module --- doc/expansion/cuckoo_submit.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/expansion/cuckoo_submit.json diff --git a/doc/expansion/cuckoo_submit.json b/doc/expansion/cuckoo_submit.json new file mode 100644 index 00000000..7fe80674 --- /dev/null +++ b/doc/expansion/cuckoo_submit.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion module to submit files and URLs to Cuckoo Sandbox.", + "logo": "logos/cuckoo.png", + "requirements": ["Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)"], + "input": "A malware-sample or attachment for files. A url or domain for URLs.", + "output": "A text field containing 'Cuckoo task id: '", + "references": ["https://cuckoosandbox.org/", "https://cuckoo.sh/docs/"], + "features": "The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox.\n The returned task id can be used to retrieve results when the analysis completed." +} From cafa1a6229649358ac04e2d1326bc07747628046 Mon Sep 17 00:00:00 2001 From: Ricardo van Zutphen Date: Mon, 22 Apr 2019 22:45:38 +0200 Subject: [PATCH 10/27] Generate latest version of documentation --- doc/README.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/doc/README.md b/doc/README.md index c32a8c4f..02506abf 100644 --- a/doc/README.md +++ b/doc/README.md @@ -178,6 +178,25 @@ Module to query Crowdstrike Falcon. ----- +#### [cuckoo_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) + + + +An expansion module to submit files and URLs to Cuckoo Sandbox. +- **features**: +>The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox. +> The returned task id can be used to retrieve results when the analysis completed. +- **input**: +>A malware-sample or attachment for files. A url or domain for URLs. +- **output**: +>A text field containing 'Cuckoo task id: ' +- **references**: +>https://cuckoosandbox.org/, https://cuckoo.sh/docs/ +- **requirements**: +>Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key) + +----- + #### [cve](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) @@ -1081,7 +1100,13 @@ OSQuery export of a MISP event. Simple export of a MISP event to PDF. - **features**: ->The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event. +>The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf. +> 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata +> 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event +> 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies. +> 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event ! +> 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation. +> 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option - **input**: >MISP Event - **output**: @@ -1089,7 +1114,7 @@ Simple export of a MISP event to PDF. - **references**: >https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html - **requirements**: ->PyMISP, asciidoctor +>PyMISP, reportlab ----- From c85ab8d93ca9485e5534cd779cc91e5ba8c9d2c8 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Tue, 23 Apr 2019 11:38:56 +0200 Subject: [PATCH 11/27] initial version of QR code reader Module accepts attachments and processes pictures. It tries to identify and analyze an existing QR code. Identified values can be inserted into the event. --- misp_modules/modules/expansion/qrcode.py | 86 ++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 misp_modules/modules/expansion/qrcode.py diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py new file mode 100644 index 00000000..5357e6dd --- /dev/null +++ b/misp_modules/modules/expansion/qrcode.py @@ -0,0 +1,86 @@ +import json +from pyzbar import pyzbar +import cv2 +import re +import binascii +import np + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['url', 'btc']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'QR code decoder', + 'module-type': ['expansion', 'hover']} + +debug = True +debug_prefix = "[DEBUG] QR Code module: " +# format example: bitcoin:1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh?amount=0.15424 +# format example: http://example.com +cryptocurrencies = {'bitcoin'} +schemas = {'http://', 'https://', 'ftp://'} +moduleconfig = [] + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + img_array = np.fromstring(binascii.a2b_base64(q['data']), np.uint8) + except: + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) + if q: + barcodes = pyzbar.decode(image) + for item in barcodes: + try: + result = item.data.decode() + except Exception as e: + print(e) + return + if debug: + print(debug_prefix + result) + for item in cryptocurrencies: + if item in result: + try: + currency, address, extra = re.split('\:|\?', result) + except Exception as e: + print(e) + if currency in cryptocurrencies: + try: + amount = re.split('=', extra)[1] + if debug: + print(debug_prefix + address) + print(debug_prefix + amount) + return {'results': [{'types': ['btc'], 'values': address, 'comment': "BTC: " + amount + " from file " + filename}]} + except Exception as e: + print(e) + else: + print(address) + for item in schemas: + if item in result: + try: + url = result + if debug: + print(debug_prefix + url) + return {'results': [{'types': ['url'], 'values': url, 'comment': "from QR code of file " + filename}]} + except Exception as e: + print(e) + else: + try: + return {'results': [{'types': ['text'], 'values': result, 'comment': "from QR code of file " + filename}]} + except Exception as e: + print(e) + misperrors['error'] = "Couldn't decode QR code in attachment." + return misperrors + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From d5180e7e79584cf256ad88db1422f3b155445122 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:37:27 +0200 Subject: [PATCH 12/27] chg: [qrcode] various fixes to make it PEP compliant --- misp_modules/modules/expansion/qrcode.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py index 5357e6dd..82f4f885 100644 --- a/misp_modules/modules/expansion/qrcode.py +++ b/misp_modules/modules/expansion/qrcode.py @@ -20,6 +20,7 @@ cryptocurrencies = {'bitcoin'} schemas = {'http://', 'https://', 'ftp://'} moduleconfig = [] + def handler(q=False): if q is False: return False @@ -27,7 +28,7 @@ def handler(q=False): filename = q['attachment'] try: img_array = np.fromstring(binascii.a2b_base64(q['data']), np.uint8) - except: + except Exception as e: err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" misperrors['error'] = err print(err) @@ -46,7 +47,7 @@ def handler(q=False): for item in cryptocurrencies: if item in result: try: - currency, address, extra = re.split('\:|\?', result) + currency, address, extra = re.split('\:|\?', result) except Exception as e: print(e) if currency in cryptocurrencies: @@ -60,7 +61,7 @@ def handler(q=False): print(e) else: print(address) - for item in schemas: + for item in schemas: if item in result: try: url = result @@ -77,6 +78,7 @@ def handler(q=False): misperrors['error'] = "Couldn't decode QR code in attachment." return misperrors + def introspection(): return mispattributes From 44050ec4da5b830161b2f9397e7910309cf1bdd5 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:44:00 +0200 Subject: [PATCH 13/27] chg: [qrcode] flake8 needs some drugs --- misp_modules/modules/expansion/qrcode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py index 82f4f885..437de605 100644 --- a/misp_modules/modules/expansion/qrcode.py +++ b/misp_modules/modules/expansion/qrcode.py @@ -32,6 +32,7 @@ def handler(q=False): err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" misperrors['error'] = err print(err) + print(e) return misperrors image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) if q: @@ -47,7 +48,7 @@ def handler(q=False): for item in cryptocurrencies: if item in result: try: - currency, address, extra = re.split('\:|\?', result) + currency, address, extra = re.split(r'\:|\?', result) except Exception as e: print(e) if currency in cryptocurrencies: From e55ae11a1e126ad040eef1e17ae6836c770f2629 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:45:12 +0200 Subject: [PATCH 14/27] chg: [qrcode] added to the __init__ --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 42f74a3e..ff847a46 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,4 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors'] + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', 'qrcode'] From 32430a15cb1794bb41596e5a4b80a2f2fce77109 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:49:02 +0200 Subject: [PATCH 15/27] chg: [qrcode] add requirements --- Pipfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Pipfile b/Pipfile index 2f2d1727..6a88fcca 100644 --- a/Pipfile +++ b/Pipfile @@ -42,6 +42,9 @@ misp-modules = {editable = true,path = "."} pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} backscatter = "*" +pyzbar = "*" +opencv-python = "*" +np = "*" [requires] python_version = "3.6" From 5adb9bfcfab220f5db5af4f0ec89a9611984db8b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:54:05 +0200 Subject: [PATCH 16/27] chg: [doc] qrcode and Cisco FireSight added --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 18b55481..94a57788 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [qrcode](misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values. * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. * [securitytrails](misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/). @@ -68,6 +69,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Export modules * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). +* [Cisco FireSight Manager ACL rule](misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL. * [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). * [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. * [PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. From 8acbb1762d07669d41ab9215f563507535a288c1 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 14:59:42 +0200 Subject: [PATCH 17/27] chg: [travis] because everyone need a bar --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e8fea8ea..b0d73edb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ python: - "3.7-dev" install: + - apt-get install libzbar0 libzbar-dev - pip install pipenv - pipenv install --dev From 72cd5e3c1f0ccfae85e30f3644cef578d1239f06 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 15:02:32 +0200 Subject: [PATCH 18/27] chg: [travis] because we all need sudo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b0d73edb..52a77420 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ python: - "3.7-dev" install: - - apt-get install libzbar0 libzbar-dev + - sudo apt-get install libzbar0 libzbar-dev - pip install pipenv - pipenv install --dev From 2d8aaf09c20849b1ce6f251dfe07417341b0d29a Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Tue, 23 Apr 2019 15:40:22 +0200 Subject: [PATCH 19/27] brackets are difficult... --- misp_modules/modules/expansion/qrcode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py index 437de605..9a628270 100644 --- a/misp_modules/modules/expansion/qrcode.py +++ b/misp_modules/modules/expansion/qrcode.py @@ -16,8 +16,8 @@ debug = True debug_prefix = "[DEBUG] QR Code module: " # format example: bitcoin:1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh?amount=0.15424 # format example: http://example.com -cryptocurrencies = {'bitcoin'} -schemas = {'http://', 'https://', 'ftp://'} +cryptocurrencies = ['bitcoin'] +schemas = ['http://', 'https://', 'ftp://'] moduleconfig = [] From b787aa7961976f560c638db9eff3223a6be07d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Tue, 23 Apr 2019 17:02:21 +0200 Subject: [PATCH 20/27] chg: Require python3 instead of python 3.6 --- Pipfile | 2 +- Pipfile.lock | 99 ++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/Pipfile b/Pipfile index 6a88fcca..ba650fec 100644 --- a/Pipfile +++ b/Pipfile @@ -47,4 +47,4 @@ opencv-python = "*" np = "*" [requires] -python_version = "3.6" +python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 4a5c092b..3191d4bd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "23dec0fa6400c828e294ea9981b433903c17358ca61d7abdaec8df5a1c89f08c" + "sha256": "7fee9399d8a7151a79b6f8bbce64564062fd562b0a091fd45a875884d3fb954e" }, "pipfile-spec": 6, "requires": { - "python_version": "3.6" + "python_version": "3" }, "sources": [ { @@ -158,9 +158,10 @@ }, "httplib2": { "hashes": [ - "sha256:4ba6b8fd77d0038769bf3c33c9a96a6f752bc4cdf739701fdcaf210121f399d4" + "sha256:23914b5487dfe8ef09db6656d6d63afb0cf3054ad9ebc50868ddc8e166b5f8e8", + "sha256:a18121c7c72a56689efbf1aef990139ad940fee1e64c6f2458831736cd593600" ], - "version": "==0.12.1" + "version": "==0.12.3" }, "idna": { "hashes": [ @@ -236,6 +237,41 @@ ], "version": "==4.5.2" }, + "np": { + "hashes": [ + "sha256:781265283f3823663ad8fb48741aae62abcf4c78bc19f908f8aa7c1d3eb132f8" + ], + "index": "pypi", + "version": "==1.0.2" + }, + "numpy": { + "hashes": [ + "sha256:0e2eed77804b2a6a88741f8fcac02c5499bba3953ec9c71e8b217fad4912c56c", + "sha256:1c666f04553ef70fda54adf097dbae7080645435fc273e2397f26bbf1d127bbb", + "sha256:1f46532afa7b2903bfb1b79becca2954c0a04389d19e03dc73f06b039048ac40", + "sha256:315fa1b1dfc16ae0f03f8fd1c55f23fd15368710f641d570236f3d78af55e340", + "sha256:3d5fcea4f5ed40c3280791d54da3ad2ecf896f4c87c877b113576b8280c59441", + "sha256:48241759b99d60aba63b0e590332c600fc4b46ad597c9b0a53f350b871ef0634", + "sha256:4b4f2924b36d857cf302aec369caac61e43500c17eeef0d7baacad1084c0ee84", + "sha256:54fe3b7ed9e7eb928bbc4318f954d133851865f062fa4bbb02ef8940bc67b5d2", + "sha256:5a8f021c70e6206c317974c93eaaf9bc2b56295b6b1cacccf88846e44a1f33fc", + "sha256:754a6be26d938e6ca91942804eb209307b73f806a1721176278a6038869a1686", + "sha256:771147e654e8b95eea1293174a94f34e2e77d5729ad44aefb62fbf8a79747a15", + "sha256:78a6f89da87eeb48014ec652a65c4ffde370c036d780a995edaeb121d3625621", + "sha256:7fde5c2a3a682a9e101e61d97696687ebdba47637611378b4127fe7e47fdf2bf", + "sha256:80d99399c97f646e873dd8ce87c38cfdbb668956bbc39bc1e6cac4b515bba2a0", + "sha256:88a72c1e45a0ae24d1f249a529d9f71fe82e6fa6a3fd61414b829396ec585900", + "sha256:a4f4460877a16ac73302a9c077ca545498d9fe64e6a81398d8e1a67e4695e3df", + "sha256:a61255a765b3ac73ee4b110b28fccfbf758c985677f526c2b4b39c48cc4b509d", + "sha256:ab4896a8c910b9a04c0142871d8800c76c8a2e5ff44763513e1dd9d9631ce897", + "sha256:abbd6b1c2ef6199f4b7ca9f818eb6b31f17b73a6110aadc4e4298c3f00fab24e", + "sha256:b16d88da290334e33ea992c56492326ea3b06233a00a1855414360b77ca72f26", + "sha256:b78a1defedb0e8f6ae1eb55fa6ac74ab42acc4569c3a2eacc2a407ee5d42ebcb", + "sha256:cfef82c43b8b29ca436560d51b2251d5117818a8d1fb74a8384a83c096745dad", + "sha256:d160e57731fcdec2beda807ebcabf39823c47e9409485b5a3a1db3a8c6ce763e" + ], + "version": "==1.16.3" + }, "oauth2": { "hashes": [ "sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6", @@ -244,6 +280,39 @@ "index": "pypi", "version": "==1.9.0.post1" }, + "opencv-python": { + "hashes": [ + "sha256:1703a296a96d3d46615e5053f224867977accb4240bcaa0fcabcb0768bf5ac13", + "sha256:1777ce7535ee7a1995cae168a107a1320e9df13648b930e72a1a2c2eccd64cda", + "sha256:1e5520482fb18fbd64d079e7f17ac0018f195fd75f6360a53bb82d7903106b50", + "sha256:25522dcf2529614750a71112a6659759080b4bdc2323f19d47f4d895960fd796", + "sha256:2af5f2842ad44c65ae2647377e0ff198719e1a1cfc9c6a19bc0c525c035d4bd8", + "sha256:31ec48d7eca13fc25c287dea7cecab453976e372cad8f50d55c054a247efda21", + "sha256:47cf48ff5dbd554e9f58cc9e98cf0b5de3f6a971172612bffa06bc5fb79ce872", + "sha256:494f98366bb5d6c2ac7e50e6617139f353704fd97a6d12ec9d392e72817d5cb0", + "sha256:4a9845870739e640e3350a8d98d511c92c087fe3d66090e83be7bf94e0ac64f7", + "sha256:4ac29cc0847d948a6636899014e84e165c30cc8779d6218394d44363462a01ce", + "sha256:5857ace03b7854221abf8072462d306c2c2ce4e366190b21d90ee8ee8aaf5bb4", + "sha256:5b4a23d99d5a2874767034466f5a8fd37b9f93ac14955a01b1a208983c76b9ad", + "sha256:734d87a5021c037064beb62133e135e66c7128e401a63b8b842b809ae2093749", + "sha256:78005c1c5d15ef4e32e0f485557bd15b5b6d87f49c19db7fe3e9246a61ebe7e4", + "sha256:81ae2283225c5c52fc3d72debd4241c30ccff2bb922578bf7867f9851cce3acb", + "sha256:88dbf900f297fdae0f62b899d6a784d8868ec2135854c5f8a9abbad00a6f0c5b", + "sha256:8c98ea7b8d327a31cd6028782a06147d0e0329ae8e829e881fb5d02f7ed8aec9", + "sha256:937d4686fef6967921145290f5b50c01c00c5b5d3542a6519e8a85cd88448723", + "sha256:a057958c0e362b3c4f03b9af1cbdb6d5af035fd22ecd7fd794eba8fdeb049eb8", + "sha256:c41eab31fa2c641226c6187caa391a688d064c99f078d604574f1912296b771f", + "sha256:cf4f7e62d1f80d1fa85a1693a3500def5cde54b2b75212b3609e552e4c25acfb", + "sha256:d90d60143e18334330c149f293071c9f2f3c79c896f33dc4ec65099e58baaaa7", + "sha256:db3106b7ca86999a7bd1f2fcc93e49314e5e6e451356774e421a69428df5020b", + "sha256:dbaf264db56f4771dfac6624f438bc4dc670aa94f61a6138848fcab7e9e77380", + "sha256:e65206c4cf651dc9cf0829962fae8bec986767c9f123d6a1ad17f9356bf7257e", + "sha256:eac94ddc78c58e891cff7180274317dad2938a4ddfc6ced1c04846c7f50e77e9", + "sha256:f2e828711f044a965509c862b3a59b3181e9c56c145a950cb53d43fec54e66d2" + ], + "index": "pypi", + "version": "==4.1.0.25" + }, "passivetotal": { "hashes": [ "sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d" @@ -401,6 +470,15 @@ ], "version": "==5.1" }, + "pyzbar": { + "hashes": [ + "sha256:0e204b904e093e5e75aa85e0203bb0e02888105732a509b51f31cff400f34265", + "sha256:496249b546be70ec98c0ff0ad9151e73daaffff129266df86150a15dcd8dac4c", + "sha256:7d6c01d2c0a352fa994aa91b5540d1caeaeaac466656eb41468ca5df33be9f2e" + ], + "index": "pypi", + "version": "==0.1.8" + }, "rdflib": { "hashes": [ "sha256:58d5994610105a457cff7fdfe3d683d87786c5028a45ae032982498a7e913d6f", @@ -564,18 +642,19 @@ }, "wand": { "hashes": [ - "sha256:91810d241ab0851d40e67c946beb960b869c4f4160c397eac291ec6283ee3e3f", - "sha256:ae7c0958509a22f531b7b97e93adfd3f1208f0ac1c593af9e5f0cffa4ac06d5b" + "sha256:63ab24dee0264a44f5f045d4ecc0d392bc1cc195e5a2f80ce537b2c205c3033b", + "sha256:a2c318993791fab4fcfd460045415176f81d42f8c6fd8a88fb8d74d2f0f34b97", + "sha256:f68f32f2e4eca663a361d36148f06372de560442dcf8c785a53a64ee282572c9" ], "index": "pypi", - "version": "==0.5.2" + "version": "==0.5.3" }, "xlsxwriter": { "hashes": [ - "sha256:3a4e4a24a6753f046dc5a5e5bc5f443fce6a18988486885a258db6963eb54163", - "sha256:92a2ba339ca939815f0e125fcde728e94ccdb3e97e1acd3275ecf25a3cacfdc6" + "sha256:2a40b427dac0f640031e5b33abe97e761de6e0f12d4d346e7b2e2b67cf6ee927", + "sha256:431edc9ba1132eec1996939aa83fffe41885d3042ab09d47c3086f41a156c430" ], - "version": "==1.1.6" + "version": "==1.1.7" }, "yara-python": { "hashes": [ From 4631c17286dacc52c71eda700a26a2403162b85b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Tue, 23 Apr 2019 19:49:58 +0200 Subject: [PATCH 21/27] chg: [doc] cuckoo_submit module added --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 94a57788..081120d9 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). +* [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. * [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. From e893a17583a36c84d8dd6deb112eaf8c41476eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 24 Apr 2019 11:29:16 +0200 Subject: [PATCH 22/27] chg: Bump dependencies, update REQUIREMENTS file --- Pipfile.lock | 8 ++++---- REQUIREMENTS | 47 +++++++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 3191d4bd..8a073a90 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -633,12 +633,12 @@ }, "vulners": { "hashes": [ - "sha256:6506f3ad45bf3fd72f9cd1ebd5fbc13f814e7cf62faed6897e33db949fe4584a", - "sha256:a0e86015343ecf1c3313f6101567749988b5eb5299a672f19fd2974121817444", - "sha256:f243fe025a84b85bd6f37e45d6cf693e24697d30661695fe5d29b652dff6a5a1" + "sha256:146ef130f215b50cdff790b06b4886c7edb325c075e9fce4bf1d3ab8d64a10d0", + "sha256:53406a86126159eaee9575fa667c99459bfdf9dd8c06bd0ce73fbe536b305e30", + "sha256:a258ccdbaee586207bc80d3590f0315ff151cfe16ea54f2e1629a6018fd9f2a3" ], "index": "pypi", - "version": "==1.4.9" + "version": "==1.5.0" }, "wand": { "hashes": [ diff --git a/REQUIREMENTS b/REQUIREMENTS index 99e1c028..b9d198a2 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,9 +1,9 @@ -i https://pypi.org/simple -e . --e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client --e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client +-e git+https://github.com/D4-project/BGP-Ranking.git/@4e0741056bcc0077de1120b8724a31330b26033e#egg=pybgpranking&subdirectory=client +-e git+https://github.com/D4-project/IPASN-History.git/@c0c2bbf8d70811982dad065ea463a7e01593a38d#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@b8759673b91e733c307698abdc0d5ed82fd7e0de#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@921f414e0e026a3a4b77112012cf930242a33b04#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe @@ -16,7 +16,7 @@ beautifulsoup4==4.7.1 blockchain==1.4.4 certifi==2019.3.9 chardet==3.0.4 -click-plugins==1.0.4 +click-plugins==1.1.1 click==7.0 colorama==0.4.1 dnspython==1.16.0 @@ -24,44 +24,47 @@ domaintools-api==0.3.3 enum-compat==0.0.2 ez-setup==0.9 future==0.17.1 -httplib2==0.12.1 -idna-ssl==1.1.0 ; python_version < '3.7' +httplib2==0.12.3 idna==2.8 isodate==0.6.0 jsonschema==3.0.1 maclookup==1.0.3 multidict==4.5.2 +np==1.0.2 +numpy==1.16.3 oauth2==1.9.0.post1 +opencv-python==4.1.0.25 passivetotal==1.0.30 -pillow==5.4.1 -psutil==5.6.0 +pillow==6.0.0 +psutil==5.6.1 pyeupi==1.0 pygeoip==0.3.2 -pyparsing==2.3.1 +pyparsing==2.4.0 pypdns==1.3 pypssl==2.1 pyrsistent==0.14.11 pytesseract==0.2.6 python-dateutil==2.8.0 -pyyaml==3.13 +pyyaml==5.1 +pyzbar==0.1.8 rdflib==4.2.2 -redis==3.2.0 -reportlab==3.5.13 -requests-cache==0.4.13 +redis==3.2.1 +reportlab==3.5.19 +requests-cache==0.5.0 requests==2.21.0 -shodan==1.11.1 -sigmatools==0.9 +shodan==1.12.1 +sigmatools==0.10 six==1.12.0 -soupsieve==1.8 -sparqlwrapper==1.8.2 +soupsieve==1.9.1 +sparqlwrapper==1.8.4 stix2-patterns==1.1.0 tabulate==0.8.3 -tornado==6.0.1 +tornado==6.0.2 url-normalize==1.4.1 urlarchiver==0.2 -urllib3==1.24.1 -vulners==1.4.5 -wand==0.5.1 -xlsxwriter==1.1.5 +urllib3==1.24.2 +vulners==1.5.0 +wand==0.5.3 +xlsxwriter==1.1.7 yara-python==3.8.1 yarl==1.3.0 From 7171c8ce92432e7850b8f5d6a51420c3543a36b0 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 13:54:21 +0200 Subject: [PATCH 23/27] initial version of OCR expansion module --- misp_modules/modules/expansion/__init__.py | 3 +- misp_modules/modules/expansion/ocr.py | 51 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/ocr.py diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 994b2898..ec78e9b8 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,5 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', 'qrcode'] + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', + 'qrcode', 'ocr'] diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr.py new file mode 100644 index 00000000..afdf3433 --- /dev/null +++ b/misp_modules/modules/expansion/ocr.py @@ -0,0 +1,51 @@ +import json +import re +import binascii +import cv2 +import np +import pytesseract + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'OCR decoder', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + img_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + image = img_array + image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) + try: + decoded = pytesseract.image_to_string(image) + return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename}, + {'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file type. Only images are supported right now." + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 614fc1354b49bf281b746d3c6ca0bbbffcc21bff Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 24 Apr 2019 14:01:08 +0200 Subject: [PATCH 24/27] chg: [ocr] re module not used - removed --- misp_modules/modules/expansion/ocr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr.py index afdf3433..cd6baca8 100644 --- a/misp_modules/modules/expansion/ocr.py +++ b/misp_modules/modules/expansion/ocr.py @@ -1,5 +1,4 @@ import json -import re import binascii import cv2 import np From 81b0082ae5cb2a42303c4629dd53af954713cb25 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 24 Apr 2019 14:01:48 +0200 Subject: [PATCH 25/27] chg: [init] removed trailing whitespace --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index ec78e9b8..03a78998 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,5 +8,5 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', 'qrcode', 'ocr'] From 5104bce4519d994aa868f9b576b803e393ca1ce7 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 14:53:03 +0200 Subject: [PATCH 26/27] renamed module --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index ec78e9b8..38d59add 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -9,4 +9,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', - 'qrcode', 'ocr'] + 'qrcode', 'ocr-enrich'] From 07f759b07a5994e2fea4410a4a230e2304159009 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 14:53:16 +0200 Subject: [PATCH 27/27] renamed file --- misp_modules/modules/expansion/{ocr.py => ocr-enrich.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename misp_modules/modules/expansion/{ocr.py => ocr-enrich.py} (100%) diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr-enrich.py similarity index 100% rename from misp_modules/modules/expansion/ocr.py rename to misp_modules/modules/expansion/ocr-enrich.py