diff --git a/Pipfile.lock b/Pipfile.lock index 8a947a6..37f5272 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "27f2f4b2d71e59a134b4039f79a71677746f0f8cebec51a73c3936d9923dc92e" + "sha256": "e31638147f27ca5c90e27ebecdeb871f027feb37ede229b4296da35094a9516f" }, "pipfile-spec": 6, "requires": { @@ -50,6 +50,20 @@ "markers": "python_version >= '3'", "version": "==4.7.2" }, + "apiosintds": { + "hashes": [ + "sha256:9a92f3fdb265f49046a871338419709f784b8ed82b249435c3c40e47d2ab4bcf" + ], + "index": "pypi", + "version": "==1.8.2" + }, + "argparse": { + "hashes": [ + "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", + "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" + ], + "version": "==1.4.0" + }, "async-timeout": { "hashes": [ "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", @@ -123,6 +137,13 @@ ], "version": "==0.4.1" }, + "decorator": { + "hashes": [ + "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", + "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" + ], + "version": "==4.4.1" + }, "deprecated": { "hashes": [ "sha256:a515c4cf75061552e0284d123c3066fbbe398952c87333a92b8fc3dd8e4f9cc1", @@ -167,9 +188,9 @@ }, "future": { "hashes": [ - "sha256:858e38522e8fd0d3ce8f0c1feaf0603358e366d5403209674c7b617fa0c24093" + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" ], - "version": "==0.18.1" + "version": "==0.18.2" }, "geoip2": { "hashes": [ @@ -572,7 +593,7 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "3e8c36dc2f34b5d812a6b6d1bd1a619f01286657" + "ref": "87fd06a8893feafaffd461d6d611be4d02e5a4a2" }, "pyonyphe": { "editable": true, @@ -693,32 +714,37 @@ }, "reportlab": { "hashes": [ - "sha256:044d5ae40e1540e4ebdabb4b807bebabfc29351f423b5ace9452ba1558412f3c", - "sha256:20dd16472c871948f0e60a50487929b37810e143320f25d339c93bbf0739af63", - "sha256:2b05e607fd9b24767a30bfb40a72388a05ccd51dda5208151bc39ed51b4959f6", - "sha256:33516fb7b15a180f5cb41b9c21245180c470d5de07c42af14684eecc53dedca1", - "sha256:3e2d2ea8ac3d63c918a2b40476c2745704d0364abe2b9c844c75992132a5eac7", - "sha256:3ef2dfd030d030f0c0ee9fcdbbe13044ed7497b6e8a41515e6fda7529d5dd3a9", - "sha256:46b042cb8c839fb5a9951dc4e6555c976f5daf0a89ad9333d3d944f14a71e4a1", - "sha256:4a0c603cd056563af5104ab4fb016538f0a66a53975291b48f27149fb783c840", - "sha256:5540792fd8eb1515b38d21ef3d84ca4f8d4b959079f015cbcb43ec10dde77689", - "sha256:55fe512159f6820f30fcd3500db1b4223bccd4840fa102c5c7b4a4f28a543363", - "sha256:60a3a41e2f59a6a02b1e38628885441334d055ec766bb785817f32944d2f6eab", - "sha256:6549611e0e88442fd83cbab2a8b01041dff7ae5c22c08b349b3832a8bad3b6bd", - "sha256:66f296d9420f6a2395399632e59545384a4f2173716ed595263342dbce8e8e3a", - "sha256:784f185fbbff0063577e7c3392caf1aaf27d25548d086329b43b9804bd476304", - "sha256:8cdcb85df200e49501cd9aa864743c7fc51d4e55571e57eb2ead9cf5c134e3ff", - "sha256:8f52916965d4d6f3befda9ea0ced856c0c11f30f9829dd7cccf22823c3ae0e99", - "sha256:be6b38189356cf89a227805a230c7240cda659523d58b2409336599dd4c45425", - "sha256:c08b60ae0670dbf344e03ea3cabd5c6040040e30b98c51958428a8ac3aa03dfa", - "sha256:c80388b8d2e656801dbf73ca291df2592f13240acf90e146a288c4244aab90fe", - "sha256:f25870bf8f1dc7b9a78627dd5913c6901a397794c546b1b4702ace1fb477a5e3", - "sha256:f269bd6bd31835e8e6bc1e202d85dc3dccd443e58041e06603ef374890dda0d7", - "sha256:f3e992c74135cf8fe48a06dfd008a644e8251f816dd6f1a2c8e12e261cae6da2", - "sha256:fa85c5551ccec02dee2b4d5ea22fb73dcba1285fe26611042a53b31ddae3cdde" + "sha256:149f0eeb4ea716441638b05fd6d3667d32f1463f3eac50b63e100a73a5533cdd", + "sha256:1aa9a2e1a87749db265b592ad25e498b39f70fce9f53a012cdf69f74259b6e43", + "sha256:1f5ce489adb2db2862249492e6367539cfa65b781cb06dcf13363dc52219be7e", + "sha256:23b28ba1784a6c52a926c075abd9f396d03670e71934b24db5ff684f8b870e0f", + "sha256:3d3de0f4facdd7e3c56ecbc55733a958b86c35a8e7ba6066c7b1ba383e282f58", + "sha256:484d346b8f463ba2ddaf6d365c6ac5971cd062528b6d5ba68cac02b9435366c5", + "sha256:4da2467def21f2e20720b21f6c18e7f7866720a955c716b990e94e3979fe913f", + "sha256:5ebdf22daee7d8e630134d94f477fe6abd65a65449d4eec682a7b458b5249604", + "sha256:655a1b68be18a73fec5233fb5d81f726b4db32269e487aecf5b6853cca926d86", + "sha256:6c535a304888dafe50c2c24d4924aeefc11e0542488ee6965f6133d415e86bbc", + "sha256:7560ef655ac6448bb257fd34bfdfb8d546f9c7c0900ed8963fb8509f75e8ca80", + "sha256:7a1c2fa3e6310dbe47efee2020dc0f25be7a75ff09a8fedc4a87d4397f3810c1", + "sha256:817c344b9aa53b5bfc2f58ff82111a1e85ca4c8b68d1add088b547360a6ebcfa", + "sha256:81d950e398d6758aeaeeb267aa1a62940735414c980f77dd0a270cef1782a43d", + "sha256:83ef44936ef4e9c432d62bc2b72ec8d772b87af319d123e827a72e9b6884c851", + "sha256:9f975adc2c7a236403f0bc91d7a3916e644e47b1f1e3990325f15e73b83581ec", + "sha256:a5ca59e2b7e70a856de6db9dadd3e11a1b3b471c999585284d5c1d479c01cf5d", + "sha256:ad2cf5a673c05fae9e91e987994b95205c13c5fa55d7393cf8b06f9de6f92990", + "sha256:b8c3d76276372f87b7c8ff22065dbc072cca5ffb06ba0267edc298df7acf942d", + "sha256:b93f7f908e916d9413dd8c04da1ccb3977e446803f59078424decdc0de449133", + "sha256:c0ecd0af92c759edec0d24ba92f4a18c28d4a19229ae7c8249f94e82f3d76288", + "sha256:c9e38eefc90a02c072a87a627ff66b2d67c23f6f82274d2aa7fb28e644e8f409", + "sha256:ca2a1592d2e181a04372d0276ee847308ea206dfe7c86fe94769e7ac126e6e85", + "sha256:ce1dfc9beec83e66250ca3afaf5ddf6b9a3ce70a30a9526dec7c6bec3266baf1", + "sha256:d3550c90751132b26b72a78954905974f33b1237335fbe0d8be957f9636c376a", + "sha256:e35a574f4e5ec0fdd5dc354e74ec143d853abd7f76db435ffe2a57d0161a22eb", + "sha256:ee5cafca6ef1a38fef8cbf3140dd2198ad1ee82331530b546039216ef94f93cb", + "sha256:fa1c969176cb3594a785c6818bcb943ebd49453791f702380b13a35fa23b385a" ], "index": "pypi", - "version": "==3.5.31" + "version": "==3.5.32" }, "requests": { "hashes": [ @@ -824,6 +850,12 @@ "ref": "411572840eba4c72dc321c549b36a54ed5cea9de", "subdirectory": "client" }, + "validators": { + "hashes": [ + "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" + ], + "version": "==0.14.0" + }, "vulners": { "hashes": [ "sha256:245c07e49e55a604efde43cba723ac7b9345247e5ac8c4f998dcd36c05e4b1b9", @@ -986,11 +1018,11 @@ }, "flake8": { "hashes": [ - "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", - "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" ], "index": "pypi", - "version": "==3.7.8" + "version": "==3.7.9" }, "idna": { "hashes": [ diff --git a/README.md b/README.md index 462e4c1..dbd7e77 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj ### Expansion modules +* [apiosintDS](misp_modules/modules/expansion/apiosintds.py) - a hover and expansion module to query the OSINT.digitalside.it API. * [Backscatter.io](misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations. * [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. * [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused. @@ -30,8 +31,9 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. * [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. -* [docx-enrich](misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). +* [docx-enrich](misp_modules/modules/expansion/docx_enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. +* [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate EQL queries from attributes. * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. @@ -45,15 +47,15 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [Joe Sandbox query](misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data. * [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). * [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. -* [ocr-enrich](misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP. -* [ods-enrich](misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). -* [odt-enrich](misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). +* [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP. +* [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). +* [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). * [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. * [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. -* [pdf-enrich](misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser). -* [pptx-enrich](misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser). +* [pdf-enrich](misp_modules/modules/expansion/pdf_enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser). +* [pptx-enrich](misp_modules/modules/expansion/pptx_enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser). * [qrcode](misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values. * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. @@ -75,7 +77,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [whois](misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd). * [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. * [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. -* [xlsx-enrich](misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser). +* [xlsx-enrich](misp_modules/modules/expansion/xlsx_enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser). * [YARA query](misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes. * [YARA syntax validator](misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator. diff --git a/doc/README.md b/doc/README.md index af52175..54100c0 100644 --- a/doc/README.md +++ b/doc/README.md @@ -2,6 +2,26 @@ ## Expansion Modules +#### [apiosintds](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/apiosintds.py) + +On demand query API for OSINT.digitalside.it project. +- **features**: +>The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute. +> +>The result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls. +> +>Furthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it +- **input**: +>A domain, ip, url or hash attribute. +- **output**: +>Hashes and urls resulting from the query to OSINT.digitalside.it +- **references**: +>https://osint.digitalside.it/#About +- **requirements**: +>The apiosintDS python library to query the OSINT.digitalside.it API. + +----- + #### [backscatter_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) @@ -306,6 +326,22 @@ DomainTools MISP expansion module. ----- +#### [eql](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eql.py) + + + +Generates EQL queries from attributes +- **features**: +>The module simply generates EQL rules out of the input attribute. +- **input**: +>A filename or ip attribute. +- **output**: +>The EQL query generated from the input attribute. +- **references**: +>https://eql.readthedocs.io/en/latest/ + +----- + #### [eupi](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) diff --git a/doc/expansion/apiosintds.json b/doc/expansion/apiosintds.json new file mode 100644 index 0000000..81a1eec --- /dev/null +++ b/doc/expansion/apiosintds.json @@ -0,0 +1,8 @@ +{ + "description": "On demand query API for OSINT.digitalside.it project.", + "requirements": ["The apiosintDS python library to query the OSINT.digitalside.it API."], + "input": "A domain, ip, url or hash attribute.", + "output": "Hashes and urls resulting from the query to OSINT.digitalside.it", + "references": ["https://osint.digitalside.it/#About"], + "features": "The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute.\n\nThe result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls.\n\nFurthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it" +} diff --git a/doc/expansion/eql.json b/doc/expansion/eql.json new file mode 100644 index 0000000..d800ab6 --- /dev/null +++ b/doc/expansion/eql.json @@ -0,0 +1,9 @@ +{ + "description": "Generates EQL queries from attributes", + "logo": "logos/eql.png", + "requirements": [], + "input": "A filename or ip attribute.", + "output": "The EQL query generated from the input attribute.", + "references": ["https://eql.readthedocs.io/en/latest/"], + "features": "The module simply generates EQL rules out of the input attribute." +} diff --git a/doc/logos/eql.png b/doc/logos/eql.png new file mode 100644 index 0000000..4cddb91 Binary files /dev/null and b/doc/logos/eql.png differ diff --git a/misp_modules/modules/expansion/vulners.py b/misp_modules/modules/expansion/vulners.py index 557fdb6..c2ec7de 100644 --- a/misp_modules/modules/expansion/vulners.py +++ b/misp_modules/modules/expansion/vulners.py @@ -21,7 +21,10 @@ def handler(q=False): exploit_summary = '' vuln_summary = '' - key = request['config'].get('apikey') + if not request.get('config') or not request['config'].get('apikey'): + return {'error': "A Vulners api key is required for this module."} + + key = request['config']['apikey'] vulners_api = vulners.Vulners(api_key=key) vulnerability = request.get('vulnerability') vulners_document = vulners_api.document(vulnerability) @@ -44,8 +47,8 @@ def handler(q=False): ai_summary += 'Vulners AI Score is ' + str(vulners_ai_score[0]) + " " if vulners_exploits: - exploit_summary += " || " + str(len(vulners_exploits[0])) + " Public exploits available:\n " - for exploit in vulners_exploits[0]: + exploit_summary += " || " + str(len(vulners_exploits)) + " Public exploits available:\n " + for exploit in vulners_exploits: exploit_summary += exploit['title'] + " " + exploit['href'] + "\n " exploit_summary += "|| Vulnerability Description: " + vuln_summary diff --git a/misp_modules/modules/expansion/whois.py b/misp_modules/modules/expansion/whois.py index 4aec40c..22c4850 100755 --- a/misp_modules/modules/expansion/whois.py +++ b/misp_modules/modules/expansion/whois.py @@ -29,8 +29,8 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('apikey') and request['config'].et('url')): - misperrors['error'] = 'EUPI authentication is missing' + if not request.get('config') or (not request['config'].get('server') and not request['config'].get('port')): + misperrors['error'] = 'Whois local instance address is missing' return misperrors uwhois = Uwhois(request['config']['server'], int(request['config']['port']))