diff --git a/Pipfile.lock b/Pipfile.lock
index 8a947a6..37f5272 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "27f2f4b2d71e59a134b4039f79a71677746f0f8cebec51a73c3936d9923dc92e"
+ "sha256": "e31638147f27ca5c90e27ebecdeb871f027feb37ede229b4296da35094a9516f"
},
"pipfile-spec": 6,
"requires": {
@@ -50,6 +50,20 @@
"markers": "python_version >= '3'",
"version": "==4.7.2"
},
+ "apiosintds": {
+ "hashes": [
+ "sha256:9a92f3fdb265f49046a871338419709f784b8ed82b249435c3c40e47d2ab4bcf"
+ ],
+ "index": "pypi",
+ "version": "==1.8.2"
+ },
+ "argparse": {
+ "hashes": [
+ "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4",
+ "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314"
+ ],
+ "version": "==1.4.0"
+ },
"async-timeout": {
"hashes": [
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
@@ -123,6 +137,13 @@
],
"version": "==0.4.1"
},
+ "decorator": {
+ "hashes": [
+ "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce",
+ "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"
+ ],
+ "version": "==4.4.1"
+ },
"deprecated": {
"hashes": [
"sha256:a515c4cf75061552e0284d123c3066fbbe398952c87333a92b8fc3dd8e4f9cc1",
@@ -167,9 +188,9 @@
},
"future": {
"hashes": [
- "sha256:858e38522e8fd0d3ce8f0c1feaf0603358e366d5403209674c7b617fa0c24093"
+ "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
],
- "version": "==0.18.1"
+ "version": "==0.18.2"
},
"geoip2": {
"hashes": [
@@ -572,7 +593,7 @@
"pymisp": {
"editable": true,
"git": "https://github.com/MISP/PyMISP.git",
- "ref": "3e8c36dc2f34b5d812a6b6d1bd1a619f01286657"
+ "ref": "87fd06a8893feafaffd461d6d611be4d02e5a4a2"
},
"pyonyphe": {
"editable": true,
@@ -693,32 +714,37 @@
},
"reportlab": {
"hashes": [
- "sha256:044d5ae40e1540e4ebdabb4b807bebabfc29351f423b5ace9452ba1558412f3c",
- "sha256:20dd16472c871948f0e60a50487929b37810e143320f25d339c93bbf0739af63",
- "sha256:2b05e607fd9b24767a30bfb40a72388a05ccd51dda5208151bc39ed51b4959f6",
- "sha256:33516fb7b15a180f5cb41b9c21245180c470d5de07c42af14684eecc53dedca1",
- "sha256:3e2d2ea8ac3d63c918a2b40476c2745704d0364abe2b9c844c75992132a5eac7",
- "sha256:3ef2dfd030d030f0c0ee9fcdbbe13044ed7497b6e8a41515e6fda7529d5dd3a9",
- "sha256:46b042cb8c839fb5a9951dc4e6555c976f5daf0a89ad9333d3d944f14a71e4a1",
- "sha256:4a0c603cd056563af5104ab4fb016538f0a66a53975291b48f27149fb783c840",
- "sha256:5540792fd8eb1515b38d21ef3d84ca4f8d4b959079f015cbcb43ec10dde77689",
- "sha256:55fe512159f6820f30fcd3500db1b4223bccd4840fa102c5c7b4a4f28a543363",
- "sha256:60a3a41e2f59a6a02b1e38628885441334d055ec766bb785817f32944d2f6eab",
- "sha256:6549611e0e88442fd83cbab2a8b01041dff7ae5c22c08b349b3832a8bad3b6bd",
- "sha256:66f296d9420f6a2395399632e59545384a4f2173716ed595263342dbce8e8e3a",
- "sha256:784f185fbbff0063577e7c3392caf1aaf27d25548d086329b43b9804bd476304",
- "sha256:8cdcb85df200e49501cd9aa864743c7fc51d4e55571e57eb2ead9cf5c134e3ff",
- "sha256:8f52916965d4d6f3befda9ea0ced856c0c11f30f9829dd7cccf22823c3ae0e99",
- "sha256:be6b38189356cf89a227805a230c7240cda659523d58b2409336599dd4c45425",
- "sha256:c08b60ae0670dbf344e03ea3cabd5c6040040e30b98c51958428a8ac3aa03dfa",
- "sha256:c80388b8d2e656801dbf73ca291df2592f13240acf90e146a288c4244aab90fe",
- "sha256:f25870bf8f1dc7b9a78627dd5913c6901a397794c546b1b4702ace1fb477a5e3",
- "sha256:f269bd6bd31835e8e6bc1e202d85dc3dccd443e58041e06603ef374890dda0d7",
- "sha256:f3e992c74135cf8fe48a06dfd008a644e8251f816dd6f1a2c8e12e261cae6da2",
- "sha256:fa85c5551ccec02dee2b4d5ea22fb73dcba1285fe26611042a53b31ddae3cdde"
+ "sha256:149f0eeb4ea716441638b05fd6d3667d32f1463f3eac50b63e100a73a5533cdd",
+ "sha256:1aa9a2e1a87749db265b592ad25e498b39f70fce9f53a012cdf69f74259b6e43",
+ "sha256:1f5ce489adb2db2862249492e6367539cfa65b781cb06dcf13363dc52219be7e",
+ "sha256:23b28ba1784a6c52a926c075abd9f396d03670e71934b24db5ff684f8b870e0f",
+ "sha256:3d3de0f4facdd7e3c56ecbc55733a958b86c35a8e7ba6066c7b1ba383e282f58",
+ "sha256:484d346b8f463ba2ddaf6d365c6ac5971cd062528b6d5ba68cac02b9435366c5",
+ "sha256:4da2467def21f2e20720b21f6c18e7f7866720a955c716b990e94e3979fe913f",
+ "sha256:5ebdf22daee7d8e630134d94f477fe6abd65a65449d4eec682a7b458b5249604",
+ "sha256:655a1b68be18a73fec5233fb5d81f726b4db32269e487aecf5b6853cca926d86",
+ "sha256:6c535a304888dafe50c2c24d4924aeefc11e0542488ee6965f6133d415e86bbc",
+ "sha256:7560ef655ac6448bb257fd34bfdfb8d546f9c7c0900ed8963fb8509f75e8ca80",
+ "sha256:7a1c2fa3e6310dbe47efee2020dc0f25be7a75ff09a8fedc4a87d4397f3810c1",
+ "sha256:817c344b9aa53b5bfc2f58ff82111a1e85ca4c8b68d1add088b547360a6ebcfa",
+ "sha256:81d950e398d6758aeaeeb267aa1a62940735414c980f77dd0a270cef1782a43d",
+ "sha256:83ef44936ef4e9c432d62bc2b72ec8d772b87af319d123e827a72e9b6884c851",
+ "sha256:9f975adc2c7a236403f0bc91d7a3916e644e47b1f1e3990325f15e73b83581ec",
+ "sha256:a5ca59e2b7e70a856de6db9dadd3e11a1b3b471c999585284d5c1d479c01cf5d",
+ "sha256:ad2cf5a673c05fae9e91e987994b95205c13c5fa55d7393cf8b06f9de6f92990",
+ "sha256:b8c3d76276372f87b7c8ff22065dbc072cca5ffb06ba0267edc298df7acf942d",
+ "sha256:b93f7f908e916d9413dd8c04da1ccb3977e446803f59078424decdc0de449133",
+ "sha256:c0ecd0af92c759edec0d24ba92f4a18c28d4a19229ae7c8249f94e82f3d76288",
+ "sha256:c9e38eefc90a02c072a87a627ff66b2d67c23f6f82274d2aa7fb28e644e8f409",
+ "sha256:ca2a1592d2e181a04372d0276ee847308ea206dfe7c86fe94769e7ac126e6e85",
+ "sha256:ce1dfc9beec83e66250ca3afaf5ddf6b9a3ce70a30a9526dec7c6bec3266baf1",
+ "sha256:d3550c90751132b26b72a78954905974f33b1237335fbe0d8be957f9636c376a",
+ "sha256:e35a574f4e5ec0fdd5dc354e74ec143d853abd7f76db435ffe2a57d0161a22eb",
+ "sha256:ee5cafca6ef1a38fef8cbf3140dd2198ad1ee82331530b546039216ef94f93cb",
+ "sha256:fa1c969176cb3594a785c6818bcb943ebd49453791f702380b13a35fa23b385a"
],
"index": "pypi",
- "version": "==3.5.31"
+ "version": "==3.5.32"
},
"requests": {
"hashes": [
@@ -824,6 +850,12 @@
"ref": "411572840eba4c72dc321c549b36a54ed5cea9de",
"subdirectory": "client"
},
+ "validators": {
+ "hashes": [
+ "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae"
+ ],
+ "version": "==0.14.0"
+ },
"vulners": {
"hashes": [
"sha256:245c07e49e55a604efde43cba723ac7b9345247e5ac8c4f998dcd36c05e4b1b9",
@@ -986,11 +1018,11 @@
},
"flake8": {
"hashes": [
- "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548",
- "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696"
+ "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb",
+ "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca"
],
"index": "pypi",
- "version": "==3.7.8"
+ "version": "==3.7.9"
},
"idna": {
"hashes": [
diff --git a/README.md b/README.md
index 462e4c1..dbd7e77 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
### Expansion modules
+* [apiosintDS](misp_modules/modules/expansion/apiosintds.py) - a hover and expansion module to query the OSINT.digitalside.it API.
* [Backscatter.io](misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations.
* [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking.
* [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
@@ -30,8 +31,9 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox.
* [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name.
* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
-* [docx-enrich](misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser).
+* [docx-enrich](misp_modules/modules/expansion/docx_enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser).
* [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
+* [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate EQL queries from attributes.
* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
* [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
* [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
@@ -45,15 +47,15 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [Joe Sandbox query](misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data.
* [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
-* [ocr-enrich](misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP.
-* [ods-enrich](misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
-* [odt-enrich](misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
+* [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP.
+* [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
+* [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
* [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe.
* [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe.
* [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).
* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
-* [pdf-enrich](misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser).
-* [pptx-enrich](misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser).
+* [pdf-enrich](misp_modules/modules/expansion/pdf_enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser).
+* [pptx-enrich](misp_modules/modules/expansion/pptx_enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser).
* [qrcode](misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values.
* [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
* [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
@@ -75,7 +77,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
* [whois](misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd).
* [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
* [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
-* [xlsx-enrich](misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser).
+* [xlsx-enrich](misp_modules/modules/expansion/xlsx_enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser).
* [YARA query](misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes.
* [YARA syntax validator](misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator.
diff --git a/doc/README.md b/doc/README.md
index af52175..54100c0 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -2,6 +2,26 @@
## Expansion Modules
+#### [apiosintds](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/apiosintds.py)
+
+On demand query API for OSINT.digitalside.it project.
+- **features**:
+>The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute.
+>
+>The result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls.
+>
+>Furthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it
+- **input**:
+>A domain, ip, url or hash attribute.
+- **output**:
+>Hashes and urls resulting from the query to OSINT.digitalside.it
+- **references**:
+>https://osint.digitalside.it/#About
+- **requirements**:
+>The apiosintDS python library to query the OSINT.digitalside.it API.
+
+-----
+
#### [backscatter_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py)
@@ -306,6 +326,22 @@ DomainTools MISP expansion module.
-----
+#### [eql](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eql.py)
+
+
+
+Generates EQL queries from attributes
+- **features**:
+>The module simply generates EQL rules out of the input attribute.
+- **input**:
+>A filename or ip attribute.
+- **output**:
+>The EQL query generated from the input attribute.
+- **references**:
+>https://eql.readthedocs.io/en/latest/
+
+-----
+
#### [eupi](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py)
diff --git a/doc/expansion/apiosintds.json b/doc/expansion/apiosintds.json
new file mode 100644
index 0000000..81a1eec
--- /dev/null
+++ b/doc/expansion/apiosintds.json
@@ -0,0 +1,8 @@
+{
+ "description": "On demand query API for OSINT.digitalside.it project.",
+ "requirements": ["The apiosintDS python library to query the OSINT.digitalside.it API."],
+ "input": "A domain, ip, url or hash attribute.",
+ "output": "Hashes and urls resulting from the query to OSINT.digitalside.it",
+ "references": ["https://osint.digitalside.it/#About"],
+ "features": "The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute.\n\nThe result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls.\n\nFurthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it"
+}
diff --git a/doc/expansion/eql.json b/doc/expansion/eql.json
new file mode 100644
index 0000000..d800ab6
--- /dev/null
+++ b/doc/expansion/eql.json
@@ -0,0 +1,9 @@
+{
+ "description": "Generates EQL queries from attributes",
+ "logo": "logos/eql.png",
+ "requirements": [],
+ "input": "A filename or ip attribute.",
+ "output": "The EQL query generated from the input attribute.",
+ "references": ["https://eql.readthedocs.io/en/latest/"],
+ "features": "The module simply generates EQL rules out of the input attribute."
+}
diff --git a/doc/logos/eql.png b/doc/logos/eql.png
new file mode 100644
index 0000000..4cddb91
Binary files /dev/null and b/doc/logos/eql.png differ
diff --git a/misp_modules/modules/expansion/vulners.py b/misp_modules/modules/expansion/vulners.py
index 557fdb6..c2ec7de 100644
--- a/misp_modules/modules/expansion/vulners.py
+++ b/misp_modules/modules/expansion/vulners.py
@@ -21,7 +21,10 @@ def handler(q=False):
exploit_summary = ''
vuln_summary = ''
- key = request['config'].get('apikey')
+ if not request.get('config') or not request['config'].get('apikey'):
+ return {'error': "A Vulners api key is required for this module."}
+
+ key = request['config']['apikey']
vulners_api = vulners.Vulners(api_key=key)
vulnerability = request.get('vulnerability')
vulners_document = vulners_api.document(vulnerability)
@@ -44,8 +47,8 @@ def handler(q=False):
ai_summary += 'Vulners AI Score is ' + str(vulners_ai_score[0]) + " "
if vulners_exploits:
- exploit_summary += " || " + str(len(vulners_exploits[0])) + " Public exploits available:\n "
- for exploit in vulners_exploits[0]:
+ exploit_summary += " || " + str(len(vulners_exploits)) + " Public exploits available:\n "
+ for exploit in vulners_exploits:
exploit_summary += exploit['title'] + " " + exploit['href'] + "\n "
exploit_summary += "|| Vulnerability Description: " + vuln_summary
diff --git a/misp_modules/modules/expansion/whois.py b/misp_modules/modules/expansion/whois.py
index 4aec40c..22c4850 100755
--- a/misp_modules/modules/expansion/whois.py
+++ b/misp_modules/modules/expansion/whois.py
@@ -29,8 +29,8 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
- if not request.get('config') and not (request['config'].get('apikey') and request['config'].et('url')):
- misperrors['error'] = 'EUPI authentication is missing'
+ if not request.get('config') or (not request['config'].get('server') and not request['config'].get('port')):
+ misperrors['error'] = 'Whois local instance address is missing'
return misperrors
uwhois = Uwhois(request['config']['server'], int(request['config']['port']))