Merge branch 'master' of github.com:MISP/misp-modules into new_module

pull/305/head
chrisr3d 2019-05-29 10:50:39 +10:00
commit f541b1f4ba
6 changed files with 192 additions and 35 deletions

View File

@ -55,6 +55,7 @@ pdftotext = "*"
lxml = "*"
xlrd = "*"
idna-ssl = {markers="python_version < '3.7'"}
jbxapi = "*"
[requires]
python_version = "3"

60
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "9aac0a9c45df16b9502c13f9468095cf5ffdb8bc407fe2b55faee3ff53d8eba3"
"sha256": "3b1ae107ffee673cfabae67742774ee8ebdc3b82313608b529c2c4cf4a41ddc9"
},
"pipfile-spec": 6,
"requires": {
@ -192,6 +192,13 @@
],
"version": "==0.6.0"
},
"jbxapi": {
"hashes": [
"sha256:ff7c74b3cc06aebd3f2d99a1ffb042b842d527faff1d6006f6224907fcf6ce6f"
],
"index": "pypi",
"version": "==3.1.3"
},
"jsonschema": {
"hashes": [
"sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d",
@ -455,7 +462,7 @@
"pybgpranking": {
"editable": true,
"git": "https://github.com/D4-project/BGP-Ranking.git/",
"ref": "aa0d4581a836503d2298ee046bea49c501eefdd1",
"ref": "429cea9c0787876820984a2df4e982449a84c10e",
"subdirectory": "client"
},
"pydnstrails": {
@ -492,7 +499,7 @@
"pymisp": {
"editable": true,
"git": "https://github.com/MISP/PyMISP.git",
"ref": "e8bba395bc67bf56e41ddd022ebae670c5b0d64b"
"ref": "583fb6592495ea358aad47a8a1ec92d43c13348a"
},
"pyonyphe": {
"editable": true,
@ -523,9 +530,9 @@
},
"pyrsistent": {
"hashes": [
"sha256:5403d37f4d55ff4572b5b5676890589f367a9569529c6f728c11046c4ea4272b"
"sha256:16692ee739d42cf5e39cef8d27649a8c1fdb7aa99887098f1460057c5eb75c3a"
],
"version": "==0.15.1"
"version": "==0.15.2"
},
"pytesseract": {
"hashes": [
@ -637,11 +644,11 @@
},
"requests": {
"hashes": [
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
],
"index": "pypi",
"version": "==2.21.0"
"version": "==2.22.0"
},
"requests-cache": {
"hashes": [
@ -728,10 +735,10 @@
},
"urllib3": {
"hashes": [
"sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4",
"sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb"
"sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
"sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
],
"version": "==1.24.3"
"version": "==1.25.3"
},
"uwhois": {
"editable": true,
@ -927,10 +934,10 @@
},
"pluggy": {
"hashes": [
"sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f",
"sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746"
"sha256:25a1bc1d148c9a640211872b4ff859878d422bccb59c9965e04eed468a0aa180",
"sha256:964cedd2b27c492fbf0b7f58b3284a09cf7f99b0f715941fb24a439b3af1bd1a"
],
"version": "==0.9.0"
"version": "==0.11.0"
},
"py": {
"hashes": [
@ -955,19 +962,19 @@
},
"pytest": {
"hashes": [
"sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d",
"sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5"
"sha256:1a8aa4fa958f8f451ac5441f3ac130d9fc86ea38780dd2715e6d5c5882700b24",
"sha256:b8bf138592384bd4e87338cb0f256bf5f615398a649d4bd83915f0e4047a5ca6"
],
"index": "pypi",
"version": "==4.4.1"
"version": "==4.5.0"
},
"requests": {
"hashes": [
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
],
"index": "pypi",
"version": "==2.21.0"
"version": "==2.22.0"
},
"six": {
"hashes": [
@ -978,10 +985,17 @@
},
"urllib3": {
"hashes": [
"sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4",
"sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb"
"sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
"sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
],
"version": "==1.24.3"
"version": "==1.25.3"
},
"wcwidth": {
"hashes": [
"sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
"sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
],
"version": "==0.1.7"
}
}
}

View File

@ -40,6 +40,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
* [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com).
* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
* [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
* [Joe Sandbox](misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox.
* [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
* [ocr-enrich](misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP.

View File

@ -1,9 +1,9 @@
-i https://pypi.org/simple
-e .
-e git+https://github.com/D4-project/BGP-Ranking.git/@4e0741056bcc0077de1120b8724a31330b26033e#egg=pybgpranking&subdirectory=client
-e git+https://github.com/D4-project/IPASN-History.git/@c0c2bbf8d70811982dad065ea463a7e01593a38d#egg=pyipasnhistory&subdirectory=client
-e git+https://github.com/D4-project/BGP-Ranking.git/@429cea9c0787876820984a2df4e982449a84c10e#egg=pybgpranking&subdirectory=client
-e git+https://github.com/D4-project/IPASN-History.git/@47cd0f2658ab172fce42126ff3a1dbcddfb0b5fb#egg=pyipasnhistory&subdirectory=client
-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
-e git+https://github.com/MISP/PyMISP.git@582dda0ce2a8ca8e1dd2cf3842e0491caca51c62#egg=pymisp
-e git+https://github.com/MISP/PyMISP.git@583fb6592495ea358aad47a8a1ec92d43c13348a#egg=pymisp
-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client
-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
@ -30,6 +30,7 @@ httplib2==0.12.3
idna-ssl==1.1.0 ; python_version < '3.7'
idna==2.8
isodate==0.6.0
jbxapi==3.1.3
jsonschema==3.0.1
lxml==4.3.3
maclookup==1.0.3
@ -47,22 +48,22 @@ psutil==5.6.2
pyeupi==1.0
pygeoip==0.3.2
pyparsing==2.4.0
pypdns==1.3
pypdns==1.4.1
pypssl==2.1
pyrsistent==0.15.1
pyrsistent==0.15.2
pytesseract==0.2.6
python-dateutil==2.8.0
python-docx==0.8.10
python-pptx==0.6.17
python-pptx==0.6.18
pytz==2019.1
pyyaml==5.1
pyzbar==0.1.8
rdflib==4.2.2
redis==3.2.1
reportlab==3.5.20
reportlab==3.5.21
requests-cache==0.5.0
requests==2.21.0
shodan==1.12.1
requests==2.22.0
shodan==1.13.0
sigmatools==0.10
six==1.12.0
soupsieve==1.9.1
@ -72,10 +73,10 @@ tabulate==0.8.3
tornado==6.0.2
url-normalize==1.4.1
urlarchiver==0.2
urllib3==1.24.2
urllib3==1.25.3
vulners==1.5.0
wand==0.5.3
xlrd==1.2.0
xlsxwriter==1.1.7
xlsxwriter==1.1.8
yara-python==3.8.1
yarl==1.3.0

View File

@ -10,4 +10,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io',
'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors',
'qrcode', 'ocr-enrich', 'pdf-enrich', 'docx-enrich', 'xlsx-enrich', 'pptx-enrich',
'ods-enrich', 'odt-enrich', 'urlhaus']
'ods-enrich', 'odt-enrich', 'joesandbox_submit', 'urlhaus']

View File

@ -0,0 +1,140 @@
import jbxapi
import base64
import io
import json
import logging
import sys
import zipfile
import re
from urllib.parse import urljoin
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
sh = logging.StreamHandler(sys.stdout)
sh.setLevel(logging.DEBUG)
fmt = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
sh.setFormatter(fmt)
log.addHandler(sh)
moduleinfo = {
"version": "1.0",
"author": "Joe Security LLC",
"description": "Submit files and URLs to Joe Sandbox",
"module-type": ["expansion", "hover"]
}
moduleconfig = [
"apiurl",
"apikey",
"accept-tac",
"report-cache",
"systems",
]
mispattributes = {
"input": ["attachment", "malware-sample", "url", "domain"],
"output": ["link"],
}
def handler(q=False):
if q is False:
return False
request = json.loads(q)
apiurl = request["config"].get("apiurl") or "https://jbxcloud.joesecurity.org/api"
apikey = request["config"].get("apikey")
# systems
systems = request["config"].get("systems") or ""
systems = [s.strip() for s in re.split(r"[\s,;]", systems) if s.strip()]
try:
accept_tac = _parse_bool(request["config"].get("accept-tac"), "accept-tac")
report_cache = _parse_bool(request["config"].get("report-cache"), "report-cache")
except _ParseError as e:
return {"error": str(e)}
params = {
"report-cache": report_cache,
"systems": systems,
}
if not apikey:
return {"error": "No API key provided"}
joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent="MISP joesandbox_submit", accept_tac=accept_tac)
try:
is_url_submission = "url" in request or "domain" in request
if is_url_submission:
url = request.get("url") or request.get("domain")
log.info("Submitting URL: %s", url)
result = joe.submit_url(url, params=params)
else:
if "malware-sample" in request:
filename = request.get("malware-sample").split("|", 1)[0]
data = _decode_malware(request["data"], True)
elif "attachment" in request:
filename = request["attachment"]
data = _decode_malware(request["data"], False)
data_fp = io.BytesIO(data)
log.info("Submitting sample: %s", filename)
result = joe.submit_sample((filename, data_fp), params=params)
assert "submission_id" in result
except jbxapi.JoeException as e:
return {"error": str(e)}
link_to_analysis = urljoin(apiurl, "../submissions/{}".format(result["submission_id"]))
return {
"results": [{
"types": "link",
"categories": "External analysis",
"values": link_to_analysis,
}]
}
def introspection():
return mispattributes
def version():
moduleinfo["config"] = moduleconfig
return moduleinfo
def _decode_malware(data, is_encrypted):
data = base64.b64decode(data)
if is_encrypted:
with zipfile.ZipFile(io.BytesIO(data)) as zipf:
data = zipf.read(zipf.namelist()[0], pwd=b"infected")
return data
class _ParseError(Exception):
pass
def _parse_bool(value, name="bool"):
if value is None or value == "":
return None
if value == "true":
return True
if value == "false":
return False
raise _ParseError("Cannot parse {}. Must be 'true' or 'false'".format(name))