Added apiosintDS module to query OSINT.digitalside.it services

REQUIREMENTS

@@ -9,6 +9,7 @@
 -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
 -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe
 aiohttp==3.4.4
+apiosintDS==1.8.1
 antlr4-python3-runtime==4.7.2 ; python_version >= '3'
 async-timeout==3.0.1
 attrs==19.1.0

misp_modules/modules/expansion/__init__.py

@@ -14,4 +14,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors',
            'qrcode', 'ocr_enrich', 'pdf_enrich', 'docx_enrich', 'xlsx_enrich', 'pptx_enrich',
            'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus',
-           'virustotal_public']
+           'virustotal_public', 'apiosintds']

misp_modules/modules/expansion/apiosintds.py

@@ -0,0 +1,141 @@
+import json
+import logging
+import sys
+import os
+from apiosintDS import apiosintDS
+
+log = logging.getLogger('apiosintDS')
+log.setLevel(logging.DEBUG)
+apiodbg = logging.StreamHandler(sys.stdout)
+apiodbg.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+apiodbg.setFormatter(formatter)
+log.addHandler(apiodbg)
+
+misperrors = {'error': 'Error'}
+
+mispattributes = {'input': ["domain", "domain|ip", "hostname", "ip-dst", "ip-src", "ip-dst|port", "ip-src|port", "url",
+                            "md5", "sha1", "sha256", "filename|md5", "filename|sha1", "filename|sha256"],
+                  'output': ["domain", "ip-dst", "url", "comment", "md5", "sha1", "sha256"]
+                  }
+
+moduleinfo = {'version': '0.1', 'author': 'Davide Baglieri aka davidonzo',
+              'description': 'On demand query API for OSINT.digitalside.it project.',
+              'module-type': ['expansion', 'hover']}
+
+moduleconfig = ['import_related_hashes', 'cache', 'cache_directory']
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    tosubmit = []
+    if request.get('domain'):
+        tosubmit.append(request['domain'])
+    elif request.get('domain|ip'):
+        tosubmit.append(request['domain|ip'].split('|')[0])
+        tosubmit.append(request['domain|ip'].split('|')[1])
+    elif request.get('hostname'):
+        tosubmit.append(request['hostname'])
+    elif request.get('ip-dst'):
+        tosubmit.append(request['ip-dst'])
+    elif request.get('ip-src'):
+        tosubmit.append(request['ip-src'])
+    elif request.get('ip-dst|port'):
+        tosubmit.append(request['ip-dst|port'].split('|')[0])
+    elif request.get('ip-src|port'):
+        tosubmit.append(request['ip-src|port'].split('|')[0])
+    elif request.get('url'):
+        tosubmit.append(request['url'])
+    elif request.get('md5'):
+        tosubmit.append(request['md5'])
+    elif request.get('sha1'):
+        tosubmit.append(request['sha1'])
+    elif request.get('sha256'):
+        tosubmit.append(request['sha256'])
+    elif request.get('filename|md5'):
+        tosubmit.append(request['filename|md5'].split('|')[1])
+    elif request.get('filename|sha1'):
+        tosubmit.append(request['filename|sha1'].split('|')[1])
+    elif request.get('filename|sha256'):
+        tosubmit.append(request['filename|sha256'].split('|')[1])
+    else:
+        return False
+
+    submitcache = False
+    submitcache_directory = False
+    import_related_hashes = False
+
+    r = {"results": []}
+
+    if request.get('config'):
+        if request['config'].get('cache').lower() == "yes":
+            submitcache = True
+            if len(request['config'].get('cache_directory').strip()) > 0:
+                if os.access(request['config'].get('cache_directory'), os.W_OK):
+                    submitcache_directory = request['config'].get('cache_directory')
+                else:
+                    ErrorMSG = "Cache directory is not writable. Please fix it before."
+                    log.debug(str(ErrorMSG))
+                    misperrors['error'] = ErrorMSG
+                    return misperrors
+            else:
+                ErrorMSG = "Value for Plugin.Enrichment_apiosintds_cache_directory is empty but cache option is enabled as recommended. Please set a writable cache directory in plugin settings."
+                log.debug(str(ErrorMSG))
+                misperrors['error'] = ErrorMSG
+                return misperrors
+        else:
+            log.debug("Cache option is set to "+request['config'].get('cache')+". You are not using the internal cache system and this is NOT recommended!")
+            log.debug("Please, consider to turn on the cache setting it to 'Yes' and specifing a writable directory for the cache directory option.")
+    try:
+        response = apiosintDS.request(entities=tosubmit, cache=submitcache, cachedirectory=submitcache_directory, verbose=True)
+        if request['config'].get('import_related_hashes').lower() == "yes":
+            import_related_hashes = True
+        r["results"] += reversed(apiosintParser(response, import_related_hashes))
+    except Exception as e:
+        log.debug(str(e))
+        misperrors['error'] = str(e)
+    return r
+
+def apiosintParser(response, import_related_hashes):
+    ret = []
+    if isinstance(response, dict):
+        for key in response:
+            for item in response[key]["items"]:
+                if item["response"]:
+                    comment = item["item"]+" IS listed by OSINT.digitalside.it. Date list: "+response[key]["list"]["date"]
+                    if key == "url":
+                        if "hashes" in item.keys():
+                            if "sha256" in item["hashes"].keys():
+                                ret.append({"types": ["sha256"], "values": [item["hashes"]["sha256"]]})
+                            if "sha1" in item["hashes"].keys():
+                                ret.append({"types": ["sha1"], "values": [item["hashes"]["sha1"]]})
+                            if "md5" in item["hashes"].keys():
+                                ret.append({"types": ["md5"], "values": [item["hashes"]["md5"]]})
+
+                    if len(item["related_urls"]) > 0:
+                        for urls in item["related_urls"]:
+                            if isinstance(urls, dict):
+                                itemToInclude = urls["url"]
+                                if import_related_hashes:
+                                    if "hashes" in urls.keys():
+                                        if "sha256" in urls["hashes"].keys():
+                                            ret.append({"types": ["sha256"], "values": [urls["hashes"]["sha256"]], "comment": "Related to: "+itemToInclude})
+                                        if "sha1" in urls["hashes"].keys():
+                                            ret.append({"types": ["sha1"], "values": [urls["hashes"]["sha1"]], "comment": "Related to: "+itemToInclude})
+                                        if "md5" in urls["hashes"].keys():
+                                            ret.append({"types": ["md5"], "values": [urls["hashes"]["md5"]], "comment": "Related to: "+itemToInclude})
+                                ret.append({"types": ["url"], "values": [itemToInclude], "comment": "Related to: "+item["item"]})
+                            else:
+                                ret.append({"types": ["url"], "values": [urls], "comment": "Related URL to: "+item["item"]})
+                else:
+                    comment = item["item"]+" IS NOT listed by OSINT.digitalside.it. Date list: "+response[key]["list"]["date"]
+                ret.append({"types": ["text"], "values": [comment]})
+    return ret
+
+def introspection():
+    return mispattributes
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo