misp-modules/misp_modules/modules/expansion/apiosintds.py

import json
import logging
import sys
import os
from apiosintDS import apiosintDS

log = logging.getLogger('apiosintDS')
log.setLevel(logging.DEBUG)
apiodbg = logging.StreamHandler(sys.stdout)
apiodbg.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
apiodbg.setFormatter(formatter)
log.addHandler(apiodbg)

misperrors = {'error': 'Error'}

mispattributes = {'input': ["domain", "domain|ip", "hostname", "ip-dst", "ip-src", "ip-dst|port", "ip-src|port", "url",
                            "md5", "sha1", "sha256", "filename|md5", "filename|sha1", "filename|sha256"],
                  'output': ["domain", "ip-dst", "url", "comment", "md5", "sha1", "sha256"]
                  }

moduleinfo = {'version': '0.1', 'author': 'Davide Baglieri aka davidonzo',
              'description': 'On demand query API for OSINT.digitalside.it project.',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['import_related_hashes', 'cache', 'cache_directory']


def handler(q=False):
    if q is False:
        return False
    request = json.loads(q)
    tosubmit = []
    if request.get('domain'):
        tosubmit.append(request['domain'])
    elif request.get('domain|ip'):
        tosubmit.append(request['domain|ip'].split('|')[0])
        tosubmit.append(request['domain|ip'].split('|')[1])
    elif request.get('hostname'):
        tosubmit.append(request['hostname'])
    elif request.get('ip-dst'):
        tosubmit.append(request['ip-dst'])
    elif request.get('ip-src'):
        tosubmit.append(request['ip-src'])
    elif request.get('ip-dst|port'):
        tosubmit.append(request['ip-dst|port'].split('|')[0])
    elif request.get('ip-src|port'):
        tosubmit.append(request['ip-src|port'].split('|')[0])
    elif request.get('url'):
        tosubmit.append(request['url'])
    elif request.get('md5'):
        tosubmit.append(request['md5'])
    elif request.get('sha1'):
        tosubmit.append(request['sha1'])
    elif request.get('sha256'):
        tosubmit.append(request['sha256'])
    elif request.get('filename|md5'):
        tosubmit.append(request['filename|md5'].split('|')[1])
    elif request.get('filename|sha1'):
        tosubmit.append(request['filename|sha1'].split('|')[1])
    elif request.get('filename|sha256'):
        tosubmit.append(request['filename|sha256'].split('|')[1])
    else:
        return False

    submitcache = False
    submitcache_directory = False
    import_related_hashes = False

    r = {"results": []}

    if request.get('config'):
        if request['config'].get('cache') and request['config']['cache'].lower() == "yes":
            submitcache = True
        if request['config'].get('import_related_hashes') and request['config']['import_related_hashes'].lower() == "yes":
            import_related_hashes = True
        if submitcache:
            cache_directory = request['config'].get('cache_directory')
            if cache_directory and len(cache_directory) > 0:
                if os.access(cache_directory, os.W_OK):
                    submitcache_directory = cache_directory
                else:
                    ErrorMSG = "Cache directory is not writable. Please fix it before."
                    log.debug(str(ErrorMSG))
                    misperrors['error'] = ErrorMSG
                    return misperrors
            else:
                ErrorMSG = "Value for Plugin.Enrichment_apiosintds_cache_directory is empty but cache option is enabled as recommended. Please set a writable cache directory in plugin settings."
                log.debug(str(ErrorMSG))
                misperrors['error'] = ErrorMSG
                return misperrors
        else:
            log.debug("Cache option is set to " + str(submitcache) + ". You are not using the internal cache system and this is NOT recommended!")
            log.debug("Please, consider to turn on the cache setting it to 'Yes' and specifing a writable directory for the cache directory option.")
    try:
        response = apiosintDS.request(entities=tosubmit, cache=submitcache, cachedirectory=submitcache_directory, verbose=True)
        r["results"] += reversed(apiosintParser(response, import_related_hashes))
    except Exception as e:
        log.debug(str(e))
        misperrors['error'] = str(e)
    return r


def apiosintParser(response, import_related_hashes):
    ret = []
    if isinstance(response, dict):
        for key in response:
            for item in response[key]["items"]:
                if item["response"]:
                    comment = item["item"]+" IS listed by OSINT.digitalside.it. Date list: "+response[key]["list"]["date"]
                    if key == "url":
                        if "hashes" in item.keys():
                            if "sha256" in item["hashes"].keys():
                                ret.append({"types": ["sha256"], "values": [item["hashes"]["sha256"]]})
                            if "sha1" in item["hashes"].keys():
                                ret.append({"types": ["sha1"], "values": [item["hashes"]["sha1"]]})
                            if "md5" in item["hashes"].keys():
                                ret.append({"types": ["md5"], "values": [item["hashes"]["md5"]]})

                    if len(item["related_urls"]) > 0:
                        for urls in item["related_urls"]:
                            if isinstance(urls, dict):
                                itemToInclude = urls["url"]
                                if import_related_hashes:
                                    if "hashes" in urls.keys():
                                        if "sha256" in urls["hashes"].keys():
                                            ret.append({"types": ["sha256"], "values": [urls["hashes"]["sha256"]], "comment": "Related to: "+itemToInclude})
                                        if "sha1" in urls["hashes"].keys():
                                            ret.append({"types": ["sha1"], "values": [urls["hashes"]["sha1"]], "comment": "Related to: "+itemToInclude})
                                        if "md5" in urls["hashes"].keys():
                                            ret.append({"types": ["md5"], "values": [urls["hashes"]["md5"]], "comment": "Related to: "+itemToInclude})
                                ret.append({"types": ["url"], "values": [itemToInclude], "comment": "Related to: "+item["item"]})
                            else:
                                ret.append({"types": ["url"], "values": [urls], "comment": "Related URL to: "+item["item"]})
                else:
                    comment = item["item"]+" IS NOT listed by OSINT.digitalside.it. Date list: "+response[key]["list"]["date"]
                ret.append({"types": ["text"], "values": [comment]})
    return ret


def introspection():
    return mispattributes


def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo