diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a444958..510a469 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -46,5 +46,7 @@ jobs: # Run server in background misp-modules -l 127.0.0.1 -s & sleep 5 + # Check if modules are running + curl -sS localhost:6666/modules # Run tests pytest tests diff --git a/Pipfile b/Pipfile index 85226be..bdf2c98 100644 --- a/Pipfile +++ b/Pipfile @@ -17,9 +17,9 @@ passivetotal = "*" pypdns = "*" pypssl = "*" pyeupi = "*" -pymisp = { extras = ["fileobjects,openioc,pdfexport,email"], version = "*" } -pyonyphe = { editable = true, git = "https://github.com/sebdraven/pyonyphe" } -pydnstrails = { editable = true, git = "https://github.com/sebdraven/pydnstrails" } +pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" } +pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" } +pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" } pytesseract = "*" pygeoip = "*" beautifulsoup4 = "*" @@ -31,20 +31,20 @@ maclookup = "*" vulners = "*" blockchain = "*" reportlab = "*" -pyintel471 = { editable = true, git = "https://github.com/MISP/PyIntel471.git" } +pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" } shodan = "*" Pillow = ">=8.2.0" Wand = "*" SPARQLWrapper = "*" domaintools_api = "*" -misp-modules = { editable = true, path = "." } -pybgpranking = { editable = true, git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client" } -pyipasnhistory = { editable = true, git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client" } +misp-modules = { path = "." } +pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" } +pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" } backscatter = "*" pyzbar = "*" opencv-python = "*" np = "*" -ODTReader = { editable = true, git = "https://github.com/cartertemm/ODTReader.git/" } +ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" } python-pptx = "*" python-docx = "*" ezodf = "*" @@ -59,7 +59,7 @@ geoip2 = "*" apiosintDS = "*" assemblyline_client = "*" vt-graph-api = "*" -trustar = { editable = true, git = "https://github.com/SteveClement/trustar-python.git" } +trustar = { git = "https://github.com/SteveClement/trustar-python.git" } markdownify = "==0.5.3" socialscan = "*" dnsdb2 = "*" @@ -67,6 +67,10 @@ clamd = "*" aiohttp = ">=3.7.4" tau-clients = "*" vt-py = ">=0.7.1" +crowdstrike-falconpy = "0.9.0" +censys = "2.0.9" +mwdblib = "3.4.1" +ndjson = "0.3.1" [requires] -python_version = "3.6" +python_version = "3.7" diff --git a/README.md b/README.md index 6b96be2..e1e5a0f 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. * [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload. * [McAfee MVISION Insights](misp_modules/modules/expansion/mcafee_insights_enrich.py) - an expansion module enrich IOCs with McAfee MVISION Insights. +* [Mmdb server lookup](misp_modules/modules/expansion/mmdb_lookup.py) - an expansion module to enrich an ip with geolocation information from an mmdb server such as ip.circl.lu. * [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP. * [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). * [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). diff --git a/REQUIREMENTS b/REQUIREMENTS index 4c76aba..5d4e50e 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -6,152 +6,165 @@ # -i https://pypi.org/simple --e . --e git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client --e git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client --e git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471 --e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader --e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails --e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe -git+https://github.com/SteveClement/trustar-python.git -aiohttp==3.7.4 +. +aiohttp==3.8.1 +aiosignal==1.2.0; python_version >= '3.6' antlr4-python3-runtime==4.8; python_version >= '3' apiosintds==1.8.3 +appdirs==1.4.4 argparse==1.4.0 -assemblyline-client==4.1.0 -async-timeout==3.0.1; python_full_version >= '3.5.3' -attrs==21.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +assemblyline-client==4.2.2 +async-timeout==4.0.2; python_version >= '3.6' +asynctest==0.13.0; python_version < '3.8' +attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +backoff==1.11.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' backports.zoneinfo==0.2.1; python_version < '3.9' backscatter==0.2.4 -beautifulsoup4==4.9.3 -bidict==0.21.2; python_version >= '3.6' +beautifulsoup4==4.10.0 +bidict==0.21.4; python_version >= '3.6' blockchain==1.4.4 -certifi==2021.5.30 -censys==2.0.9 -cffi==1.14.6 -#chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -chardet -charset-normalizer==2.0.4; python_version >= '3' +censys==2.1.2 +certifi==2021.10.8 +cffi==1.15.0 +chardet==4.0.0 +charset-normalizer==2.0.11; python_version >= '3' clamd==1.0.2 click-plugins==1.1.1 -click==8.0.1; python_version >= '3.6' +click==8.0.3; python_version >= '3.6' colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -colorclass==2.2.0 +colorclass==2.2.2; python_version >= '2.6' +commonmark==0.9.1 compressed-rtf==1.0.6 -configparser==5.0.2; python_version >= '3.6' -crowdstrike-falconpy==0.9.0 -cryptography==3.4.7; python_version >= '3.6' -decorator==5.0.9; python_version >= '3.5' -deprecated==1.2.12; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +configparser==5.2.0; python_version >= '3.6' +crowdstrike-falconpy==1.0.0 +cryptography==36.0.1; python_version >= '3.6' +decorator==5.1.1; python_version >= '3.5' +deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' dnsdb2==1.1.3 -dnspython==2.1.0 -domaintools-api==0.5.4 +dnspython==2.2.0 +domaintools-api==0.6.1 easygui==0.98.2 ebcdic==1.1.1 enum-compat==0.0.3 extract-msg==0.28.7 -ez-setup==0.9 ezodf==0.3.2 -filelock==3.0.12 +filelock==3.4.2; python_version >= '3.7' +frozenlist==1.3.0; python_version >= '3.7' future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' -futures==3.1.1 -geoip2==4.2.0 -httplib2==0.19.1 +geoip2==4.5.0 +git+https://github.com/D4-project/BGP-Ranking.git/@68de39f6c5196f796055c1ac34504054d688aa59#egg=pybgpranking&subdirectory=client +git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client +git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471 +git+https://github.com/SteveClement/trustar-python.git@6954eae38e0c77eaeef26084b6c5fd033925c1c7#egg=trustar +git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader +git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails +git+https://github.com/sebdraven/pyonyphe@aed008ee5a27e3a5e4afbb3e5cbfc47170108452#egg=pyonyphe +httplib2==0.20.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' idna-ssl==1.1.0; python_version < '3.7' -idna==3.2; python_version >= '3' +idna==3.3; python_version >= '3' imapclient==2.1.0 -isodate==0.6.0 +importlib-metadata==4.10.1; python_version < '3.8' +isodate==0.6.1 itsdangerous==2.0.1; python_version >= '3.6' jbxapi==3.17.2 -json-log-formatter==0.4.0 +jeepney==0.7.1; sys_platform == 'linux' +json-log-formatter==0.5.1 jsonschema==3.2.0 -lark-parser==0.11.3 +keyring==23.5.0; python_version >= '3.7' +lark-parser==0.12.0 lief==0.11.5 lxml==4.7.1 maclookup==1.0.3 markdownify==0.5.3 -maxminddb==2.0.3; python_version >= '3.6' -more-itertools==8.8.0; python_version >= '3.5' -msoffcrypto-tool==4.12.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin') -multidict==5.1.0; python_version >= '3.6' +maxminddb==2.2.0; python_version >= '3.6' +more-itertools==8.12.0; python_version >= '3.5' +msoffcrypto-tool==5.0.0; python_version >= '3' and platform_python_implementation != 'PyPy' or (platform_system != 'Windows' and platform_system != 'Darwin') +multidict==6.0.2; python_version >= '3.7' +mwdblib==4.0.0 +ndjson==0.3.1 np==1.0.2 -numpy==1.21.2; python_version < '3.11' and python_version >= '3.7' +numpy==1.21.5; python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64' oauth2==1.9.0.post1 olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -oletools==0.56.2 -opencv-python==4.5.3.56 +oletools==0.60 +opencv-python==4.5.5.62 +packaging==21.3; python_version >= '3.6' pandas-ods-reader==0.1.2 pandas==1.3.5 -passivetotal==2.5.4 +passivetotal==2.5.8 pcodedmp==1.2.6 -pdftotext==2.2.0 -pillow==8.3.2 -progressbar2==3.53.1 -psutil==5.8.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' -pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -pycryptodome==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -pycryptodomex==3.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +pdftotext==2.2.2 +pillow==9.0.1 +progressbar2==4.0.0; python_version >= '3.7' +psutil==5.9.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' +pycparser==2.21 +pycryptodome==3.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +pycryptodomex==3.14.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pydeep==0.4 pyeupi==1.1 +pyfaup==1.2 pygeoip==0.3.2 -pymisp[email,fileobjects,openioc,pdfexport]==2.4.148 +pygments==2.11.2; python_version >= '3.5' +pymisp[email,fileobjects,openioc,pdfexport,url]==2.4.152 pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' pypdns==1.5.2 pypssl==2.2 -pyrsistent==0.18.0; python_version >= '3.6' +pyrsistent==0.18.1; python_version >= '3.7' pytesseract==0.3.8 python-baseconv==1.2.2 python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-docx==0.8.11 -python-engineio==4.2.1; python_version >= '3.6' -python-magic==0.4.24 -python-pptx==0.6.19 -python-socketio[client]==5.4.0; python_version >= '3.6' -python-utils==2.5.6 +python-engineio==4.3.1; python_version >= '3.6' +python-magic==0.4.25 +python-pptx==0.6.21 +python-socketio[client]==5.5.1; python_version >= '3.6' +python-utils==3.1.0; python_version >= '3.7' +pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' pytz==2019.3 -pyyaml==5.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +pyyaml==6.0; python_version >= '3.6' pyzbar==0.1.8 pyzipper==0.3.5; python_version >= '3.5' -rdflib==6.0.0; python_version >= '3.7' -redis==3.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -reportlab==3.6.1 +rdflib==6.1.1; python_version >= '3.7' +redis==4.1.2; python_version >= '3.6' +reportlab==3.6.6 requests-cache==0.6.4; python_version >= '3.6' requests-file==1.5.1 -requests[security]==2.26.0 +requests[security]==2.27.1 +rich==11.1.0; python_full_version >= '3.6.2' and python_full_version < '4.0.0' rtfde==0.0.2 -ruamel.yaml.clib==0.2.6; python_version < '3.10' and platform_python_implementation == 'CPython' -ruamel.yaml==0.17.13; python_version >= '3' -shodan==1.25.0 +secretstorage==3.3.1; sys_platform == 'linux' +setuptools==60.7.1; python_version >= '3.7' +shodan==1.26.1 sigmatools==0.19.1 six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' socialscan==1.4.2 socketio-client==0.5.7.4 -soupsieve==2.2.1; python_version >= '3' +soupsieve==2.3.1; python_version >= '3.6' sparqlwrapper==1.8.5 stix2==3.0.1 stix2-patterns==1.3.2 tabulate==0.8.9 -tau-clients==0.1.3 +tau-clients==0.1.9 taxii2-client==2.3.0 -tldextract==3.1.0; python_version >= '3.5' +tldextract==3.1.2; python_version >= '3.6' tornado==6.1; python_version >= '3.5' -tqdm==4.62.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -typing-extensions==3.10.0.0 -tzlocal==3.0; python_version >= '3.6' +tqdm==4.62.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +typing-extensions==4.0.1; python_version < '3.8' +tzdata==2021.5; python_version >= '3.6' +tzlocal==4.1; python_version >= '3.6' unicodecsv==0.14.1 url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' urlarchiver==0.2 -urllib3==1.26.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' +urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_full_version < '4.0.0' validators==0.14.0 -vt-graph-api==1.1.2 -vt-py==0.7.2 -vulners==1.5.12 +vt-graph-api==1.1.3 +vt-py==0.13.1 +vulners==2.0.0 wand==0.6.7 -websocket-client==1.2.1 -wrapt==1.12.1 +websocket-client==1.2.3; python_version >= '3.6' +wrapt==1.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' xlrd==2.0.1 -xlsxwriter==3.0.1; python_version >= '3.4' +xlsxwriter==3.0.2; python_version >= '3.4' yara-python==3.8.1 -yarl==1.6.3; python_version >= '3.6' -ndjson==0.3.1 -mwdblib==3.4.1 +yarl==1.7.2; python_version >= '3.6' +zipp==3.7.0; python_version >= '3.7' diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index d20fe34..63ae8e3 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -18,7 +18,7 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar', 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich', 'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh', - 'qintel_qsentry', 'mwdb', 'hashlookup'] + 'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup'] minimum_required_fields = ('type', 'uuid', 'value') diff --git a/misp_modules/modules/expansion/apivoid.py b/misp_modules/modules/expansion/apivoid.py index a71b5e6..3b0ce72 100755 --- a/misp_modules/modules/expansion/apivoid.py +++ b/misp_modules/modules/expansion/apivoid.py @@ -4,8 +4,8 @@ from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} -mispattributes = {'input': ['domain', 'hostname'], 'format': 'misp_standard'} -moduleinfo = {'version': '0.1', 'author': 'Christian Studer', +mispattributes = {'input': ['domain', 'hostname', 'email'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', 'description': 'On demand query API for APIVoid.', 'module-type': ['expansion', 'hover']} moduleconfig = ['apikey'] @@ -43,6 +43,31 @@ class APIVoidParser(): ssl = requests.get(f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}').json() self._parse_ssl_certificate(ssl['data']['certificate']) + def handle_email(self, apikey): + feature = 'emailverify' + if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.06: + self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'} + return + emaillookup = requests.get(f'{self.url.format(feature, apikey)}email={self.attribute.value}').json() + email_verification = MISPObject('apivoid-email-verification') + boolean_attributes = ['valid_format', 'suspicious_username', 'suspicious_email', 'dirty_words_username', + 'suspicious_email', 'valid_tld', 'disposable', 'has_a_records', 'has_mx_records', + 'has_spf_records', 'is_spoofable', 'dmarc_configured', 'dmarc_enforced', 'free_email', + 'russian_free_email', 'china_free_email', 'suspicious_domain', 'dirty_words_domain', + 'domain_popular', 'risky_tld', 'police_domain', 'government_domain', 'educational_domain', + 'should_block'] + for boolean_attribute in boolean_attributes: + email_verification.add_attribute(boolean_attribute, + **{'type': 'boolean', 'value': emaillookup['data'][boolean_attribute]}) + email_verification.add_attribute('email', **{'type': 'email', 'value': emaillookup['data']['email']}) + email_verification.add_attribute('username', **{'type': 'text', 'value': emaillookup['data']['username']}) + email_verification.add_attribute('role_address', + **{'type': 'boolean', 'value': emaillookup['data']['role_address']}) + email_verification.add_attribute('domain', **{'type': 'domain', 'value': emaillookup['data']['domain']}) + email_verification.add_attribute('score', **{'type': 'float', 'value': emaillookup['data']['score']}) + email_verification.add_reference(self.attribute['uuid'], 'related-to') + self.misp_event.add_object(email_verification) + def _handle_dns_record(self, item, record_type, relationship): dns_record = MISPObject('dns-record') dns_record.add_attribute('queried-domain', type='domain', value=item['host']) @@ -82,7 +107,10 @@ def handler(q=False): return {'error': 'Unsupported attribute type.'} apikey = request['config']['apikey'] apivoid_parser = APIVoidParser(attribute) - apivoid_parser.parse_domain(apikey) + if attribute['type'] in ['domain', 'hostname']: + apivoid_parser.parse_domain(apikey) + else: + apivoid_parser.handle_email(apikey) return apivoid_parser.get_results() diff --git a/misp_modules/modules/expansion/mmdb_lookup.py b/misp_modules/modules/expansion/mmdb_lookup.py new file mode 100644 index 0000000..0c54ba8 --- /dev/null +++ b/misp_modules/modules/expansion/mmdb_lookup.py @@ -0,0 +1,129 @@ +import json +import requests +from . import check_input_attribute, standard_error_message +from pymisp import MISPEvent, MISPObject + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'], 'format': 'misp_standard'} +moduleinfo = {'version': '1', 'author': 'Jeroen Pinoy', + 'description': "An expansion module to enrich an ip with geolocation information from an mmdb server " + "such as ip.circl.lu.", + 'module-type': ['expansion', 'hover']} +moduleconfig = ["custom_API", "db_source_filter"] +mmdblookup_url = 'https://ip.circl.lu/' + + +class MmdbLookupParser(): + def __init__(self, attribute, mmdblookupresult, api_url): + self.attribute = attribute + self.mmdblookupresult = mmdblookupresult + self.api_url = api_url + self.misp_event = MISPEvent() + self.misp_event.add_attribute(**attribute) + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + def parse_mmdblookup_information(self): + # There is a chance some db's have a hit while others don't so we have to check if entry is empty each time + for result_entry in self.mmdblookupresult: + if result_entry['country_info']: + mmdblookup_object = MISPObject('geolocation') + mmdblookup_object.add_attribute('country', + **{'type': 'text', 'value': result_entry['country_info']['Country']}) + mmdblookup_object.add_attribute('countrycode', + **{'type': 'text', 'value': result_entry['country']['iso_code']}) + mmdblookup_object.add_attribute('latitude', + **{'type': 'float', + 'value': result_entry['country_info']['Latitude (average)']}) + mmdblookup_object.add_attribute('longitude', + **{'type': 'float', + 'value': result_entry['country_info']['Longitude (average)']}) + mmdblookup_object.add_attribute('text', + **{'type': 'text', + 'value': 'db_source: {}. build_db: {}. Latitude and longitude are country average.'.format( + result_entry['meta']['db_source'], + result_entry['meta']['build_db'])}) + mmdblookup_object.add_reference(self.attribute['uuid'], 'related-to') + self.misp_event.add_object(mmdblookup_object) + if 'AutonomousSystemNumber' in result_entry['country']: + mmdblookup_object_asn = MISPObject('asn') + mmdblookup_object_asn.add_attribute('asn', + **{'type': 'text', + 'value': result_entry['country'][ + 'AutonomousSystemNumber']}) + mmdblookup_object_asn.add_attribute('description', + **{'type': 'text', + 'value': 'ASNOrganization: {}. db_source: {}. build_db: {}.'.format( + result_entry['country'][ + 'AutonomousSystemOrganization'], + result_entry['meta']['db_source'], + result_entry['meta']['build_db'])}) + mmdblookup_object_asn.add_reference(self.attribute['uuid'], 'related-to') + self.misp_event.add_object(mmdblookup_object_asn) + + +def check_url(url): + return "{}/".format(url) if not url.endswith('/') else url + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + attribute = request['attribute'] + if attribute.get('type') == 'ip-src': + toquery = attribute['value'] + elif attribute.get('type') == 'ip-src|port': + toquery = attribute['value'].split('|')[0] + elif attribute.get('type') == 'ip-dst': + toquery = attribute['value'] + elif attribute.get('type') == 'ip-dst|port': + toquery = attribute['value'].split('|')[0] + else: + misperrors['error'] = 'There is no attribute of type ip-src or ip-dst provided as input' + return misperrors + api_url = check_url(request['config']['custom_API']) if 'config' in request and request['config'].get( + 'custom_API') else mmdblookup_url + r = requests.get("{}/geolookup/{}".format(api_url, toquery)) + if r.status_code == 200: + mmdblookupresult = r.json() + if not mmdblookupresult or len(mmdblookupresult) == 0: + misperrors['error'] = 'Empty result returned by server' + return misperrors + if 'config' in request and request['config'].get('db_source_filter'): + db_source_filter = request['config'].get('db_source_filter') + mmdblookupresult = [entry for entry in mmdblookupresult if entry['meta']['db_source'] == db_source_filter] + if not mmdblookupresult or len(mmdblookupresult) == 0: + misperrors['error'] = 'There was no result with the selected db_source' + return misperrors + # Server might return one or multiple entries which could all be empty, we check if there is at least one + # non-empty result below + empty_result = True + for lookup_result_entry in mmdblookupresult: + if lookup_result_entry['country_info']: + empty_result = False + break + if empty_result: + misperrors['error'] = 'Empty result returned by server' + return misperrors + else: + misperrors['error'] = 'API not accessible - http status code {} was returned'.format(r.status_code) + return misperrors + parser = MmdbLookupParser(attribute, mmdblookupresult, api_url) + parser.parse_mmdblookup_information() + result = parser.get_result() + return result + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo