MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'main' of github.com:MISP/misp-modules

pull/709/head
Christian Studer 2024-11-26 10:23:05 +01:00
parent 59c994678d 03165ad2cd
commit 328a85ca2f
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
11 changed files with 2532 additions and 2285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/test-package.yml vendored
View File

@ -37,7 +37,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:

2
misp_modules/modules/expansion/__init__.py
View File

@ -23,7 +23,7 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'circl_passivedns', 'circl_passivess
'extract_url_components', 'ipinfo', 'whoisfreaks', 'ip2locationio', 'stairwell',
'google_threat_intelligence', 'vulnerability_lookup', 'vysion', 'mcafee_insights_enrich',
'threatfox', 'yeti', 'abuseipdb', 'vmware_nsx', 'sigmf_expand', 'google_safe_browsing',
'google_search', 'whois', 'triage_submit', 'virustotal_upload', 'malshare_upload' ]
'google_search', 'whois', 'triage_submit', 'virustotal_upload', 'malshare_upload', 'convert_markdown_to_pdf' ]
minimum_required_fields = ('type', 'uuid', 'value')

179
misp_modules/modules/expansion/convert_markdown_to_pdf.py Executable file
View File

@ -0,0 +1,179 @@
#!/usr/bin/env python\
import json
import base64
import pandoc
import random
import string
import subprocess
import os
import shutil
installationNotes = '''
1. Install pandoc for your distribution
2. Install wkhtmltopdf
- Ensure You have install the version with patched qt
- Ensure it supports margin options
- You can check the above by inspecting the extended help `wkhtmltopdf --extended-help`
3. Install mermaid
- `npm install --global @mermaid-js/mermaid-cli`
4. Install the pandoc-mermaid-filter from https://github.com/DavidCruciani/pandoc-mermaid-filter
- Easiest is to install the following:
```bash
pip3 install git+https://github.com/DavidCruciani/pandoc-mermaid-filter
```
'''
misperrors = {'error': 'Error'}
mispattributes = {'input': ['text'], 'output': ['text']}
moduleinfo = {
'version': '0.3',
'author': 'Sami Mokaddem',
'description': 'Render the markdown (under GFM) into PDF. Requires pandoc (https://pandoc.org/), wkhtmltopdf (https://wkhtmltopdf.org/) and mermaid dependencies.',
'module-type': ['expansion'],
'name': 'Markdown to PDF converter',
'logo': '',
'requirements': ['pandoc'],
'features': '',
'references': [installationNotes],
'input': '',
'output': '',
}
moduleconfig = [
]
def randomFilename(length=10):
characters = string.ascii_lowercase + string.digits # Lowercase letters and digits
return ''.join(random.choices(characters, k=length))
def convert(markdown, margin='3'):
doc = pandoc.read(markdown, format='gfm')
elt = doc
# wrap/unwrap Inline or MetaInlines into [Inline]
if isinstance(elt, pandoc.types.Inline):
inline = elt
elt = [inline]
elif isinstance(elt, pandoc.types.MetaInlines):
meta_inlines = elt
elt = meta_inlines[0]
# wrap [Inline] into a Plain element
if isinstance(elt, list) and all(isinstance(elt_, pandoc.types.Inline) for elt_ in elt):
inlines = elt
elt = pandoc.types.Plain(inlines)
# wrap/unwrap Block or MetaBlocks into [Block]
if isinstance(elt, pandoc.types.Block):
block = elt
elt = [block]
elif isinstance(elt, pandoc.types.MetaBlocks):
meta_blocks = elt
elt = meta_blocks[0]
# wrap [Block] into a Pandoc element
if isinstance(elt, list) and all(isinstance(elt_, pandoc.types.Block) for elt_ in elt):
blocks = elt
elt = pandoc.types.Pandoc(pandoc.types.Meta({}), blocks)
if not isinstance(elt, pandoc.types.Pandoc):
raise TypeError(f"{elt!r} is not a Pandoc, Block or Inline instance.")
doc = elt
# options = [
# '--pdf-engine=wkhtmltopdf',
# f'-V margin-left={margin}',
# f'-V margin-right={margin}',
# f'-V margin-top={margin}',
# f'-V margin-bottom={margin}',
# '--pdf-engine-opt="--disable-smart-shrinking"',
# ]
randomFn = randomFilename()
command = [
"/usr/bin/pandoc",
"-t", "pdf",
"-o", f"/tmp/{randomFn}/output",
"--pdf-engine=wkhtmltopdf",
"-V", f"margin-left={margin}",
"-V", f"margin-right={margin}",
"-V", f"margin-top={margin}",
"-V", f"margin-bottom={margin}",
"--pdf-engine-opt=--disable-smart-shrinking",
"--filter=pandoc-mermaid",
"-f", "json",
f"/tmp/{randomFn}/input.js"
]
# try:
# # For some reasons, options are not passed correctly or not parsed correctly by wkhtmltopdf..
# # converted = pandoc.write(doc, format='pdf', options=options)
# except Exception as e:
# print(e)
os.makedirs(f'/tmp/{randomFn}', exist_ok=True)
# Write parsed file structure to be fed to the converter
with open(f'/tmp/{randomFn}/input.js', 'bw') as f:
configuration = pandoc.configure(read=True)
if pandoc.utils.version_key(configuration["pandoc_types_version"]) < [1, 17]:
json_ = pandoc.write_json_v1(doc)
else:
json_ = pandoc.write_json_v2(doc)
json_str = json.dumps(json_)
f.write(json_str.encode("utf-8"))
# Do conversion by manually invoking pandoc
try:
subprocess.run(command, check=True)
except subprocess.CalledProcessError as e:
print(f"Command failed with error: {e}")
# Read output and returns it
with open(f'/tmp/{randomFn}/output', 'br') as f:
converted = f.read()
# Clean up generated files
folderPath = f'/tmp/{randomFn}'
try:
shutil.rmtree(folderPath)
print(f"Folder '{folderPath}' deleted successfully.")
except FileNotFoundError:
print(f"Folder '{folderPath}' does not exist.")
except Exception as e:
print(f"Error deleting folder '{folderPath}': {e}")
return base64.b64encode(converted).decode()
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if request.get('text'):
data = request['text']
else:
return False
data = json.loads(data)
markdown = data.get('markdown')
try:
margin = '3'
if 'config' in request['config']:
if request['config'].get('margin'):
margin = request['config'].get('margin')
rendered = convert(markdown, margin=margin)
except Exception as e:
rendered = f'Error: {e}'
r = {'results': [{'types': mispattributes['output'],
'values':[rendered]}]}
return r
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

2
misp_modules/modules/expansion/dns.py
View File

@ -7,7 +7,7 @@ mispattributes = {'input': ['hostname', 'domain', 'domain|ip'], 'output': ['ip-s
moduleinfo = {
'version': '0.3',
'author': 'Alexandre Dulaunoy',
'description': 'jj',
'description': 'Simple DNS expansion service to resolve IP address from MISP attributes',
'module-type': ['expansion', 'hover'],
'name': 'DNS Resolver',
'logo': '',

73
misp_modules/modules/expansion/mmdb_lookup.py
View File

@ -5,30 +5,29 @@ from pymisp import MISPEvent, MISPObject
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'], 'format': 'misp_standard'}
moduleinfo = {
'version': '1',
'author': 'Jeroen Pinoy',
'description': "A hover and expansion module to enrich an ip with geolocation and ASN information from an mmdb server instance, such as CIRCL's ip.circl.lu.",
'module-type': ['expansion', 'hover'],
'name': 'GeoIP Enrichment',
'logo': 'circl.png',
'requirements': [],
'features': 'The module takes an IP address related attribute as input.\n It queries the public CIRCL.lu mmdb-server instance, available at ip.circl.lu, by default. The module can be configured with a custom mmdb server url if required.\n It is also possible to filter results on 1 db_source by configuring db_source_filter.',
'references': ['https://data.public.lu/fr/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/', 'https://github.com/adulau/mmdb-server'],
'input': 'An IP address attribute (for example ip-src or ip-src|port).',
'output': 'Geolocation and asn objects.',
}
moduleconfig = ["custom_API", "db_source_filter"]
moduleinfo = {'version': '1',
'author': 'Jeroen Pinoy',
'description': "A hover and expansion module to enrich an ip with geolocation and ASN information from an mmdb server instance, such as CIRCL's ip.circl.lu.",
'module-type': ['expansion', 'hover'],
'name': 'GeoIP Enrichment',
'logo': 'circl.png',
'requirements': [],
'features': 'The module takes an IP address related attribute as input.\n It queries the public CIRCL.lu mmdb-server instance, available at ip.circl.lu, by default. The module can be configured with a custom mmdb server url if required.\n It is also possible to filter results on 1 db_source by configuring db_source_filter.',
'references': ['https://data.public.lu/fr/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/', 'https://github.com/adulau/mmdb-server'],
'input': 'An IP address attribute (for example ip-src or ip-src|port).',
'output': 'Geolocation and asn objects.'}
moduleconfig = ["custom_API", "db_source_filter", "max_country_info_qt"]
mmdblookup_url = 'https://ip.circl.lu/'
class MmdbLookupParser():
def __init__(self, attribute, mmdblookupresult, api_url):
def __init__(self, attribute, mmdblookupresult, api_url, max_country_info_qt=0):
self.attribute = attribute
self.mmdblookupresult = mmdblookupresult
self.api_url = api_url
self.misp_event = MISPEvent()
self.misp_event.add_attribute(**attribute)
self.max_country_info_qt = int(max_country_info_qt)
def get_result(self):
event = json.loads(self.misp_event.to_json())
@ -37,26 +36,29 @@ class MmdbLookupParser():
def parse_mmdblookup_information(self):
# There is a chance some db's have a hit while others don't so we have to check if entry is empty each time
country_info_qt = 0
for result_entry in self.mmdblookupresult:
if result_entry['country_info']:
mmdblookup_object = MISPObject('geolocation')
mmdblookup_object.add_attribute('country',
**{'type': 'text', 'value': result_entry['country_info']['Country']})
mmdblookup_object.add_attribute('countrycode',
**{'type': 'text', 'value': result_entry['country']['iso_code']})
mmdblookup_object.add_attribute('latitude',
**{'type': 'float',
'value': result_entry['country_info']['Latitude (average)']})
mmdblookup_object.add_attribute('longitude',
**{'type': 'float',
'value': result_entry['country_info']['Longitude (average)']})
mmdblookup_object.add_attribute('text',
**{'type': 'text',
'value': 'db_source: {}. build_db: {}. Latitude and longitude are country average.'.format(
result_entry['meta']['db_source'],
result_entry['meta']['build_db'])})
mmdblookup_object.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(mmdblookup_object)
if (self.max_country_info_qt == 0) or (self.max_country_info_qt > 0 and country_info_qt < self.max_country_info_qt):
mmdblookup_object = MISPObject('geolocation')
mmdblookup_object.add_attribute('country',
**{'type': 'text', 'value': result_entry['country_info']['Country']})
mmdblookup_object.add_attribute('countrycode',
**{'type': 'text', 'value': result_entry['country']['iso_code']})
mmdblookup_object.add_attribute('latitude',
**{'type': 'float',
'value': result_entry['country_info']['Latitude (average)']})
mmdblookup_object.add_attribute('longitude',
**{'type': 'float',
'value': result_entry['country_info']['Longitude (average)']})
mmdblookup_object.add_attribute('text',
**{'type': 'text',
'value': 'db_source: {}. build_db: {}. Latitude and longitude are country average.'.format(
result_entry['meta']['db_source'],
result_entry['meta']['build_db'])})
mmdblookup_object.add_reference(self.attribute['uuid'], 'related-to')
self.misp_event.add_object(mmdblookup_object)
country_info_qt += 1
if 'AutonomousSystemNumber' in result_entry['country']:
mmdblookup_object_asn = MISPObject('asn')
mmdblookup_object_asn.add_attribute('asn',
@ -96,6 +98,9 @@ def handler(q=False):
else:
misperrors['error'] = 'There is no attribute of type ip-src or ip-dst provided as input'
return misperrors
max_country_info_qt = request['config'].get('max_country_info_qt', 0)
if max_country_info_qt is None:
max_country_info_qt = 0
api_url = check_url(request['config']['custom_API']) if 'config' in request and request['config'].get(
'custom_API') else mmdblookup_url
r = requests.get("{}/geolookup/{}".format(api_url, toquery))
@ -123,7 +128,7 @@ def handler(q=False):
else:
misperrors['error'] = 'API not accessible - http status code {} was returned'.format(r.status_code)
return misperrors
parser = MmdbLookupParser(attribute, mmdblookupresult, api_url)
parser = MmdbLookupParser(attribute, mmdblookupresult, api_url, max_country_info_qt)
parser.parse_mmdblookup_information()
result = parser.get_result()
return result

73
misp_modules/modules/expansion/module_misp_standard.py.skeleton Normal file
View File

@ -0,0 +1,73 @@
import json
from pymisp import MISPEvent
from . import check_input_attribute, standard_error_message
misperrors = {'error': 'Error'}
mispattributes = {
'input': [
# 'hostname',
# 'domain',
# 'ip-dst',
# 'url',
# Any other Attribute type...
],
'format': 'misp_standard'
}
moduleinfo = {
'version': '1',
'author': 'MISP',
'description': 'MISP module using the MISP standard skeleton',
'module-type': [ # possible module-types: 'expansion', 'hover' or both
'expansion',
'hover'
]
}
# config fields that your code expects from the site admin
moduleconfig = [
'config_name_1',
]
def DO_STUFF(misp_event, attribute):
return misp_event
def handler(q=False):
if q is False:
return False
request = json.loads(q)
# Input sanity check
if not request.get('attribute') or not check_input_attribute(request['attribute']):
return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
attribute = request['attribute']
# Make sure the Attribute's type is one of the expected type
if attribute['type'] not in mispattributes['input']:
return {'error': 'Unsupported attribute type.'}
# Use PyMISP to create compatible MISP Format
misp_event = MISPEvent()
DO_STUFF(misp_event, attribute)
# Convert to the format understood by MISP
results = {}
event = misp_event.to_dict()
for key in ('Attribute', 'Object', 'EventReport'):
if key in event:
results[key] = event[key]
return {'results': results}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

41
misp_modules/modules/expansion/urlscan.py
View File

@ -81,6 +81,8 @@ def lookup_indicator(client, query):
for request in result['data']['requests']:
if request['response'].get('failed'):
if request['response']['failed']['errorText']:
if request['response']['failed']['errorText'] in ["net::ERR_ABORTED", "net::ERR_FAILED", "net::ERR_QUIC_PROTOCOL_ERROR"]:
continue
log.debug('The page could not load')
r.append(
{'error': 'Domain could not be resolved: {}'.format(request['response']['failed']['errorText'])})
@ -91,14 +93,21 @@ def lookup_indicator(client, query):
r.append({'types': 'domain',
'categories': ['Network activity'],
'values': misp_val,
'comment': misp_comment})
'comment': f"{misp_comment} - Domain"})
if result['page'].get('ip'):
misp_val = result['page']['ip']
r.append({'types': 'ip-dst',
'categories': ['Network activity'],
'values': misp_val,
'comment': misp_comment})
'comment': f"{misp_comment} - IP"})
if result['page'].get('ptr'):
misp_val = result['page']['ptr']
r.append({'types': 'hostname',
'categories': ['Network activity'],
'values': misp_val,
'comment': f"{misp_comment} - PTR"})
if result['page'].get('country'):
misp_val = 'country: ' + result['page']['country']
@ -107,18 +116,40 @@ def lookup_indicator(client, query):
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': misp_comment})
'comment': f"{misp_comment} - Country/City"})
if result['page'].get('asn'):
misp_val = result['page']['asn']
r.append({'types': 'AS', 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment})
r.append({'types': 'AS', 'categories': ['External analysis'], 'values': misp_val, 'comment': f"{misp_comment} - ASN"})
if result['page'].get('asnname'):
misp_val = result['page']['asnname']
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': misp_comment})
'comment': f"{misp_comment} - ASN name"})
if result['page'].get('tlsIssuer'):
misp_val = result['page']['tlsIssuer']
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': f"{misp_comment} - TLS Issuer"})
if result['page'].get('title'):
misp_val = result['page']['title']
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': f"{misp_comment} - Page title"})
if result['page'].get('server'):
misp_val = result['page']['server']
r.append({'types': 'text',
'categories': ['External analysis'],
'values': misp_val,
'comment': f"{misp_comment} - Server"})
if result.get('stats'):
if result['stats'].get('malicious'):

4423
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

14
pyproject.toml
View File

@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "misp-modules"
version = "2.4.197"
version = "2.4.199"
description = "MISP modules are autonomous modules that can be used for expansion and other services in MISP"
authors = ["Alexandre Dulaunoy <alexandre.dulaunoy@circl.lu>"]
license = "AGPL-3.0-only"
@ -30,20 +30,20 @@ misp-modules = "misp_modules:main"
[tool.poetry.dependencies]
## platform (pin this to your python version, for 'poetry export' to work)
python = ">=3.8.*,<3.13"
python = ">=3.9.*,<3.13"
## core dependencies
psutil = "*"
pyparsing = "*"
redis = "*"
tornado = "*"
urllib3 = ">=1.26,<2"
## module dependencies (if a dependency fails loading with '*', pin it here)
censys = "2.0.9"
socialscan = "<2.0.0"
yara-python = "4.5.0"
# required to support both python 3.8 and wheel builds on python 3.12
numpy = [{version = "1.24.4", python = "3.8.*"}, {version = ">=1.26.4,<2.0.0", python = ">=3.9"}]
pandas = [{version = "1.5.3", python = "3.8.*"}, {version = ">=2.0.0", python = ">=3.9"}]
pandas_ods_reader = [{version = "0.1.4", python = "3.8.*"}, {version = ">=1.0.0", python = ">=3.9"}]
numpy = ">=1.26.4,<2.0.0"
pandas = ">=2.0.0"
pandas_ods_reader = ">=1.0.0"
## module dependencies
apiosintds = "*"
assemblyline_client = "*"
@ -66,6 +66,7 @@ np = "*"
oauth2 = "*"
opencv-python = "*"
openpyxl = "*"
pandoc = "*"
passivetotal = "*"
pdftotext = "*"
pycountry = "*"
@ -84,6 +85,7 @@ python-docx = "*"
python-pptx = "*"
pyzbar = "*"
requests = { version = "*", extras = ["security"] }
setuptools = "*"
shodan = "*"
sigmatools = "*"
sigmf = "*"

6
website/app/utils/utils.py
View File

@ -8,7 +8,11 @@ from pathlib import Path
import configparser
config = configparser.ConfigParser()
CONF_PATH = os.path.join(os.getcwd(), "conf", "config.cfg")
config.read(CONF_PATH)
if os.path.isfile(CONF_PATH):
config.read(CONF_PATH)
else:
print("[-] No conf file found. Copy config.cfg.sample to config.cfg")
exit()
MODULES = []

2
website/requirements.txt
View File

@ -6,7 +6,7 @@ Flask-WTF
Flask-Migrate
Flask-Login
WTForms
Werkzeug==3.0.3
Werkzeug==3.0.6
flask-restx
python-dateutil
schedule