fix: [url_import/url] added in __init__

Add crowdsec module

README.md

@@ -27,6 +27,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
 * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
 * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate(s) seen.
 * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
+* [CrowdSec](misp_modules/modules/expansion/crowdsec.py) - a hover module to expand using CrowdSec's CTI API.
 * [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API.
 * [CPE](misp_modules/modules/expansion/cpe.py) - An expansion module to query the CVE Search API with a cpe code, to get its related vulnerabilities.
 * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).

REQUIREMENTS

@@ -106,6 +106,7 @@ git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12
 pyeupi==1.1
 pyfaup==1.2
 pygeoip==0.3.2
+pycountry==22.3.5
 pygments==2.13.0 ; python_version >= '3.6'
 git+https://github.com/MISP/PyIntel471.git@917272fafa8e12102329faca52173e90c5256968#egg=pyintel471
 git+https://github.com/D4-project/IPASN-History.git/@a2853c39265cecdd0c0d16850bd34621c0551b87#egg=pyipasnhistory&subdirectory=client

misp_modules/modules/expansion/__init__.py

@@ -19,7 +19,7 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
            'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich',
            'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan', 'passive-ssh',
            'qintel_qsentry', 'mwdb', 'hashlookup', 'mmdb_lookup', 'ipqs_fraud_and_risk_scoring',
-           'clamav', 'jinja_template_rendering','hyasinsight', 'variotdbs']
+           'clamav', 'jinja_template_rendering','hyasinsight', 'variotdbs', 'crowdsec', 'extract_url_components']
 
 
 minimum_required_fields = ('type', 'uuid', 'value')

misp_modules/modules/expansion/crowdsec.py

@@ -0,0 +1,155 @@
+import json
+
+from pymisp import MISPEvent, MISPObject
+import pycountry
+import requests
+
+mispattributes = {"input": ["ip-dst", "ip-src"], "output": ["text"]}
+moduleinfo = {
+    "version": "1.0",
+    "author": "Shivam Sandbhor <shivam@crowdsec.net>",
+    "description": "Module to access CrowdSec CTI API.",
+    "module-type": ["hover"],
+}
+moduleconfig = ["api_key", "api_version"]
+
+
+def handler(q=False):
+    if q is False:
+        return False
+
+    request = json.loads(q)
+    if not request.get("config"):
+        return {"error": "Missing CrowdSec Config"}
+
+    if not request["config"].get("api_key"):
+        return {"error": "Missing CrowdSec API key"}
+
+    if not request["config"].get("api_version"):
+        return {"error": "Missing CrowdSec API version parameter"}
+
+    if request["config"]["api_version"] == "v2":
+        return _handler_v2(request)
+    return {"error": f'API version {request["config"]["api_version"]} not supported'}
+
+
+def _handler_v2(request_data):
+    if request_data.get("ip-dst"):
+        ip = request_data.get("ip-dst")
+    elif request_data.get("ip-src"):
+        ip = request_data.get("ip-src")
+
+    crowdsec_cti = requests.get(
+        f"https://cti.api.crowdsec.net/v2/smoke/{ip}",
+        headers={"x-api-key": request_data["config"]["api_key"]},
+    )
+    crowdsec_cti.raise_for_status()
+    crowdsec_cti = crowdsec_cti.json()
+
+    misp_event = MISPEvent()
+    crowdsec_context_object = MISPObject("crowdsec-ip-context")
+    crowdsec_context_object.add_attribute("IP Address", **{"type": "text", "value": ip})
+    crowdsec_context_object.add_attribute(
+        "IP Range", **{"type": "text", "value": crowdsec_cti["ip_range"]}
+    )
+    crowdsec_context_object.add_attribute(
+        "IP Range Score", **{"type": "text", "value": crowdsec_cti["ip_range_score"]}
+    )
+    crowdsec_context_object.add_attribute(
+        "Country",
+        **{
+            "type": "text",
+            "value": get_country_name_from_alpha_2(crowdsec_cti["location"]["country"]),
+        },
+    )
+    if crowdsec_cti["location"]["city"]:
+        crowdsec_context_object.add_attribute(
+            "City", **{"type": "text", "value": crowdsec_cti["location"]["city"]}
+        )
+
+    crowdsec_context_object.add_attribute(
+        "Latitude", **{"type": "float", "value": crowdsec_cti["location"]["latitude"]}
+    )
+    crowdsec_context_object.add_attribute(
+        "Longitude", **{"type": "float", "value": crowdsec_cti["location"]["longitude"]}
+    )
+
+    crowdsec_context_object.add_attribute(
+        "AS Name", **{"type": "text", "value": crowdsec_cti["as_name"]}
+    )
+
+    crowdsec_context_object.add_attribute(
+        "AS Number", **{"type": "AS", "value": crowdsec_cti["as_num"]}
+    )
+
+    crowdsec_context_object.add_attribute(
+        "Reverse DNS", **{"type": "domain", "value": crowdsec_cti["reverse_dns"]}
+    )
+
+    crowdsec_context_object.add_attribute(
+        "Attack Categories",
+        **{
+            "type": "text",
+            "value": ",".join(
+                [attack_category["label"] for attack_category in crowdsec_cti["behaviors"]]
+            ),
+        },
+    )
+
+    crowdsec_context_object.add_attribute(
+        "Triggered Scenarios",
+        **{
+            "type": "text",
+            "value": ",".join([scenario["name"] for scenario in crowdsec_cti["attack_details"]]),
+        },
+    )
+
+    crowdsec_context_object.add_attribute(
+        "Top 10 Target Countries",
+        **{
+            "type": "float",
+            "value": ",".join(
+                map(get_country_name_from_alpha_2, crowdsec_cti["target_countries"].keys())
+            ),
+        },
+    )
+
+    crowdsec_context_object.add_attribute(
+        "Trust", **{"type": "float", "value": crowdsec_cti["scores"]["overall"]["trust"]}
+    )
+
+    crowdsec_context_object.add_attribute(
+        "First Seen", **{"type": "datetime", "value": crowdsec_cti["history"]["first_seen"]}
+    )
+
+    crowdsec_context_object.add_attribute(
+        "Last Seen", **{"type": "datetime", "value": crowdsec_cti["history"]["last_seen"]}
+    )
+
+    for time_period, indicators in crowdsec_cti["scores"].items():
+        tp = " ".join(map(str.capitalize, time_period.split("_")))
+
+        for indicator_type, indicator_value in indicators.items():
+            crowdsec_context_object.add_attribute(
+                f"{tp} {indicator_type.capitalize()}", **{"type": "float", "value": indicator_value}
+            )
+
+    misp_event.add_object(crowdsec_context_object)
+
+    event = json.loads(misp_event.to_json())
+    results = {key: event[key] for key in ("Attribute", "Object") if (key in event and event[key])}
+    return {"results": results}
+
+
+def get_country_name_from_alpha_2(alpha_2):
+    country_info = pycountry.countries.get(alpha_2=alpha_2)
+    return country_info.name
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo["config"] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/extract_url_components.py

@@ -0,0 +1,70 @@
+import json
+from pymisp import MISPEvent, MISPObject
+from . import check_input_attribute, standard_error_message
+from pyfaup.faup import Faup
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['url'], 'format': 'misp_standard'}
+moduleinfo = {'version': '1', 'author': 'MISP Team',
+              'description': "Extract URL components",
+              'module-type': ['expansion', 'hover']}
+moduleconfig = []
+
+
+def createObjectFromURL(url):
+    f = Faup()
+    f.decode(url)
+    parsed = f.get()
+    obj = MISPObject('url')
+    obj.add_attribute('url', type='url', value=url)
+    if parsed['tld'] is not None:
+        obj.add_attribute('tld', type='text', value=parsed['tld'])
+    if parsed['subdomain'] is not None:
+        obj.add_attribute('subdomain', type='text', value=parsed['subdomain'])
+    obj.add_attribute('scheme', type='text', value=parsed['scheme'])
+    obj.add_attribute('resource_path', type='text', value=parsed['resource_path'])
+    obj.add_attribute('query_string', type='text', value=parsed['query_string'])
+    obj.add_attribute('port', type='port', value=parsed['port'])
+    obj.add_attribute('host', type='hostname', value=parsed['host'])
+    if parsed['fragment'] is not None:
+        obj.add_attribute('fragment', type='text', value=parsed['fragment'])
+    obj.add_attribute('domain_without_tld', type='text', value=parsed['domain_without_tld'])
+    obj.add_attribute('domain', type='domain', value=parsed['domain'])
+    return obj
+
+
+def createEvent(urlObject, attributeUUID, urlAttribute):
+    mispEvent = MISPEvent()
+    mispEvent.add_attribute(**urlAttribute)
+    urlObject.add_reference(attributeUUID, 'generated-from')
+    mispEvent.add_object(urlObject)
+    return mispEvent
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('attribute') or not check_input_attribute(request['attribute']):
+        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
+    attribute = request['attribute']
+    
+    if attribute['type'] not in mispattributes['input']:
+        return {'error': 'Bad attribute type'} 
+
+    url = attribute['value'] 
+    urlObject = createObjectFromURL(url)
+
+    event = createEvent(urlObject, attribute['uuid'], attribute)
+    event = json.loads(event.to_json())
+    
+    result = {'results': {'Object': event['Object']}}
+    return result
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/import_mod/__init__.py

@@ -15,5 +15,6 @@ __all__ = [
     'csvimport',
     'cof2misp',
     'joe_import',
-    'taxii21'
+    'taxii21',
+    'url_import'
 ]

misp_modules/modules/import_mod/import_blueprint.py

@@ -0,0 +1,84 @@
+import json
+import base64
+from pymisp import MISPEvent, MISPObject, MISPAttribute
+
+misperrors = {'error': 'Error'}
+userConfig = {
+    'number1': {
+        'type': 'Integer',
+        'regex': '/^[0-4]$/i',
+        'errorMessage': 'Expected a number in range [0-4]',
+        'message': 'Column number used for value'
+    },
+    'some_string': {
+        'type': 'String',
+        'message': 'A text field'
+    },
+    'boolean_field': {
+        'type': 'Boolean',
+        'message': 'Boolean field test'
+    },
+    'comment': {
+        'type': 'Integer',
+        'message': 'Column number used for comment'
+    }
+}
+
+mispattributes = {
+    'inputSource': ['file', 'paste'],
+    'output': ['MISP Format'],
+    'format': 'misp_standard'
+}
+
+
+moduleinfo = {'version': '0.1', 'author': 'Sami Mokaddem',
+              'description': 'Generic blueprint to be copy-pasted to quickly boostrap creation of import module.',
+              'module-type': ['import']}
+
+moduleconfig = []
+
+
+
+def generateData(event, data, config):
+    # attr = MISPAttribute()
+    # attr.from_dict(**{
+    #     'type': 'ip-src',
+    #     'value': '8.8.8.8',
+    #     'distribution': 2
+    # })
+    # event.add_attribute(attr)
+    pass
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    data = getUploadedData(request)
+    config = getPassedConfig(request)
+    event = MISPEvent()
+    generateData(event, data, config)
+    return {"results": json.loads(event.to_json())}
+
+
+def getUploadedData(request):
+    return base64.b64decode(request['data']).decode('utf8')
+
+
+def getPassedConfig(request):
+    return request['config']
+
+
+def introspection():
+    modulesetup = mispattributes
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/import_mod/url_import.py

@@ -0,0 +1,86 @@
+import json
+import base64
+from pymisp import MISPEvent, MISPObject, MISPAttribute
+from pyfaup.faup import Faup
+
+misperrors = {'error': 'Error'}
+userConfig = {
+    'include_scheme': {
+        'type': 'Boolean',
+        'message': 'Include scheme'
+    },
+}
+
+mispattributes = {
+    'inputSource': ['file', 'paste'],
+    'output': ['MISP Format'],
+    'format': 'misp_standard'
+}
+
+
+moduleinfo = {'version': '0.1', 'author': 'Sami Mokaddem',
+              'description': 'Generic blueprint to be copy-pasted to quickly boostrap creation of import module.',
+              'module-type': ['import']}
+
+moduleconfig = []
+
+fp = Faup()
+
+def generateData(event, data, config):
+    for url in data.splitlines():
+        fp.decode(url)
+        parsed = fp.get()
+        obj = MISPObject('url')
+        obj.add_attribute('url', type='url', value=url)
+        if parsed['tld'] is not None:
+            obj.add_attribute('tld', type='text', value=parsed['tld'])
+        if parsed['subdomain'] is not None:
+            obj.add_attribute('subdomain', type='text', value=parsed['subdomain'])
+        if config['include_scheme'] is True:
+            obj.add_attribute('scheme', type='text', value=parsed['scheme'])
+        obj.add_attribute('resource_path', type='text', value=parsed['resource_path'])
+        obj.add_attribute('query_string', type='text', value=parsed['query_string'])
+        obj.add_attribute('port', type='port', value=parsed['port'])
+        obj.add_attribute('host', type='hostname', value=parsed['host'])
+        if parsed['fragment'] is not None:
+            obj.add_attribute('fragment', type='text', value=parsed['fragment'])
+        obj.add_attribute('domain_without_tld', type='text', value=parsed['domain_without_tld'])
+        obj.add_attribute('domain', type='domain', value=parsed['domain'])
+        event.objects.append(obj)
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    data = getUploadedData(request)
+    config = getPassedConfig(request)
+    event = MISPEvent()
+    generateData(event, data, config)
+    return {"results": json.loads(event.to_json())}
+
+
+def getUploadedData(request):
+    return base64.b64decode(request['data']).decode('utf8')
+
+
+def getPassedConfig(request):
+    for k, v in userConfig.items():
+        if v['type'] == 'Boolean':
+            request['config'][k] = True if request['config'][k] == '1' else False
+    return request['config']
+
+
+def introspection():
+    modulesetup = mispattributes
+    try:
+        userConfig
+        modulesetup['userConfig'] = userConfig
+    except NameError:
+        pass
+    return modulesetup
+
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo