Merge branch 'master' of github.com:MISP/misp-modules

doc/README.md

@@ -613,6 +613,7 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re
 Query Lastline with an analysis link and parse the report into MISP attributes and objects.
 The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module.
 - **features**:
+>The module requires a Lastline Portal `username` and `password`.
 >The module uses the new format and it is able to return MISP attributes and objects.
 >The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module.
 - **input**:
@@ -630,7 +631,7 @@ The analysis link can also be retrieved from the output of the [lastline_submit]
 
 Module to submit a file or URL to Lastline.
 - **features**:
->The module requires a Lastline API key and token (or username and password).
+>The module requires a Lastline Analysis `api_token` and `key`.
 >When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module.
 - **input**:
 >File or URL to submit to Lastline.
@@ -1701,6 +1702,7 @@ A module to import data from a Joe Sandbox analysis json report.
 
 Module to import and parse reports from Lastline analysis links.
 - **features**:
+>The module requires a Lastline Portal `username` and `password`.
 >The module uses the new format and it is able to return MISP attributes and objects.
 >The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module.
 - **input**:

doc/expansion/lastline_query.json

@@ -5,5 +5,5 @@
     "input": "Link to a Lastline analysis.",
     "output": "MISP attributes and objects parsed from the analysis report.",
     "references": ["https://www.lastline.com"],
-    "features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module."
+    "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module."
 }

doc/expansion/lastline_submit.json

@@ -5,5 +5,5 @@
     "input": "File or URL to submit to Lastline.",
     "output": "Link to the report generated by Lastline.",
     "references": ["https://www.lastline.com"],
-    "features": "The module requires a Lastline API key and token (or username and password).\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module."
+    "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module."
 }

doc/import_mod/lastline_import.json

@@ -5,5 +5,5 @@
     "input": "Link to a Lastline analysis.",
     "output": "MISP attributes and objects parsed from the analysis report.",
     "references": ["https://www.lastline.com"],
-    "features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module."
+    "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module."
 }

misp_modules/modules/expansion/lastline_query.py

@@ -27,8 +27,6 @@ moduleinfo = {
 }
 
 moduleconfig = [
-    "api_key",
-    "api_token",
     "username",
     "password",
 ]
@@ -51,24 +49,25 @@ def handler(q=False):
 
     # Parse the init parameters
     try:
-        auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request)
+        config = request["config"]
+        auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
         analysis_link = request['attribute']['value']
         # The API url changes based on the analysis link host name
-        api_url = lastline_api.get_api_url_from_link(analysis_link)
+        api_url = lastline_api.get_portal_url_from_task_link(analysis_link)
     except Exception as e:
         misperrors["error"] = "Error parsing configuration: {}".format(e)
         return misperrors
 
     # Parse the call parameters
     try:
-        task_uuid = lastline_api.get_uuid_from_link(analysis_link)
+        task_uuid = lastline_api.get_uuid_from_task_link(analysis_link)
     except (KeyError, ValueError) as e:
         misperrors["error"] = "Error processing input parameters: {}".format(e)
         return misperrors
 
     # Make the API calls
     try:
-        api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data)
+        api_client = lastline_api.PortalClient(api_url, auth_data)
         response = api_client.get_progress(task_uuid)
         if response.get("completed") != 1:
             raise ValueError("Analysis is not finished yet.")
@@ -108,7 +107,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
     c = configparser.ConfigParser()
     c.read(args.config_file)
-    a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name)
+    a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
 
     j = json.dumps(
         {

misp_modules/modules/expansion/lastline_submit.py

@@ -33,13 +33,9 @@ moduleinfo = {
 }
 
 moduleconfig = [
-    "api_url",
+    "url",
-    "api_key",
     "api_token",
-    "username",
+    "key",
-    "password",
-    # Module options
-    "bypass_cache",
 ]
 
 
@@ -75,31 +71,31 @@ def handler(q=False):
 
     # Parse the init parameters
     try:
-        auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request)
+        config = request.get("config", {})
-        api_url = request.get("config", {}).get("api_url", lastline_api.DEFAULT_LASTLINE_API)
+        auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
+        api_url = config.get("url", lastline_api.DEFAULT_LL_ANALYSIS_API_URL)
     except Exception as e:
         misperrors["error"] = "Error parsing configuration: {}".format(e)
         return misperrors
 
     # Parse the call parameters
     try:
-        bypass_cache = request.get("config", {}).get("bypass_cache", False)
+        call_args = {}
-        call_args = {"bypass_cache": __str_to_bool(bypass_cache)}
         if "url" in request:
             # URLs are text strings
-            api_method = lastline_api.LastlineCommunityAPIClient.submit_url
+            api_method = lastline_api.AnalysisClient.submit_url
             call_args["url"] = request.get("url")
         else:
             data = request.get("data")
             # Malware samples are zip-encrypted and then base64 encoded
             if "malware-sample" in request:
-                api_method = lastline_api.LastlineCommunityAPIClient.submit_file
+                api_method = lastline_api.AnalysisClient.submit_file
                 call_args["file_data"] = __unzip(base64.b64decode(data), DEFAULT_ZIP_PASSWORD)
                 call_args["file_name"] = request.get("malware-sample").split("|", 1)[0]
                 call_args["password"] = DEFAULT_ZIP_PASSWORD
             # Attachments are just base64 encoded
             elif "attachment" in request:
-                api_method = lastline_api.LastlineCommunityAPIClient.submit_file
+                api_method = lastline_api.AnalysisClient.submit_file
                 call_args["file_data"] = base64.b64decode(data)
                 call_args["file_name"] = request.get("attachment")
 
@@ -112,7 +108,7 @@ def handler(q=False):
 
     # Make the API call
     try:
-        api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data)
+        api_client = lastline_api.AnalysisClient(api_url, auth_data)
         response = api_method(api_client, **call_args)
         task_uuid = response.get("task_uuid")
         if not task_uuid:
@@ -127,7 +123,7 @@ def handler(q=False):
         return misperrors
 
     # Assemble and return
-    analysis_link = lastline_api.get_analysis_link(api_url, task_uuid)
+    analysis_link = lastline_api.get_task_link(task_uuid, analysis_url=api_url)
 
     return {
         "results": [
@@ -152,12 +148,12 @@ if __name__ == "__main__":
     args = parser.parse_args()
     c = configparser.ConfigParser()
     c.read(args.config_file)
-    a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name)
+    a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
 
     j = json.dumps(
         {
             "config": a,
-            "url": "https://www.google.com",
+            "url": "https://www.google.exe.com",
         }
     )
     print(json.dumps(handler(j), indent=4, sort_keys=True))

misp_modules/modules/expansion/virustotal_public.py

@@ -85,7 +85,9 @@ class DomainQuery(VirusTotalParser):
             whois_object = MISPObject(whois)
             whois_object.add_attribute('text', type='text', value=query_result[whois])
             self.misp_event.add_object(**whois_object)
+        if 'domain_siblings' in query_result:
             siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings'])
+            if 'subdomains' in query_result:
                 self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings)
         self.parse_urls(query_result)
 

misp_modules/modules/import_mod/lastline_import.py

@@ -29,8 +29,6 @@ moduleinfo = {
 }
 
 moduleconfig = [
-    "api_key",
-    "api_token",
     "username",
     "password",
 ]
@@ -65,24 +63,25 @@ def handler(q=False):
 
     # Parse the init parameters
     try:
-        auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request)
+        config = request["config"]
+        auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
         analysis_link = request["config"]["analysis_link"]
         # The API url changes based on the analysis link host name
-        api_url = lastline_api.get_api_url_from_link(analysis_link)
+        api_url = lastline_api.get_portal_url_from_task_link(analysis_link)
     except Exception as e:
         misperrors["error"] = "Error parsing configuration: {}".format(e)
         return misperrors
 
     # Parse the call parameters
     try:
-        task_uuid = lastline_api.get_uuid_from_link(analysis_link)
+        task_uuid = lastline_api.get_uuid_from_task_link(analysis_link)
     except (KeyError, ValueError) as e:
         misperrors["error"] = "Error processing input parameters: {}".format(e)
         return misperrors
 
     # Make the API calls
     try:
-        api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data)
+        api_client = lastline_api.PortalClient(api_url, auth_data)
         response = api_client.get_progress(task_uuid)
         if response.get("completed") != 1:
             raise ValueError("Analysis is not finished yet.")
@@ -122,7 +121,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
     c = configparser.ConfigParser()
     c.read(args.config_file)
-    a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name)
+    a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
 
     j = json.dumps(
         {