Merge branch 'master' of github.com:MISP/misp-modules

pull/367/head
chrisr3d 2020-01-21 22:04:38 +01:00
commit 840ada484b
9 changed files with 642 additions and 476 deletions

View File

@ -613,6 +613,7 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re
Query Lastline with an analysis link and parse the report into MISP attributes and objects. Query Lastline with an analysis link and parse the report into MISP attributes and objects.
The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module. The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module.
- **features**: - **features**:
>The module requires a Lastline Portal `username` and `password`.
>The module uses the new format and it is able to return MISP attributes and objects. >The module uses the new format and it is able to return MISP attributes and objects.
>The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module. >The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module.
- **input**: - **input**:
@ -630,7 +631,7 @@ The analysis link can also be retrieved from the output of the [lastline_submit]
Module to submit a file or URL to Lastline. Module to submit a file or URL to Lastline.
- **features**: - **features**:
>The module requires a Lastline API key and token (or username and password). >The module requires a Lastline Analysis `api_token` and `key`.
>When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module. >When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module.
- **input**: - **input**:
>File or URL to submit to Lastline. >File or URL to submit to Lastline.
@ -1701,6 +1702,7 @@ A module to import data from a Joe Sandbox analysis json report.
Module to import and parse reports from Lastline analysis links. Module to import and parse reports from Lastline analysis links.
- **features**: - **features**:
>The module requires a Lastline Portal `username` and `password`.
>The module uses the new format and it is able to return MISP attributes and objects. >The module uses the new format and it is able to return MISP attributes and objects.
>The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module. >The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module.
- **input**: - **input**:

View File

@ -5,5 +5,5 @@
"input": "Link to a Lastline analysis.", "input": "Link to a Lastline analysis.",
"output": "MISP attributes and objects parsed from the analysis report.", "output": "MISP attributes and objects parsed from the analysis report.",
"references": ["https://www.lastline.com"], "references": ["https://www.lastline.com"],
"features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module."
} }

View File

@ -5,5 +5,5 @@
"input": "File or URL to submit to Lastline.", "input": "File or URL to submit to Lastline.",
"output": "Link to the report generated by Lastline.", "output": "Link to the report generated by Lastline.",
"references": ["https://www.lastline.com"], "references": ["https://www.lastline.com"],
"features": "The module requires a Lastline API key and token (or username and password).\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module."
} }

View File

@ -5,5 +5,5 @@
"input": "Link to a Lastline analysis.", "input": "Link to a Lastline analysis.",
"output": "MISP attributes and objects parsed from the analysis report.", "output": "MISP attributes and objects parsed from the analysis report.",
"references": ["https://www.lastline.com"], "references": ["https://www.lastline.com"],
"features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module."
} }

File diff suppressed because it is too large Load Diff

View File

@ -27,8 +27,6 @@ moduleinfo = {
} }
moduleconfig = [ moduleconfig = [
"api_key",
"api_token",
"username", "username",
"password", "password",
] ]
@ -51,24 +49,25 @@ def handler(q=False):
# Parse the init parameters # Parse the init parameters
try: try:
auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) config = request["config"]
auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
analysis_link = request['attribute']['value'] analysis_link = request['attribute']['value']
# The API url changes based on the analysis link host name # The API url changes based on the analysis link host name
api_url = lastline_api.get_api_url_from_link(analysis_link) api_url = lastline_api.get_portal_url_from_task_link(analysis_link)
except Exception as e: except Exception as e:
misperrors["error"] = "Error parsing configuration: {}".format(e) misperrors["error"] = "Error parsing configuration: {}".format(e)
return misperrors return misperrors
# Parse the call parameters # Parse the call parameters
try: try:
task_uuid = lastline_api.get_uuid_from_link(analysis_link) task_uuid = lastline_api.get_uuid_from_task_link(analysis_link)
except (KeyError, ValueError) as e: except (KeyError, ValueError) as e:
misperrors["error"] = "Error processing input parameters: {}".format(e) misperrors["error"] = "Error processing input parameters: {}".format(e)
return misperrors return misperrors
# Make the API calls # Make the API calls
try: try:
api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) api_client = lastline_api.PortalClient(api_url, auth_data)
response = api_client.get_progress(task_uuid) response = api_client.get_progress(task_uuid)
if response.get("completed") != 1: if response.get("completed") != 1:
raise ValueError("Analysis is not finished yet.") raise ValueError("Analysis is not finished yet.")
@ -108,7 +107,7 @@ if __name__ == "__main__":
args = parser.parse_args() args = parser.parse_args()
c = configparser.ConfigParser() c = configparser.ConfigParser()
c.read(args.config_file) c.read(args.config_file)
a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
j = json.dumps( j = json.dumps(
{ {

View File

@ -33,13 +33,9 @@ moduleinfo = {
} }
moduleconfig = [ moduleconfig = [
"api_url", "url",
"api_key",
"api_token", "api_token",
"username", "key",
"password",
# Module options
"bypass_cache",
] ]
@ -75,31 +71,31 @@ def handler(q=False):
# Parse the init parameters # Parse the init parameters
try: try:
auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) config = request.get("config", {})
api_url = request.get("config", {}).get("api_url", lastline_api.DEFAULT_LASTLINE_API) auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
api_url = config.get("url", lastline_api.DEFAULT_LL_ANALYSIS_API_URL)
except Exception as e: except Exception as e:
misperrors["error"] = "Error parsing configuration: {}".format(e) misperrors["error"] = "Error parsing configuration: {}".format(e)
return misperrors return misperrors
# Parse the call parameters # Parse the call parameters
try: try:
bypass_cache = request.get("config", {}).get("bypass_cache", False) call_args = {}
call_args = {"bypass_cache": __str_to_bool(bypass_cache)}
if "url" in request: if "url" in request:
# URLs are text strings # URLs are text strings
api_method = lastline_api.LastlineCommunityAPIClient.submit_url api_method = lastline_api.AnalysisClient.submit_url
call_args["url"] = request.get("url") call_args["url"] = request.get("url")
else: else:
data = request.get("data") data = request.get("data")
# Malware samples are zip-encrypted and then base64 encoded # Malware samples are zip-encrypted and then base64 encoded
if "malware-sample" in request: if "malware-sample" in request:
api_method = lastline_api.LastlineCommunityAPIClient.submit_file api_method = lastline_api.AnalysisClient.submit_file
call_args["file_data"] = __unzip(base64.b64decode(data), DEFAULT_ZIP_PASSWORD) call_args["file_data"] = __unzip(base64.b64decode(data), DEFAULT_ZIP_PASSWORD)
call_args["file_name"] = request.get("malware-sample").split("|", 1)[0] call_args["file_name"] = request.get("malware-sample").split("|", 1)[0]
call_args["password"] = DEFAULT_ZIP_PASSWORD call_args["password"] = DEFAULT_ZIP_PASSWORD
# Attachments are just base64 encoded # Attachments are just base64 encoded
elif "attachment" in request: elif "attachment" in request:
api_method = lastline_api.LastlineCommunityAPIClient.submit_file api_method = lastline_api.AnalysisClient.submit_file
call_args["file_data"] = base64.b64decode(data) call_args["file_data"] = base64.b64decode(data)
call_args["file_name"] = request.get("attachment") call_args["file_name"] = request.get("attachment")
@ -112,7 +108,7 @@ def handler(q=False):
# Make the API call # Make the API call
try: try:
api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) api_client = lastline_api.AnalysisClient(api_url, auth_data)
response = api_method(api_client, **call_args) response = api_method(api_client, **call_args)
task_uuid = response.get("task_uuid") task_uuid = response.get("task_uuid")
if not task_uuid: if not task_uuid:
@ -127,7 +123,7 @@ def handler(q=False):
return misperrors return misperrors
# Assemble and return # Assemble and return
analysis_link = lastline_api.get_analysis_link(api_url, task_uuid) analysis_link = lastline_api.get_task_link(task_uuid, analysis_url=api_url)
return { return {
"results": [ "results": [
@ -152,12 +148,12 @@ if __name__ == "__main__":
args = parser.parse_args() args = parser.parse_args()
c = configparser.ConfigParser() c = configparser.ConfigParser()
c.read(args.config_file) c.read(args.config_file)
a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
j = json.dumps( j = json.dumps(
{ {
"config": a, "config": a,
"url": "https://www.google.com", "url": "https://www.google.exe.com",
} }
) )
print(json.dumps(handler(j), indent=4, sort_keys=True)) print(json.dumps(handler(j), indent=4, sort_keys=True))

View File

@ -85,8 +85,10 @@ class DomainQuery(VirusTotalParser):
whois_object = MISPObject(whois) whois_object = MISPObject(whois)
whois_object.add_attribute('text', type='text', value=query_result[whois]) whois_object.add_attribute('text', type='text', value=query_result[whois])
self.misp_event.add_object(**whois_object) self.misp_event.add_object(**whois_object)
siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) if 'domain_siblings' in query_result:
self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings'])
if 'subdomains' in query_result:
self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings)
self.parse_urls(query_result) self.parse_urls(query_result)
def parse_siblings(self, domain): def parse_siblings(self, domain):

View File

@ -29,8 +29,6 @@ moduleinfo = {
} }
moduleconfig = [ moduleconfig = [
"api_key",
"api_token",
"username", "username",
"password", "password",
] ]
@ -65,24 +63,25 @@ def handler(q=False):
# Parse the init parameters # Parse the init parameters
try: try:
auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) config = request["config"]
auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
analysis_link = request["config"]["analysis_link"] analysis_link = request["config"]["analysis_link"]
# The API url changes based on the analysis link host name # The API url changes based on the analysis link host name
api_url = lastline_api.get_api_url_from_link(analysis_link) api_url = lastline_api.get_portal_url_from_task_link(analysis_link)
except Exception as e: except Exception as e:
misperrors["error"] = "Error parsing configuration: {}".format(e) misperrors["error"] = "Error parsing configuration: {}".format(e)
return misperrors return misperrors
# Parse the call parameters # Parse the call parameters
try: try:
task_uuid = lastline_api.get_uuid_from_link(analysis_link) task_uuid = lastline_api.get_uuid_from_task_link(analysis_link)
except (KeyError, ValueError) as e: except (KeyError, ValueError) as e:
misperrors["error"] = "Error processing input parameters: {}".format(e) misperrors["error"] = "Error processing input parameters: {}".format(e)
return misperrors return misperrors
# Make the API calls # Make the API calls
try: try:
api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) api_client = lastline_api.PortalClient(api_url, auth_data)
response = api_client.get_progress(task_uuid) response = api_client.get_progress(task_uuid)
if response.get("completed") != 1: if response.get("completed") != 1:
raise ValueError("Analysis is not finished yet.") raise ValueError("Analysis is not finished yet.")
@ -122,7 +121,7 @@ if __name__ == "__main__":
args = parser.parse_args() args = parser.parse_args()
c = configparser.ConfigParser() c = configparser.ConfigParser()
c.read(args.config_file) c.read(args.config_file)
a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
j = json.dumps( j = json.dumps(
{ {