diff --git a/doc/README.md b/doc/README.md index 2049803..7e6bee3 100644 --- a/doc/README.md +++ b/doc/README.md @@ -613,6 +613,7 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re Query Lastline with an analysis link and parse the report into MISP attributes and objects. The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module. - **features**: +>The module requires a Lastline Portal `username` and `password`. >The module uses the new format and it is able to return MISP attributes and objects. >The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module. - **input**: @@ -630,7 +631,7 @@ The analysis link can also be retrieved from the output of the [lastline_submit] Module to submit a file or URL to Lastline. - **features**: ->The module requires a Lastline API key and token (or username and password). +>The module requires a Lastline Analysis `api_token` and `key`. >When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module. - **input**: >File or URL to submit to Lastline. @@ -1701,6 +1702,7 @@ A module to import data from a Joe Sandbox analysis json report. Module to import and parse reports from Lastline analysis links. - **features**: +>The module requires a Lastline Portal `username` and `password`. >The module uses the new format and it is able to return MISP attributes and objects. >The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module. - **input**: diff --git a/doc/expansion/lastline_query.json b/doc/expansion/lastline_query.json index 0d5da39..6165890 100644 --- a/doc/expansion/lastline_query.json +++ b/doc/expansion/lastline_query.json @@ -5,5 +5,5 @@ "input": "Link to a Lastline analysis.", "output": "MISP attributes and objects parsed from the analysis report.", "references": ["https://www.lastline.com"], - "features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." + "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." } diff --git a/doc/expansion/lastline_submit.json b/doc/expansion/lastline_submit.json index cf5ef57..d053f55 100644 --- a/doc/expansion/lastline_submit.json +++ b/doc/expansion/lastline_submit.json @@ -5,5 +5,5 @@ "input": "File or URL to submit to Lastline.", "output": "Link to the report generated by Lastline.", "references": ["https://www.lastline.com"], - "features": "The module requires a Lastline API key and token (or username and password).\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." + "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." } diff --git a/doc/import_mod/lastline_import.json b/doc/import_mod/lastline_import.json index 1d4c15d..99414e0 100644 --- a/doc/import_mod/lastline_import.json +++ b/doc/import_mod/lastline_import.json @@ -5,5 +5,5 @@ "input": "Link to a Lastline analysis.", "output": "MISP attributes and objects parsed from the analysis report.", "references": ["https://www.lastline.com"], - "features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." + "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." } diff --git a/misp_modules/lib/lastline_api.py b/misp_modules/lib/lastline_api.py index a6912b8..83726ad 100644 --- a/misp_modules/lib/lastline_api.py +++ b/misp_modules/lib/lastline_api.py @@ -30,19 +30,21 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import ipaddress +import abc import logging +import io +import ipaddress +import pymisp import re import requests -import pymisp - from urllib import parse -DEFAULT_LASTLINE_API = "https://user.lastline.com/papi" +DEFAULT_LL_PORTAL_API_URL = "https://user.lastline.com/papi" +DEFAULT_LL_ANALYSIS_API_URL = "https://analysis.lastline.com" -HOSTED_LASTLINE_DOMAINS = frozenset([ +LL_HOSTED_DOMAINS = frozenset([ "user.lastline.com", "user.emea.lastline.com", ]) @@ -53,61 +55,66 @@ def purge_none(d): return {k: v for k, v in d.items() if v is not None} -def get_analysis_link(api_url, uuid): +def get_task_link(uuid, analysis_url=None, portal_url=None): """ - Get the analysis link of a task given the task uuid. + Get the task link given the task uuid and at least one API url. - :param str api_url: the URL :param str uuid: the task uuid + :param str|None analysis_url: the URL to the analysis API endpoint + :param str|None portal_url: the URL to the portal API endpoint :rtype: str - :return: the analysis link + :return: the task link + :raises ValueError: if not enough parameters have been provided """ + if not analysis_url and not portal_url: + raise ValueError("Neither analysis URL or portal URL have been specified") + if analysis_url: + portal_url = "{}/papi".format(analysis_url.replace("analysis.", "user.")) portal_url_path = "../portal#/analyst/task/{}/overview".format(uuid) - analysis_link = parse.urljoin(api_url, portal_url_path) - return analysis_link + return parse.urljoin(portal_url, portal_url_path) -def get_uuid_from_link(analysis_link): +def get_portal_url_from_task_link(task_link): """ - Return task uuid from link or raise ValueError exception. + Return the portal API url related to the provided task link. - :param str analysis_link: a link + :param str task_link: a link + :rtype: str + :return: the portal API url + """ + parsed_uri = parse.urlparse(task_link) + return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri) + + +def get_uuid_from_task_link(task_link): + """ + Return the uuid from a task link. + + :param str task_link: a link :rtype: str :return: the uuid :raises ValueError: if the link contains not task uuid """ try: - return re.findall("[a-fA-F0-9]{32}", analysis_link)[0] + return re.findall("[a-fA-F0-9]{32}", task_link)[0] except IndexError: raise ValueError("Link does not contain a valid task uuid") -def is_analysis_hosted(analysis_link): +def is_task_hosted(task_link): """ - Return whether the analysis link is pointing to a hosted submission. + Return whether the portal link is pointing to a hosted submission. - :param str analysis_link: a link + :param str task_link: a link :rtype: boolean - :return: whether the link is hosted + :return: whether the link points to a hosted analysis """ - for domain in HOSTED_LASTLINE_DOMAINS: - if domain in analysis_link: + for domain in LL_HOSTED_DOMAINS: + if domain in task_link: return True return False -def get_api_url_from_link(analysis_link): - """ - Return the API url related to the provided analysis link. - - :param str analysis_link: a link - :rtype: str - :return: the API url - """ - parsed_uri = parse.urlparse(analysis_link) - return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri) - - class InvalidArgument(Exception): """Error raised invalid.""" @@ -135,6 +142,572 @@ class ApiError(Error): return "{}{}".format(self.error_msg, error_code) +class LastlineAbstractClient(abc.ABC): + """"A very basic HTTP client providing basic functionality.""" + + __metaclass__ = abc.ABCMeta + + SUB_APIS = ('analysis', 'authentication', 'knowledgebase', 'login') + FORMATS = ["json", "xml"] + + @classmethod + def sanitize_login_params(cls, api_key, api_token, username, password): + """ + Return a dictionary with either API or USER credentials. + + :param str|None api_key: the API key + :param str|None api_token: the API token + :param str|None username: the username + :param str|None password: the password + :rtype: dict[str, str] + :return: the dictionary + :raises InvalidArgument: if too many values are invalid + """ + if api_key and api_token: + return { + "key": api_key, + "api_token": api_token, + } + elif username and password: + return { + "username": username, + "password": password, + } + else: + raise InvalidArgument("Arguments provided do not contain valid data") + + @classmethod + def get_login_params_from_dict(cls, d): + """ + Get the module configuration from a ConfigParser object. + + :param dict[str, str] d: the dictionary + :rtype: dict[str, str] + :return: the parsed configuration + """ + api_key = d.get("key") + api_token = d.get("api_token") + username = d.get("username") + password = d.get("password") + return cls.sanitize_login_params(api_key, api_token, username, password) + + @classmethod + def get_login_params_from_conf(cls, conf, section_name): + """ + Get the module configuration from a ConfigParser object. + + :param ConfigParser conf: the conf object + :param str section_name: the section name + :rtype: dict[str, str] + :return: the parsed configuration + """ + api_key = conf.get(section_name, "key", fallback=None) + api_token = conf.get(section_name, "api_token", fallback=None) + username = conf.get(section_name, "username", fallback=None) + password = conf.get(section_name, "password", fallback=None) + return cls.sanitize_login_params(api_key, api_token, username, password) + + @classmethod + def load_from_conf(cls, conf, section_name): + """ + Load client from a ConfigParser object. + + :param ConfigParser conf: the conf object + :param str section_name: the section name + :rtype: T <- LastlineAbstractClient + :return: the loaded client + """ + url = conf.get(section_name, "url") + return cls(url, cls.get_login_params_from_conf(conf, section_name)) + + def __init__(self, api_url, login_params, timeout=60, verify_ssl=True): + """ + Instantiate a Lastline mini client. + + :param str api_url: the URL of the API + :param dict[str, str]: the login parameters + :param int timeout: the timeout + :param boolean verify_ssl: whether to verify the SSL certificate + """ + self._url = api_url + self._login_params = login_params + self._timeout = timeout + self._verify_ssl = verify_ssl + self._session = None + self._logger = logging.getLogger(__name__) + + @abc.abstractmethod + def _login(self): + """Login using account-based or key-based methods.""" + + def _is_logged_in(self): + """Return whether we have an active session.""" + return self._session is not None + + @staticmethod + def _parse_response(response): + """ + Parse the response. + + :param requests.Response response: the response + :rtype: tuple(str|None, Error|ApiError) + :return: a tuple with mutually exclusive fields (either the response or the error) + """ + try: + ret = response.json() + if "success" not in ret: + return None, Error("no success field in response") + + if not ret["success"]: + error_msg = ret.get("error", "") + error_code = ret.get("error_code", None) + return None, ApiError(error_msg, error_code) + + if "data" not in ret: + return None, Error("no data field in response") + + return ret["data"], None + except ValueError as e: + return None, Error("Response not json {}".format(e)) + + def _handle_response(self, response, raw=False): + """ + Check a response for issues and parse the return. + + :param requests.Response response: the response + :param boolean raw: whether the raw body should be returned + :rtype: str + :return: if raw, return the response content; if not raw, the data field + :raises: CommunicationError, ApiError, Error + """ + # Check for HTTP errors, and re-raise in case + try: + response.raise_for_status() + except requests.RequestException as e: + _, err = self._parse_response(response) + if isinstance(err, ApiError): + err_msg = "{}: {}".format(e, err.error_msg) + else: + err_msg = "{}".format(e) + raise CommunicationError(err_msg) + + # Otherwise return the data (either parsed or not) but reraise if we have an API error + if raw: + return response.content + data, err = self._parse_response(response) + if err: + raise err + return data + + def _build_url(self, sub_api, parts, requested_format="json"): + if sub_api not in self.SUB_APIS: + raise InvalidArgument(sub_api) + if requested_format not in self.FORMATS: + raise InvalidArgument(requested_format) + num_parts = 2 + len(parts) + pattern = "/".join(["%s"] * num_parts) + ".%s" + params = [self._url, sub_api] + parts + [requested_format] + return pattern % tuple(params) + + def post(self, module, function, params=None, data=None, files=None, fmt="json"): + if isinstance(function, list): + functions = function + else: + functions = [function] if function else [] + url = self._build_url(module, functions, requested_format=fmt) + return self.do_request( + url=url, + method="POST", + params=params, + data=data, + files=files, + fmt=fmt, + ) + + def get(self, module, function, params=None, fmt="json"): + if isinstance(function, list): + functions = function + else: + functions = [function] if function else [] + url = self._build_url(module, functions, requested_format=fmt) + return self.do_request( + url=url, + method="GET", + params=params, + fmt=fmt, + ) + + def do_request( + self, + method, + url, + params=None, + data=None, + files=None, + fmt="json", + raw=False, + raw_response=False, + headers=None, + stream_response=False + ): + if raw_response: + raw = True + + if fmt: + fmt = fmt.lower().strip() + if fmt not in self.FORMATS: + raise InvalidArgument("Only json, xml, html and pdf supported") + elif not raw: + raise InvalidArgument("Unformatted response requires raw=True") + + if fmt != "json" and not raw: + raise InvalidArgument("Non-json format requires raw=True") + + if method not in ["POST", "GET"]: + raise InvalidArgument("Only POST and GET supported") + + if not self._is_logged_in(): + self._login() + + try: + try: + response = self._session.request( + method=method, + url=url, + data=data, + params=params, + files=files, + verify=self._verify_ssl, + timeout=self._timeout, + stream=stream_response, + headers=headers, + ) + except requests.RequestException as e: + raise CommunicationError(e) + + if raw_response: + return response + return self._handle_response(response, raw) + + except Error as e: + raise e + + except CommunicationError as e: + raise e + + +class AnalysisClient(LastlineAbstractClient): + + def _login(self): + """ + Creates auth session for malscape-service. + + Credentials are 'key' and 'api_token'. + """ + if self._session is None: + self._session = requests.session() + url = self._build_url("authentication", ["login"]) + self.do_request("POST", url, params=purge_none(self._login_params)) + + def get_progress(self, uuid): + """ + Get the completion progress of a given task. + :param str uuid: the unique identifier of the submitted task + :rtype: dict[str, int] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + url = self._build_url('analysis', ['get_progress']) + params = {'uuid': uuid} + return self.do_request("POST", url, params=params) + + def get_result(self, uuid): + """ + Get report results for a given task. + + :param str uuid: the unique identifier of the submitted task + :rtype: dict[str, any] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + # better: use 'get_results()' but that would break + # backwards-compatibility + url = self._build_url('analysis', ['get']) + params = {'uuid': uuid} + return self.do_request("GET", url, params=params) + + def submit_file( + self, + file_data, + file_name=None, + password=None, + analysis_env=None, + allow_network_traffic=True, + analysis_timeout=None, + bypass_cache=False, + ): + """ + Upload a file to be analyzed. + + :param bytes file_data: the data as a byte sequence + :param str|None file_name: if set, represents the name of the file to submit + :param str|None password: if set, use it to extract the sample + :param str|None analysis_env: if set, e.g windowsxp + :param boolean allow_network_traffic: if set to False, deny network connections + :param int|None analysis_timeout: if set limit the duration of the analysis + :param boolean bypass_cache: whether to re-process a file (requires special permissions) + :rtype: dict[str, any] + :return: a dictionary in the following form if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + file_stream = io.BytesIO(file_data) + api_url = self._build_url("analysis", ["submit", "file"]) + params = purge_none({ + "bypass_cache": bypass_cache and 1 or None, + "analysis_timeout": analysis_timeout, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic and 1 or None, + "filename": file_name, + "password": password, + "full_report_score": -1, + }) + + files = purge_none({ + # If an explicit filename was provided, we can pass it down to + # python-requests to use it in the multipart/form-data. This avoids + # having python-requests trying to guess the filename based on stream + # attributes. + # + # The problem with this is that, if the filename is not ASCII, then + # this triggers a bug in flask/werkzeug which means the file is + # thrown away. Thus, we just force an ASCII name + "file": ('dummy-ascii-name-for-file-param', file_stream), + }) + + return self.do_request("POST", api_url, params=params, files=files) + + def submit_url( + self, + url, + referer=None, + user_agent=None, + bypass_cache=False, + ): + """ + Upload an URL to be analyzed. + + :param str url: the url to analyze + :param str|None referer: the referer + :param str|None user_agent: the user agent + :param boolean bypass_cache: bypass_cache + :rtype: dict[str, any] + :return: a dictionary like the following if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + api_url = self._build_url("analysis", ["submit", "url"]) + params = purge_none({ + "url": url, + "referer": referer, + "bypass_cache": bypass_cache and 1 or None, + "user_agent": user_agent or None, + }) + return self.do_request("POST", api_url, params=params) + + +class PortalClient(LastlineAbstractClient): + + def _login(self): + """ + Login using account-based or key-based methods. + + Credentials are 'username' and 'password' + """ + if self._session is None: + self._session = requests.session() + self.post("login", function=None, data=self._login_params) + + def get_progress(self, uuid, analysis_instance=None): + """ + Get the completion progress of a given task. + + :param str uuid: the unique identifier of the submitted task + :param str analysis_instance: if set, defines the analysis instance to query + :rtype: dict[str, int] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance}) + return self.get("analysis", "get_progress", params=params) + + def get_result(self, uuid, analysis_instance=None): + """ + Get report results for a given task. + + :param str uuid: the unique identifier of the submitted task + :param str analysis_instance: if set, defines the analysis instance to query + :rtype: dict[str, any] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + params = purge_none( + { + "uuid": uuid, + "analysis_instance": analysis_instance, + "report_format": "json", + } + ) + return self.get("analysis", "get_result", params=params) + + def submit_url( + self, + url, + referer=None, + user_agent=None, + bypass_cache=False, + ): + """ + Upload an URL to be analyzed. + + :param str url: the url to analyze + :param str|None referer: the referer + :param str|None user_agent: the user agent + :param boolean bypass_cache: bypass_cache + :rtype: dict[str, any] + :return: a dictionary like the following if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + params = purge_none( + { + "url": url, + "bypass_cache": bypass_cache, + "referer": referer, + "user_agent": user_agent + } + ) + return self.post("analysis", "submit_url", params=params) + + def submit_file( + self, + file_data, + file_name=None, + password=None, + analysis_env=None, + allow_network_traffic=True, + analysis_timeout=None, + bypass_cache=False, + ): + """ + Upload a file to be analyzed. + + :param bytes file_data: the data as a byte sequence + :param str|None file_name: if set, represents the name of the file to submit + :param str|None password: if set, use it to extract the sample + :param str|None analysis_env: if set, e.g windowsxp + :param boolean allow_network_traffic: if set to False, deny network connections + :param int|None analysis_timeout: if set limit the duration of the analysis + :param boolean bypass_cache: whether to re-process a file (requires special permissions) + :rtype: dict[str, any] + :return: a dictionary in the following form if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + params = purge_none( + { + "filename": file_name, + "password": password, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic, + "analysis_timeout": analysis_timeout, + "bypass_cache": bypass_cache, + } + ) + files = {"file": (file_name, file_data, "application/octet-stream")} + return self.post("analysis", "submit_file", params=params, files=files) + + class LastlineResultBaseParser(object): """ This is a parser to extract *basic* information from a Lastline result dictionary. @@ -247,7 +820,7 @@ class LastlineResultBaseParser(object): # Add sandbox info like score and sandbox type o = pymisp.MISPObject(name="sandbox-report") - sandbox_type = "saas" if is_analysis_hosted(analysis_link) else "on-premise" + sandbox_type = "saas" if is_task_hosted(analysis_link) else "on-premise" o.add_attribute("score", result["score"]) o.add_attribute("sandbox-type", sandbox_type) o.add_attribute("{}-sandbox".format(sandbox_type), "lastline") @@ -266,408 +839,3 @@ class LastlineResultBaseParser(object): # Add mitre techniques for technique in self._get_mitre_techniques(result): self.misp_event.add_tag(technique) - - -class LastlineCommunityHTTPClient(object): - """"A very basic HTTP client providing basic functionality.""" - - @classmethod - def sanitize_login_params(cls, api_key, api_token, username, password): - """ - Return a dictionary with either API or USER credentials. - - :param str|None api_key: the API key - :param str|None api_token: the API token - :param str|None username: the username - :param str|None password: the password - :rtype: dict[str, str] - :return: the dictionary - :raises InvalidArgument: if too many values are invalid - """ - if api_key and api_token: - return { - "api_key": api_key, - "api_token": api_token, - } - elif username and password: - return { - "username": username, - "password": password, - } - else: - raise InvalidArgument("Arguments provided do not contain valid data") - - @classmethod - def get_login_params_from_conf(cls, conf, section_name): - """ - Get the module configuration from a ConfigParser object. - - :param ConfigParser conf: the conf object - :param str section_name: the section name - :rtype: dict[str, str] - :return: the parsed configuration - """ - api_key = conf.get(section_name, "api_key", fallback=None) - api_token = conf.get(section_name, "api_token", fallback=None) - username = conf.get(section_name, "username", fallback=None) - password = conf.get(section_name, "password", fallback=None) - return cls.sanitize_login_params(api_key, api_token, username, password) - - @classmethod - def get_login_params_from_request(cls, request): - """ - Get the module configuration from a ConfigParser object. - - :param dict[str, any] request: the request object - :rtype: dict[str, str] - :return: the parsed configuration - """ - api_key = request.get("config", {}).get("api_key") - api_token = request.get("config", {}).get("api_token") - username = request.get("config", {}).get("username") - password = request.get("config", {}).get("password") - return cls.sanitize_login_params(api_key, api_token, username, password) - - def __init__(self, api_url, login_params, timeout=60, verify_ssl=True): - """ - Instantiate a Lastline mini client. - - :param str api_url: the URL of the API - :param dict[str, str]: the login parameters - :param int timeout: the timeout - :param boolean verify_ssl: whether to verify the SSL certificate - """ - self.__url = api_url - self.__login_params = login_params - self.__timeout = timeout - self.__verify_ssl = verify_ssl - self.__session = None - self.__logger = logging.getLogger(__name__) - - def __login(self): - """Login using account-based or key-based methods.""" - if self.__session is None: - self.__session = requests.session() - - login_url = "/".join([self.__url, "login"]) - try: - response = self.__session.request( - method="POST", - url=login_url, - data=self.__login_params, - verify=self.__verify_ssl, - timeout=self.__timeout, - proxies=None, - ) - except requests.RequestException as e: - raise CommunicationError(e) - - self.__handle_response(response) - - def __is_logged_in(self): - """Return whether we have an active session.""" - return self.__session is not None - - @staticmethod - def __parse_response(response): - """ - Parse the response. - - :param requests.Response response: the response - :rtype: tuple(str|None, Error|ApiError) - :return: a tuple with mutually exclusive fields (either the response or the error) - """ - try: - ret = response.json() - if "success" not in ret: - return None, Error("no success field in response") - - if not ret["success"]: - error_msg = ret.get("error", "") - error_code = ret.get("error_code", None) - return None, ApiError(error_msg, error_code) - - if "data" not in ret: - return None, Error("no data field in response") - - return ret["data"], None - except ValueError as e: - return None, Error("Response not json {}".format(e)) - - def __handle_response(self, response, raw=False): - """ - Check a response for issues and parse the return. - - :param requests.Response response: the response - :param boolean raw: whether the raw body should be returned - :rtype: str - :return: if raw, return the response content; if not raw, the data field - :raises: CommunicationError, ApiError, Error - """ - # Check for HTTP errors, and re-raise in case - try: - response.raise_for_status() - except requests.RequestException as e: - _, err = self.__parse_response(response) - if isinstance(err, ApiError): - err_msg = "{}: {}".format(e, err.error_msg) - else: - err_msg = "{}".format(e) - raise CommunicationError(err_msg) - - # Otherwise return the data (either parsed or not) but reraise if we have an API error - if raw: - return response.content - data, err = self.__parse_response(response) - if err: - raise err - return data - - def do_request( - self, - method, - module, - function, - params=None, - data=None, - files=None, - url=None, - fmt="JSON", - raw=False, - raw_response=False, - headers=None, - stream_response=False - ): - if raw_response: - raw = True - - if fmt: - fmt = fmt.lower().strip() - if fmt not in ["json", "xml", "html", "pdf"]: - raise InvalidArgument("Only json, xml, html and pdf supported") - elif not raw: - raise InvalidArgument("Unformatted response requires raw=True") - - if fmt != "json" and not raw: - raise InvalidArgument("Non-json format requires raw=True") - - if method not in ["POST", "GET"]: - raise InvalidArgument("Only POST and GET supported") - - function = function.strip(" /") - if not function: - raise InvalidArgument("No function provided") - - # Login after we verified that all arguments are fine - if not self.__is_logged_in(): - self.__login() - - url_parts = [url or self.__url] - module = module.strip(" /") - if module: - url_parts.append(module) - if fmt: - function_part = "%s.%s" % (function, fmt) - else: - function_part = function - url_parts.append(function_part) - url = "/".join(url_parts) - - try: - try: - response = self.__session.request( - method=method, - url=url, - data=data, - params=params, - files=files, - verify=self.__verify_ssl, - timeout=self.__timeout, - stream=stream_response, - headers=headers, - ) - except requests.RequestException as e: - raise CommunicationError(e) - - if raw_response: - return response - return self.__handle_response(response, raw) - - except Error as e: - raise e - - except CommunicationError as e: - raise e - - -class LastlineCommunityAPIClient(object): - """"A very basic API client providing basic functionality.""" - - def __init__(self, api_url, login_params): - """ - Instantiate the API client. - - :param str api_url: the URL to the API server - :param dict[str, str] login_params: the login parameters - """ - self._client = LastlineCommunityHTTPClient(api_url, login_params) - self._logger = logging.getLogger(__name__) - - def _post(self, module, function, params=None, data=None, files=None, fmt="JSON"): - return self._client.do_request( - method="POST", - module=module, - function=function, - params=params, - data=data, - files=files, - fmt=fmt, - ) - - def _get(self, module, function, params=None, fmt="JSON"): - return self._client.do_request( - method="GET", - module=module, - function=function, - params=params, - fmt=fmt, - ) - - def get_progress(self, uuid, analysis_instance=None): - """ - Get the completion progress of a given task. - - :param str uuid: the unique identifier of the submitted task - :param str analysis_instance: if set, defines the analysis instance to query - :rtype: dict[str, int] - :return: a dictionary like the the following: - { - "completed": 1, - "progress": 100 - } - """ - params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance}) - return self._get("analysis", "get_progress", params=params) - - def get_result(self, uuid, analysis_instance=None): - """ - Get report results for a given task. - - :param str uuid: the unique identifier of the submitted task - :param str analysis_instance: if set, defines the analysis instance to query - :rtype: dict[str, any] - :return: a dictionary like the the following: - { - "completed": 1, - "progress": 100 - } - """ - params = purge_none( - { - "uuid": uuid, - "analysis_instance": analysis_instance, - "report_format": "json", - } - ) - return self._get("analysis", "get_result", params=params) - - def submit_url( - self, - url, - referer=None, - user_agent=None, - bypass_cache=False, - ): - """ - Upload an URL to be analyzed. - - :param str url: the url to analyze - :param str|None referer: the referer - :param str|None user_agent: the user agent - :param boolean bypass_cache: bypass_cache - :rtype: dict[str, any] - :return: a dictionary like the following if the analysis is already available: - { - "submission": "2019-11-17 09:33:23", - "child_tasks": [...], - "reports": [...], - "submission_timestamp": "2019-11-18 16:11:04", - "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", - "score": 0, - "analysis_subject": { - "url": "https://www.google.com" - }, - "last_submission_timestamp": "2019-11-18 16:11:04" - } - - OR the following if the analysis is still pending: - - { - "submission_timestamp": "2019-11-18 13:59:25", - "task_uuid": "f3c0ae115d51001017ff8da768fa6049", - } - """ - params = purge_none( - { - "url": url, - "bypass_cache": bypass_cache, - "referer": referer, - "user_agent": user_agent - } - ) - return self._post(module="analysis", function="submit_url", params=params) - - def submit_file( - self, - file_data, - file_name=None, - password=None, - analysis_env=None, - allow_network_traffic=True, - analysis_timeout=None, - bypass_cache=False, - ): - """ - Upload a file to be analyzed. - - :param bytes file_data: the data as a byte sequence - :param str|None file_name: if set, represents the name of the file to submit - :param str|None password: if set, use it to extract the sample - :param str|None analysis_env: if set, e.g windowsxp - :param boolean allow_network_traffic: if set to False, deny network connections - :param int|None analysis_timeout: if set limit the duration of the analysis - :param boolean bypass_cache: whether to re-process a file (requires special permissions) - :rtype: dict[str, any] - :return: a dictionary in the following form if the analysis is already available: - { - "submission": "2019-11-17 09:33:23", - "child_tasks": [...], - "reports": [...], - "submission_timestamp": "2019-11-18 16:11:04", - "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", - "score": 0, - "analysis_subject": { - "url": "https://www.google.com" - }, - "last_submission_timestamp": "2019-11-18 16:11:04" - } - - OR the following if the analysis is still pending: - - { - "submission_timestamp": "2019-11-18 13:59:25", - "task_uuid": "f3c0ae115d51001017ff8da768fa6049", - } - """ - params = purge_none( - { - "filename": file_name, - "password": password, - "analysis_env": analysis_env, - "allow_network_traffic": allow_network_traffic, - "analysis_timeout": analysis_timeout, - "bypass_cache": bypass_cache, - } - ) - files = {"file": (file_name, file_data, "application/octet-stream")} - return self._post(module="analysis", function="submit_file", params=params, files=files) diff --git a/misp_modules/modules/expansion/lastline_query.py b/misp_modules/modules/expansion/lastline_query.py index 4019b92..9fdc9de 100644 --- a/misp_modules/modules/expansion/lastline_query.py +++ b/misp_modules/modules/expansion/lastline_query.py @@ -27,8 +27,6 @@ moduleinfo = { } moduleconfig = [ - "api_key", - "api_token", "username", "password", ] @@ -51,24 +49,25 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) + config = request["config"] + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) analysis_link = request['attribute']['value'] # The API url changes based on the analysis link host name - api_url = lastline_api.get_api_url_from_link(analysis_link) + api_url = lastline_api.get_portal_url_from_task_link(analysis_link) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - task_uuid = lastline_api.get_uuid_from_link(analysis_link) + task_uuid = lastline_api.get_uuid_from_task_link(analysis_link) except (KeyError, ValueError) as e: misperrors["error"] = "Error processing input parameters: {}".format(e) return misperrors # Make the API calls try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.PortalClient(api_url, auth_data) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") @@ -108,7 +107,7 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( { diff --git a/misp_modules/modules/expansion/lastline_submit.py b/misp_modules/modules/expansion/lastline_submit.py index 0ae475a..1572955 100644 --- a/misp_modules/modules/expansion/lastline_submit.py +++ b/misp_modules/modules/expansion/lastline_submit.py @@ -33,13 +33,9 @@ moduleinfo = { } moduleconfig = [ - "api_url", - "api_key", + "url", "api_token", - "username", - "password", - # Module options - "bypass_cache", + "key", ] @@ -75,31 +71,31 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) - api_url = request.get("config", {}).get("api_url", lastline_api.DEFAULT_LASTLINE_API) + config = request.get("config", {}) + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) + api_url = config.get("url", lastline_api.DEFAULT_LL_ANALYSIS_API_URL) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - bypass_cache = request.get("config", {}).get("bypass_cache", False) - call_args = {"bypass_cache": __str_to_bool(bypass_cache)} + call_args = {} if "url" in request: # URLs are text strings - api_method = lastline_api.LastlineCommunityAPIClient.submit_url + api_method = lastline_api.AnalysisClient.submit_url call_args["url"] = request.get("url") else: data = request.get("data") # Malware samples are zip-encrypted and then base64 encoded if "malware-sample" in request: - api_method = lastline_api.LastlineCommunityAPIClient.submit_file + api_method = lastline_api.AnalysisClient.submit_file call_args["file_data"] = __unzip(base64.b64decode(data), DEFAULT_ZIP_PASSWORD) call_args["file_name"] = request.get("malware-sample").split("|", 1)[0] call_args["password"] = DEFAULT_ZIP_PASSWORD # Attachments are just base64 encoded elif "attachment" in request: - api_method = lastline_api.LastlineCommunityAPIClient.submit_file + api_method = lastline_api.AnalysisClient.submit_file call_args["file_data"] = base64.b64decode(data) call_args["file_name"] = request.get("attachment") @@ -112,7 +108,7 @@ def handler(q=False): # Make the API call try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.AnalysisClient(api_url, auth_data) response = api_method(api_client, **call_args) task_uuid = response.get("task_uuid") if not task_uuid: @@ -127,7 +123,7 @@ def handler(q=False): return misperrors # Assemble and return - analysis_link = lastline_api.get_analysis_link(api_url, task_uuid) + analysis_link = lastline_api.get_task_link(task_uuid, analysis_url=api_url) return { "results": [ @@ -152,12 +148,12 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( { "config": a, - "url": "https://www.google.com", + "url": "https://www.google.exe.com", } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index 69c2c85..f31855e 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -85,8 +85,10 @@ class DomainQuery(VirusTotalParser): whois_object = MISPObject(whois) whois_object.add_attribute('text', type='text', value=query_result[whois]) self.misp_event.add_object(**whois_object) - siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) - self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) + if 'domain_siblings' in query_result: + siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) + if 'subdomains' in query_result: + self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) self.parse_urls(query_result) def parse_siblings(self, domain): diff --git a/misp_modules/modules/import_mod/lastline_import.py b/misp_modules/modules/import_mod/lastline_import.py index ff26b93..ebf88d8 100644 --- a/misp_modules/modules/import_mod/lastline_import.py +++ b/misp_modules/modules/import_mod/lastline_import.py @@ -29,8 +29,6 @@ moduleinfo = { } moduleconfig = [ - "api_key", - "api_token", "username", "password", ] @@ -65,24 +63,25 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) + config = request["config"] + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) analysis_link = request["config"]["analysis_link"] # The API url changes based on the analysis link host name - api_url = lastline_api.get_api_url_from_link(analysis_link) + api_url = lastline_api.get_portal_url_from_task_link(analysis_link) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - task_uuid = lastline_api.get_uuid_from_link(analysis_link) + task_uuid = lastline_api.get_uuid_from_task_link(analysis_link) except (KeyError, ValueError) as e: misperrors["error"] = "Error processing input parameters: {}".format(e) return misperrors # Make the API calls try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.PortalClient(api_url, auth_data) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") @@ -122,7 +121,7 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( {