Merge branch 'master' of github.com:MISP/misp-modules

pull/367/head
chrisr3d 3 years ago
commit 840ada484b
  1. 4
      doc/README.md
  2. 2
      doc/expansion/lastline_query.json
  3. 2
      doc/expansion/lastline_submit.json
  4. 2
      doc/import_mod/lastline_import.json
  5. 698
      misp_modules/lib/lastline_api.py
  6. 13
      misp_modules/modules/expansion/lastline_query.py
  7. 30
      misp_modules/modules/expansion/lastline_submit.py
  8. 6
      misp_modules/modules/expansion/virustotal_public.py
  9. 13
      misp_modules/modules/import_mod/lastline_import.py

@ -613,6 +613,7 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re
Query Lastline with an analysis link and parse the report into MISP attributes and objects.
The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module.
- **features**:
>The module requires a Lastline Portal `username` and `password`.
>The module uses the new format and it is able to return MISP attributes and objects.
>The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module.
- **input**:
@ -630,7 +631,7 @@ The analysis link can also be retrieved from the output of the [lastline_submit]
Module to submit a file or URL to Lastline.
- **features**:
>The module requires a Lastline API key and token (or username and password).
>The module requires a Lastline Analysis `api_token` and `key`.
>When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module.
- **input**:
>File or URL to submit to Lastline.
@ -1701,6 +1702,7 @@ A module to import data from a Joe Sandbox analysis json report.
Module to import and parse reports from Lastline analysis links.
- **features**:
>The module requires a Lastline Portal `username` and `password`.
>The module uses the new format and it is able to return MISP attributes and objects.
>The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module.
- **input**:

@ -5,5 +5,5 @@
"input": "Link to a Lastline analysis.",
"output": "MISP attributes and objects parsed from the analysis report.",
"references": ["https://www.lastline.com"],
"features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module."
"features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module."
}

@ -5,5 +5,5 @@
"input": "File or URL to submit to Lastline.",
"output": "Link to the report generated by Lastline.",
"references": ["https://www.lastline.com"],
"features": "The module requires a Lastline API key and token (or username and password).\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module."
"features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module."
}

@ -5,5 +5,5 @@
"input": "Link to a Lastline analysis.",
"output": "MISP attributes and objects parsed from the analysis report.",
"references": ["https://www.lastline.com"],
"features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module."
"features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module."
}

@ -30,19 +30,21 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""
import ipaddress
import abc
import logging
import io
import ipaddress
import pymisp
import re
import requests
import pymisp
from urllib import parse
DEFAULT_LASTLINE_API = "https://user.lastline.com/papi"
DEFAULT_LL_PORTAL_API_URL = "https://user.lastline.com/papi"
DEFAULT_LL_ANALYSIS_API_URL = "https://analysis.lastline.com"
HOSTED_LASTLINE_DOMAINS = frozenset([
LL_HOSTED_DOMAINS = frozenset([
"user.lastline.com",
"user.emea.lastline.com",
])
@ -53,61 +55,66 @@ def purge_none(d):
return {k: v for k, v in d.items() if v is not None}
def get_analysis_link(api_url, uuid):
def get_task_link(uuid, analysis_url=None, portal_url=None):
"""
Get the analysis link of a task given the task uuid.
Get the task link given the task uuid and at least one API url.
:param str api_url: the URL
:param str uuid: the task uuid
:param str|None analysis_url: the URL to the analysis API endpoint
:param str|None portal_url: the URL to the portal API endpoint
:rtype: str
:return: the analysis link
:return: the task link
:raises ValueError: if not enough parameters have been provided
"""
if not analysis_url and not portal_url:
raise ValueError("Neither analysis URL or portal URL have been specified")
if analysis_url:
portal_url = "{}/papi".format(analysis_url.replace("analysis.", "user."))
portal_url_path = "../portal#/analyst/task/{}/overview".format(uuid)
analysis_link = parse.urljoin(api_url, portal_url_path)
return analysis_link
return parse.urljoin(portal_url, portal_url_path)
def get_portal_url_from_task_link(task_link):
"""
Return the portal API url related to the provided task link.
:param str task_link: a link
:rtype: str
:return: the portal API url
"""
parsed_uri = parse.urlparse(task_link)
return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri)
def get_uuid_from_link(analysis_link):
def get_uuid_from_task_link(task_link):
"""
Return task uuid from link or raise ValueError exception.
Return the uuid from a task link.
:param str analysis_link: a link
:param str task_link: a link
:rtype: str
:return: the uuid
:raises ValueError: if the link contains not task uuid
"""
try:
return re.findall("[a-fA-F0-9]{32}", analysis_link)[0]
return re.findall("[a-fA-F0-9]{32}", task_link)[0]
except IndexError:
raise ValueError("Link does not contain a valid task uuid")
def is_analysis_hosted(analysis_link):
def is_task_hosted(task_link):
"""
Return whether the analysis link is pointing to a hosted submission.
Return whether the portal link is pointing to a hosted submission.
:param str analysis_link: a link
:param str task_link: a link
:rtype: boolean
:return: whether the link is hosted
:return: whether the link points to a hosted analysis
"""
for domain in HOSTED_LASTLINE_DOMAINS:
if domain in analysis_link:
for domain in LL_HOSTED_DOMAINS:
if domain in task_link:
return True
return False
def get_api_url_from_link(analysis_link):
"""
Return the API url related to the provided analysis link.
:param str analysis_link: a link
:rtype: str
:return: the API url
"""
parsed_uri = parse.urlparse(analysis_link)
return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri)
class InvalidArgument(Exception):
"""Error raised invalid."""
@ -135,141 +142,13 @@ class ApiError(Error):
return "{}{}".format(self.error_msg, error_code)
class LastlineResultBaseParser(object):
"""
This is a parser to extract *basic* information from a Lastline result dictionary.
Note: this is a *version 0*: the information we extract is merely related to the behaviors and
the HTTP connections. Further iterations will include host activity such as files, mutexes,
registry keys, strings, etc.
"""
def __init__(self):
"""Constructor."""
self.misp_event = None
@staticmethod
def _get_mitre_techniques(result):
return [
"misp-galaxy:mitre-attack-pattern=\"{} - {}\"".format(w[0], w[1])
for w in sorted(set([
(y["id"], y["name"])
for x in result.get("malicious_activity", [])
for y in result.get("activity_to_mitre_techniques", {}).get(x, [])
]))
]
def parse(self, analysis_link, result):
"""
Parse the analysis result into a MISP event.
:param str analysis_link: the analysis link
:param dict[str, any] result: the JSON returned by the analysis client.
:rtype: MISPEvent
:return: some results that can be consumed by MIPS.
"""
self.misp_event = pymisp.MISPEvent()
# Add analysis subject info
if "url" in result["analysis_subject"]:
o = pymisp.MISPObject("url")
o.add_attribute("url", result["analysis_subject"]["url"])
else:
o = pymisp.MISPObject("file")
o.add_attribute("md5", type="md5", value=result["analysis_subject"]["md5"])
o.add_attribute("sha1", type="sha1", value=result["analysis_subject"]["sha1"])
o.add_attribute("sha256", type="sha256", value=result["analysis_subject"]["sha256"])
o.add_attribute(
"mimetype",
type="mime-type",
value=result["analysis_subject"]["mime_type"]
)
self.misp_event.add_object(o)
# Add HTTP requests from url analyses
network_dict = result.get("report", {}).get("analysis", {}).get("network", {})
for request in network_dict.get("requests", []):
parsed_uri = parse.urlparse(request["url"])
o = pymisp.MISPObject(name='http-request')
o.add_attribute('host', parsed_uri.netloc)
o.add_attribute('method', "GET")
o.add_attribute('uri', request["url"])
o.add_attribute("ip", request["ip"])
self.misp_event.add_object(o)
# Add network behaviors from files
for subject in result.get("report", {}).get("analysis_subjects", []):
# Add DNS requests
for dns_query in subject.get("dns_queries", []):
hostname = dns_query.get("hostname")
# Skip if it is an IP address
try:
if hostname == "wpad":
continue
_ = ipaddress.ip_address(hostname)
continue
except ValueError:
pass
o = pymisp.MISPObject(name='dns-record')
o.add_attribute('queried-domain', hostname)
self.misp_event.add_object(o)
# Add HTTP conversations (as network connection and as http request)
for http_conversation in subject.get("http_conversations", []):
o = pymisp.MISPObject(name="network-connection")
o.add_attribute("ip-src", http_conversation["src_ip"])
o.add_attribute("ip-dst", http_conversation["dst_ip"])
o.add_attribute("src-port", http_conversation["src_port"])
o.add_attribute("dst-port", http_conversation["dst_port"])
o.add_attribute("hostname-dst", http_conversation["dst_host"])
o.add_attribute("layer3-protocol", "IP")
o.add_attribute("layer4-protocol", "TCP")
o.add_attribute("layer7-protocol", "HTTP")
self.misp_event.add_object(o)
method, path, http_version = http_conversation["url"].split(" ")
if http_conversation["dst_port"] == 80:
uri = "http://{}{}".format(http_conversation["dst_host"], path)
else:
uri = "http://{}:{}{}".format(
http_conversation["dst_host"],
http_conversation["dst_port"],
path
)
o = pymisp.MISPObject(name='http-request')
o.add_attribute('host', http_conversation["dst_host"])
o.add_attribute('method', method)
o.add_attribute('uri', uri)
o.add_attribute('ip', http_conversation["dst_ip"])
self.misp_event.add_object(o)
# Add sandbox info like score and sandbox type
o = pymisp.MISPObject(name="sandbox-report")
sandbox_type = "saas" if is_analysis_hosted(analysis_link) else "on-premise"
o.add_attribute("score", result["score"])
o.add_attribute("sandbox-type", sandbox_type)
o.add_attribute("{}-sandbox".format(sandbox_type), "lastline")
o.add_attribute("permalink", analysis_link)
self.misp_event.add_object(o)
# Add behaviors
o = pymisp.MISPObject(name="sb-signature")
o.add_attribute("software", "Lastline")
for activity in result.get("malicious_activity", []):
a = pymisp.MISPAttribute()
a.from_dict(type="text", value=activity)
o.add_attribute("signature", **a)
self.misp_event.add_object(o)
# Add mitre techniques
for technique in self._get_mitre_techniques(result):
self.misp_event.add_tag(technique)
class LastlineAbstractClient(abc.ABC):
""""A very basic HTTP client providing basic functionality."""
__metaclass__ = abc.ABCMeta
class LastlineCommunityHTTPClient(object):
""""A very basic HTTP client providing basic functionality."""
SUB_APIS = ('analysis', 'authentication', 'knowledgebase', 'login')
FORMATS = ["json", "xml"]
@classmethod
def sanitize_login_params(cls, api_key, api_token, username, password):
@ -286,7 +165,7 @@ class LastlineCommunityHTTPClient(object):
"""
if api_key and api_token:
return {
"api_key": api_key,
"key": api_key,
"api_token": api_token,
}
elif username and password:
@ -297,6 +176,21 @@ class LastlineCommunityHTTPClient(object):
else:
raise InvalidArgument("Arguments provided do not contain valid data")
@classmethod
def get_login_params_from_dict(cls, d):
"""
Get the module configuration from a ConfigParser object.
:param dict[str, str] d: the dictionary
:rtype: dict[str, str]
:return: the parsed configuration
"""
api_key = d.get("key")
api_token = d.get("api_token")
username = d.get("username")
password = d.get("password")
return cls.sanitize_login_params(api_key, api_token, username, password)
@classmethod
def get_login_params_from_conf(cls, conf, section_name):
"""
@ -307,26 +201,24 @@ class LastlineCommunityHTTPClient(object):
:rtype: dict[str, str]
:return: the parsed configuration
"""
api_key = conf.get(section_name, "api_key", fallback=None)
api_key = conf.get(section_name, "key", fallback=None)
api_token = conf.get(section_name, "api_token", fallback=None)
username = conf.get(section_name, "username", fallback=None)
password = conf.get(section_name, "password", fallback=None)
return cls.sanitize_login_params(api_key, api_token, username, password)
@classmethod
def get_login_params_from_request(cls, request):
def load_from_conf(cls, conf, section_name):
"""
Get the module configuration from a ConfigParser object.
Load client from a ConfigParser object.
:param dict[str, any] request: the request object
:rtype: dict[str, str]
:return: the parsed configuration
:param ConfigParser conf: the conf object
:param str section_name: the section name
:rtype: T <- LastlineAbstractClient
:return: the loaded client
"""
api_key = request.get("config", {}).get("api_key")
api_token = request.get("config", {}).get("api_token")
username = request.get("config", {}).get("username")
password = request.get("config", {}).get("password")
return cls.sanitize_login_params(api_key, api_token, username, password)
url = conf.get(section_name, "url")
return cls(url, cls.get_login_params_from_conf(conf, section_name))
def __init__(self, api_url, login_params, timeout=60, verify_ssl=True):
"""
@ -337,39 +229,23 @@ class LastlineCommunityHTTPClient(object):
:param int timeout: the timeout
:param boolean verify_ssl: whether to verify the SSL certificate
"""
self.__url = api_url
self.__login_params = login_params
self.__timeout = timeout
self.__verify_ssl = verify_ssl
self.__session = None
self.__logger = logging.getLogger(__name__)
self._url = api_url
self._login_params = login_params
self._timeout = timeout
self._verify_ssl = verify_ssl
self._session = None
self._logger = logging.getLogger(__name__)
def __login(self):
@abc.abstractmethod
def _login(self):
"""Login using account-based or key-based methods."""
if self.__session is None:
self.__session = requests.session()
login_url = "/".join([self.__url, "login"])
try:
response = self.__session.request(
method="POST",
url=login_url,
data=self.__login_params,
verify=self.__verify_ssl,
timeout=self.__timeout,
proxies=None,
)
except requests.RequestException as e:
raise CommunicationError(e)
self.__handle_response(response)
def __is_logged_in(self):
def _is_logged_in(self):
"""Return whether we have an active session."""
return self.__session is not None
return self._session is not None
@staticmethod
def __parse_response(response):
def _parse_response(response):
"""
Parse the response.
@ -394,7 +270,7 @@ class LastlineCommunityHTTPClient(object):
except ValueError as e:
return None, Error("Response not json {}".format(e))
def __handle_response(self, response, raw=False):
def _handle_response(self, response, raw=False):
"""
Check a response for issues and parse the return.
@ -408,7 +284,7 @@ class LastlineCommunityHTTPClient(object):
try:
response.raise_for_status()
except requests.RequestException as e:
_, err = self.__parse_response(response)
_, err = self._parse_response(response)
if isinstance(err, ApiError):
err_msg = "{}: {}".format(e, err.error_msg)
else:
@ -418,21 +294,57 @@ class LastlineCommunityHTTPClient(object):
# Otherwise return the data (either parsed or not) but reraise if we have an API error
if raw:
return response.content
data, err = self.__parse_response(response)
data, err = self._parse_response(response)
if err:
raise err
return data
def _build_url(self, sub_api, parts, requested_format="json"):
if sub_api not in self.SUB_APIS:
raise InvalidArgument(sub_api)
if requested_format not in self.FORMATS:
raise InvalidArgument(requested_format)
num_parts = 2 + len(parts)
pattern = "/".join(["%s"] * num_parts) + ".%s"
params = [self._url, sub_api] + parts + [requested_format]
return pattern % tuple(params)
def post(self, module, function, params=None, data=None, files=None, fmt="json"):
if isinstance(function, list):
functions = function
else:
functions = [function] if function else []
url = self._build_url(module, functions, requested_format=fmt)
return self.do_request(
url=url,
method="POST",
params=params,
data=data,
files=files,
fmt=fmt,
)
def get(self, module, function, params=None, fmt="json"):
if isinstance(function, list):
functions = function
else:
functions = [function] if function else []
url = self._build_url(module, functions, requested_format=fmt)
return self.do_request(
url=url,
method="GET",
params=params,
fmt=fmt,
)
def do_request(
self,
method,
module,
function,
url,
params=None,
data=None,
files=None,
url=None,
fmt="JSON",
fmt="json",
raw=False,
raw_response=False,
headers=None,
@ -443,7 +355,7 @@ class LastlineCommunityHTTPClient(object):
if fmt:
fmt = fmt.lower().strip()
if fmt not in ["json", "xml", "html", "pdf"]:
if fmt not in self.FORMATS:
raise InvalidArgument("Only json, xml, html and pdf supported")
elif not raw:
raise InvalidArgument("Unformatted response requires raw=True")
@ -454,35 +366,19 @@ class LastlineCommunityHTTPClient(object):
if method not in ["POST", "GET"]:
raise InvalidArgument("Only POST and GET supported")
function = function.strip(" /")
if not function:
raise InvalidArgument("No function provided")
# Login after we verified that all arguments are fine
if not self.__is_logged_in():
self.__login()
url_parts = [url or self.__url]
module = module.strip(" /")
if module:
url_parts.append(module)
if fmt:
function_part = "%s.%s" % (function, fmt)
else:
function_part = function
url_parts.append(function_part)
url = "/".join(url_parts)
if not self._is_logged_in():
self._login()
try:
try:
response = self.__session.request(
response = self._session.request(
method=method,
url=url,
data=data,
params=params,
files=files,
verify=self.__verify_ssl,
timeout=self.__timeout,
verify=self._verify_ssl,
timeout=self._timeout,
stream=stream_response,
headers=headers,
)
@ -491,7 +387,7 @@ class LastlineCommunityHTTPClient(object):
if raw_response:
return response
return self.__handle_response(response, raw)
return self._handle_response(response, raw)
except Error as e:
raise e
@ -500,38 +396,177 @@ class LastlineCommunityHTTPClient(object):
raise e
class LastlineCommunityAPIClient(object):
""""A very basic API client providing basic functionality."""
class AnalysisClient(LastlineAbstractClient):
def __init__(self, api_url, login_params):
def _login(self):
"""
Instantiate the API client.
Creates auth session for malscape-service.
:param str api_url: the URL to the API server
:param dict[str, str] login_params: the login parameters
Credentials are 'key' and 'api_token'.
"""
self._client = LastlineCommunityHTTPClient(api_url, login_params)
self._logger = logging.getLogger(__name__)
if self._session is None:
self._session = requests.session()
url = self._build_url("authentication", ["login"])
self.do_request("POST", url, params=purge_none(self._login_params))
def _post(self, module, function, params=None, data=None, files=None, fmt="JSON"):
return self._client.do_request(
method="POST",
module=module,
function=function,
params=params,
data=data,
files=files,
fmt=fmt,
)
def get_progress(self, uuid):
"""
Get the completion progress of a given task.
:param str uuid: the unique identifier of the submitted task
:rtype: dict[str, int]
:return: a dictionary like the the following:
{
"completed": 1,
"progress": 100
}
"""
url = self._build_url('analysis', ['get_progress'])
params = {'uuid': uuid}
return self.do_request("POST", url, params=params)
def _get(self, module, function, params=None, fmt="JSON"):
return self._client.do_request(
method="GET",
module=module,
function=function,
params=params,
fmt=fmt,
)
def get_result(self, uuid):
"""
Get report results for a given task.
:param str uuid: the unique identifier of the submitted task
:rtype: dict[str, any]
:return: a dictionary like the the following:
{
"completed": 1,
"progress": 100
}
"""
# better: use 'get_results()' but that would break
# backwards-compatibility
url = self._build_url('analysis', ['get'])
params = {'uuid': uuid}
return self.do_request("GET", url, params=params)
def submit_file(
self,
file_data,
file_name=None,
password=None,
analysis_env=None,
allow_network_traffic=True,
analysis_timeout=None,
bypass_cache=False,
):
"""
Upload a file to be analyzed.
:param bytes file_data: the data as a byte sequence
:param str|None file_name: if set, represents the name of the file to submit
:param str|None password: if set, use it to extract the sample
:param str|None analysis_env: if set, e.g windowsxp
:param boolean allow_network_traffic: if set to False, deny network connections
:param int|None analysis_timeout: if set limit the duration of the analysis
:param boolean bypass_cache: whether to re-process a file (requires special permissions)
:rtype: dict[str, any]
:return: a dictionary in the following form if the analysis is already available:
{
"submission": "2019-11-17 09:33:23",
"child_tasks": [...],
"reports": [...],
"submission_timestamp": "2019-11-18 16:11:04",
"task_uuid": "86097fb8e4cd00100464cb001b97ecbe",
"score": 0,
"analysis_subject": {
"url": "https://www.google.com"
},
"last_submission_timestamp": "2019-11-18 16:11:04"
}
OR the following if the analysis is still pending:
{
"submission_timestamp": "2019-11-18 13:59:25",
"task_uuid": "f3c0ae115d51001017ff8da768fa6049",
}
"""
file_stream = io.BytesIO(file_data)
api_url = self._build_url("analysis", ["submit", "file"])
params = purge_none({
"bypass_cache": bypass_cache and 1 or None,
"analysis_timeout": analysis_timeout,
"analysis_env": analysis_env,
"allow_network_traffic": allow_network_traffic and 1 or None,
"filename": file_name,
"password": password,
"full_report_score": -1,
})
files = purge_none({
# If an explicit filename was provided, we can pass it down to
# python-requests to use it in the multipart/form-data. This avoids
# having python-requests trying to guess the filename based on stream
# attributes.
#
# The problem with this is that, if the filename is not ASCII, then
# this triggers a bug in flask/werkzeug which means the file is
# thrown away. Thus, we just force an ASCII name
"file": ('dummy-ascii-name-for-file-param', file_stream),
})
return self.do_request("POST", api_url, params=params, files=files)
def submit_url(
self,
url,
referer=None,
user_agent=None,
bypass_cache=False,
):
"""
Upload an URL to be analyzed.
:param str url: the url to analyze
:param str|None referer: the referer
:param str|None user_agent: the user agent
:param boolean bypass_cache: bypass_cache
:rtype: dict[str, any]
:return: a dictionary like the following if the analysis is already available:
{
"submission": "2019-11-17 09:33:23",
"child_tasks": [...],
"reports": [...],
"submission_timestamp": "2019-11-18 16:11:04",
"task_uuid": "86097fb8e4cd00100464cb001b97ecbe",
"score": 0,
"analysis_subject": {
"url": "https://www.google.com"
},
"last_submission_timestamp": "2019-11-18 16:11:04"
}
OR the following if the analysis is still pending:
{
"submission_timestamp": "2019-11-18 13:59:25",
"task_uuid": "f3c0ae115d51001017ff8da768fa6049",
}
"""
api_url = self._build_url("analysis", ["submit", "url"])
params = purge_none({
"url": url,
"referer": referer,
"bypass_cache": bypass_cache and 1 or None,
"user_agent": user_agent or None,
})
return self.do_request("POST", api_url, params=params)
class PortalClient(LastlineAbstractClient):
def _login(self):
"""
Login using account-based or key-based methods.
Credentials are 'username' and 'password'
"""
if self._session is None:
self._session = requests.session()
self.post("login", function=None, data=self._login_params)
def get_progress(self, uuid, analysis_instance=None):
"""
@ -547,7 +582,7 @@ class LastlineCommunityAPIClient(object):
}
"""
params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance})
return self._get("analysis", "get_progress", params=params)
return self.get("analysis", "get_progress", params=params)
def get_result(self, uuid, analysis_instance=None):
"""
@ -569,7 +604,7 @@ class LastlineCommunityAPIClient(object):
"report_format": "json",
}
)
return self._get("analysis", "get_result", params=params)
return self.get("analysis", "get_result", params=params)
def submit_url(
self,
@ -615,7 +650,7 @@ class LastlineCommunityAPIClient(object):
"user_agent": user_agent
}
)
return self._post(module="analysis", function="submit_url", params=params)
return self.post("analysis", "submit_url", params=params)
def submit_file(
self,
@ -670,4 +705,137 @@ class LastlineCommunityAPIClient(object):
}
)
files = {"file": (file_name, file_data, "application/octet-stream")}
return self._post(module="analysis", function="submit_file", params=params, files=files)
return self.post("analysis", "submit_file", params=params, files=files)
class LastlineResultBaseParser(object):
"""
This is a parser to extract *basic* information from a Lastline result dictionary.
Note: this is a *version 0*: the information we extract is merely related to the behaviors and
the HTTP connections. Further iterations will include host activity such as files, mutexes,
registry keys, strings, etc.
"""
def __init__(self):
"""Constructor."""
self.misp_event = None
@staticmethod
def _get_mitre_techniques(result):
return [
"misp-galaxy:mitre-attack-pattern=\"{} - {}\"".format(w[0], w[1])
for w in sorted(set([
(y["id"], y["name"])
for x in result.get("malicious_activity", [])
for y in result.get("activity_to_mitre_techniques", {}).get(x, [])
]))
]
def parse(self, analysis_link, result):
"""
Parse the analysis result into a MISP event.
:param str analysis_link: the analysis link
:param dict[str, any] result: the JSON returned by the analysis client.
:rtype: MISPEvent
:return: some results that can be consumed by MIPS.
"""
self.misp_event = pymisp.MISPEvent()
# Add analysis subject info
if "url" in result["analysis_subject"]:
o = pymisp.MISPObject("url")
o.add_attribute("url", result["analysis_subject"]["url"])
else:
o = pymisp.MISPObject("file")
o.add_attribute("md5", type="md5", value=result["analysis_subject"]["md5"])
o.add_attribute("sha1", type="sha1", value=result["analysis_subject"]["sha1"])
o.add_attribute("sha256", type="sha256", value=result["analysis_subject"]["sha256"])
o.add_attribute(
"mimetype",
type="mime-type",
value=result["analysis_subject"]["mime_type"]
)
self.misp_event.add_object(o)
# Add HTTP requests from url analyses
network_dict = result.get("report", {}).get("analysis", {}).get("network", {})
for request in network_dict.get("requests", []):
parsed_uri = parse.urlparse(request["url"])
o = pymisp.MISPObject(name='http-request')
o.add_attribute('host', parsed_uri.netloc)
o.add_attribute('method', "GET")
o.add_attribute('uri', request["url"])
o.add_attribute("ip", request["ip"])
self.misp_event.add_object(o)
# Add network behaviors from files
for subject in result.get("report", {}).get("analysis_subjects", []):
# Add DNS requests
for dns_query in subject.get("dns_queries", []):
hostname = dns_query.get("hostname")
# Skip if it is an IP address
try:
if hostname == "wpad":
continue
_ = ipaddress.ip_address(hostname)
continue
except ValueError:
pass
o = pymisp.MISPObject(name='dns-record')
o.add_attribute('queried-domain', hostname)
self.misp_event.add_object(o)
# Add HTTP conversations (as network connection and as http request)
for http_conversation in subject.get("http_conversations", []):
o = pymisp.MISPObject(name="network-connection")
o.add_attribute("ip-src", http_conversation["src_ip"])
o.add_attribute("ip-dst", http_conversation["dst_ip"])
o.add_attribute("src-port", http_conversation["src_port"])
o.add_attribute("dst-port", http_conversation["dst_port"])
o.add_attribute("hostname-dst", http_conversation["dst_host"])
o.add_attribute("layer3-protocol", "IP")
o.add_attribute("layer4-protocol", "TCP")
o.add_attribute("layer7-protocol", "HTTP")
self.misp_event.add_object(o)
method, path, http_version = http_conversation["url"].split(" ")
if http_conversation["dst_port"] == 80:
uri = "http://{}{}".format(http_conversation["dst_host"], path)
else:
uri = "http://{}:{}{}".format(
http_conversation["dst_host"],
http_conversation["dst_port"],
path
)
o = pymisp.MISPObject(name='http-request')
o.add_attribute('host', http_conversation["dst_host"])
o.add_attribute('method', method)
o.add_attribute('uri', uri)
o.add_attribute('ip', http_conversation["dst_ip"])
self.misp_event.add_object(o)
# Add sandbox info like score and sandbox type
o = pymisp.MISPObject(name="sandbox-report")
sandbox_type = "saas" if is_task_hosted(analysis_link) else "on-premise"
o.add_attribute("score", result["score"])
o.add_attribute("sandbox-type", sandbox_type)
o.add_attribute("{}-sandbox".format(sandbox_type), "lastline")
o.add_attribute("permalink", analysis_link)
self.misp_event.add_object(o)
# Add behaviors
o = pymisp.MISPObject(name="sb-signature")
o.add_attribute("software", "Lastline")
for activity in result.get("malicious_activity", []):
a = pymisp.MISPAttribute()
a.from_dict(type="text", value=activity)
o.add_attribute("signature", **a)
self.misp_event.add_object(o)
# Add mitre techniques
for technique in self._get_mitre_techniques(result):
self.misp_event.add_tag(technique)

@ -27,8 +27,6 @@ moduleinfo = {
}
moduleconfig = [
"api_key",
"api_token",
"username",
"password",
]
@ -51,24 +49,25 @@ def handler(q=False):
# Parse the init parameters
try:
auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request)
config = request["config"]
auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
analysis_link = request['attribute']['value']
# The API url changes based on the analysis link host name
api_url = lastline_api.get_api_url_from_link(analysis_link)
api_url = lastline_api.get_portal_url_from_task_link(analysis_link)
except Exception as e:
misperrors["error"] = "Error parsing configuration: {}".format(e)
return misperrors
# Parse the call parameters
try:
task_uuid = lastline_api.get_uuid_from_link(analysis_link)
task_uuid = lastline_api.get_uuid_from_task_link(analysis_link)
except (KeyError, ValueError) as e:
misperrors["error"] = "Error processing input parameters: {}".format(e)
return misperrors
# Make the API calls
try:
api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data)
api_client = lastline_api.PortalClient(api_url, auth_data)
response = api_client.get_progress(task_uuid)
if response.get("completed") != 1:
raise ValueError("Analysis is not finished yet.")
@ -108,7 +107,7 @@ if __name__ == "__main__":
args = parser.parse_args()
c = configparser.ConfigParser()
c.read(args.config_file)
a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name)
a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
j = json.dumps(
{

@ -33,13 +33,9 @@ moduleinfo = {
}
moduleconfig = [
"api_url",
"api_key",
"url",
"api_token",
"username",
"password",
# Module options
"bypass_cache",
"key",
]
@ -75,31 +71,31 @@ def handler(q=False):
# Parse the init parameters
try:
auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request)
api_url = request.get("config", {}).get("api_url", lastline_api.DEFAULT_LASTLINE_API)
config = request.get("config", {})
auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
api_url = config.get("url", lastline_api.DEFAULT_LL_ANALYSIS_API_URL)
except Exception as e:
misperrors["error"] = "Error parsing configuration: {}".format(e)
return misperrors
# Parse the call parameters
try:
bypass_cache = request.get("config", {}).get("bypass_cache", False)
call_args = {"bypass_cache": __str_to_bool(bypass_cache)}
call_args = {}
if "url" in request:
# URLs are text strings
api_method = lastline_api.LastlineCommunityAPIClient.submit_url
api_method = lastline_api.AnalysisClient.submit_url
call_args["url"] = request.get("url")
else:
data = request.get("data")
# Malware samples are zip-encrypted and then base64 encoded
if "malware-sample" in request:
api_method = lastline_api.LastlineCommunityAPIClient.submit_file
api_method = lastline_api.AnalysisClient.submit_file
call_args["file_data"] = __unzip(base64.b64decode(data), DEFAULT_ZIP_PASSWORD)
call_args["file_name"] = request.get("malware-sample").split("|", 1)[0]
call_args["password"] = DEFAULT_ZIP_PASSWORD
# Attachments are just base64 encoded
elif "attachment" in request:
api_method = lastline_api.LastlineCommunityAPIClient.submit_file
api_method = lastline_api.AnalysisClient.submit_file
call_args["file_data"] = base64.b64decode(data)
call_args["file_name"] = request.get("attachment")
@ -112,7 +108,7 @@ def handler(q=False):
# Make the API call
try:
api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data)
api_client = lastline_api.AnalysisClient(api_url, auth_data)
response = api_method(api_client, **call_args)
task_uuid = response.get("task_uuid")
if not task_uuid:
@ -127,7 +123,7 @@ def handler(q=False):
return misperrors
# Assemble and return
analysis_link = lastline_api.get_analysis_link(api_url, task_uuid)
analysis_link = lastline_api.get_task_link(task_uuid, analysis_url=api_url)
return {
"results": [
@ -152,12 +148,12 @@ if __name__ == "__main__":
args = parser.parse_args()
c = configparser.ConfigParser()
c.read(args.config_file)
a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name)
a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
j = json.dumps(
{
"config": a,
"url": "https://www.google.com",
"url": "https://www.google.exe.com",
}
)
print(json.dumps(handler(j), indent=4, sort_keys=True))

@ -85,8 +85,10 @@ class DomainQuery(VirusTotalParser):
whois_object = MISPObject(whois)
whois_object.add_attribute('text', type='text', value=query_result[whois])
self.misp_event.add_object(**whois_object)
siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings'])
self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings)
if 'domain_siblings' in query_result:
siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings'])
if 'subdomains' in query_result:
self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings)
self.parse_urls(query_result)
def parse_siblings(self, domain):

@ -29,8 +29,6 @@ moduleinfo = {
}
moduleconfig = [
"api_key",
"api_token",
"username",
"password",
]
@ -65,24 +63,25 @@ def handler(q=False):
# Parse the init parameters
try:
auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request)
config = request["config"]
auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config)
analysis_link = request["config"]["analysis_link"]
# The API url changes based on the analysis link host name
api_url = lastline_api.get_api_url_from_link(analysis_link)
api_url = lastline_api.get_portal_url_from_task_link(analysis_link)
except Exception as e:
misperrors["error"] = "Error parsing configuration: {}".format(e)
return misperrors
# Parse the call parameters
try:
task_uuid = lastline_api.get_uuid_from_link(analysis_link)
task_uuid = lastline_api.get_uuid_from_task_link(analysis_link)
except (KeyError, ValueError) as e:
misperrors["error"] = "Error processing input parameters: {}".format(e)
return misperrors
# Make the API calls
try:
api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data)
api_client = lastline_api.PortalClient(api_url, auth_data)
response = api_client.get_progress(task_uuid)
if response.get("completed") != 1:
raise ValueError("Analysis is not finished yet.")
@ -122,7 +121,7 @@ if __name__ == "__main__":
args = parser.parse_args()
c = configparser.ConfigParser()
c.read(args.config_file)
a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name)
a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name)
j = json.dumps(
{

Loading…
Cancel
Save