@ -30,19 +30,21 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT , NEGLIGENCE OR OTHER TORTIOUS ACTION , ARISING OUT
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE .
"""
import ipaddress
import abc
import logging
import io
import ipaddress
import pymisp
import re
import requests
import pymisp
from urllib import parse
DEFAULT_LASTLINE_API = " https://user.lastline.com/papi "
DEFAULT_LL_PORTAL_API_URL = " https://user.lastline.com/papi "
DEFAULT_LL_ANALYSIS_API_URL = " https://analysis.lastline.com "
HOSTED_LASTLINE _DOMAINS = frozenset ( [
LL_ HOSTED_DOMAINS= frozenset ( [
" user.lastline.com " ,
" user.emea.lastline.com " ,
] )
@ -53,61 +55,66 @@ def purge_none(d):
return { k : v for k , v in d . items ( ) if v is not None }
def get_analysis_link ( api_url , uuid ) :
def get_task_link ( uuid , analysis_url = None , portal_url = None ) :
"""
Get the analysis link of a task given the task uuid .
Get the task link given the task uuid and at least one API url .
: param str api_url : the URL
: param str uuid : the task uuid
: param str | None analysis_url : the URL to the analysis API endpoint
: param str | None portal_url : the URL to the portal API endpoint
: rtype : str
: return : the analysis link
: return : the task link
: raises ValueError : if not enough parameters have been provided
"""
if not analysis_url and not portal_url :
raise ValueError ( " Neither analysis URL or portal URL have been specified " )
if analysis_url :
portal_url = " {} /papi " . format ( analysis_url . replace ( " analysis. " , " user. " ) )
portal_url_path = " ../portal#/analyst/task/ {} /overview " . format ( uuid )
analysis_link = parse . urljoin ( api_url , portal_url_path )
return analysis_link
return parse . urljoin ( portal_url , portal_url_path )
def get_portal_url_from_task_link ( task_link ) :
"""
Return the portal API url related to the provided task link .
: param str task_link : a link
: rtype : str
: return : the portal API url
"""
parsed_uri = parse . urlparse ( task_link )
return " {uri.scheme} :// {uri.netloc} /papi " . format ( uri = parsed_uri )
def get_uuid_from_link ( analysis_link ) :
def get_uuid_from_task_link ( task_link ) :
"""
Return task uuid from link or raise ValueError exception .
Return the uuid from a task link .
: param str analysis_link : a link
: param str task _link: a link
: rtype : str
: return : the uuid
: raises ValueError : if the link contains not task uuid
"""
try :
return re . findall ( " [a-fA-F0-9] {32} " , analysis _link) [ 0 ]
return re . findall ( " [a-fA-F0-9] {32} " , task _link) [ 0 ]
except IndexError :
raise ValueError ( " Link does not contain a valid task uuid " )
def is_analysis_hosted ( analysis _link ) :
def is_task_hosted ( task _link ) :
"""
Return whether the analysis link is pointing to a hosted submission .
Return whether the portal link is pointing to a hosted submission .
: param str analysis _link: a link
: param str task _link: a link
: rtype : boolean
: return : whether the link is hosted
: return : whether the link points to a hosted analysis
"""
for domain in HOSTED_LASTLINE _DOMAINS :
if domain in analysis _link:
for domain in LL_ HOSTED_DOMAINS:
if domain in task _link:
return True
return False
def get_api_url_from_link ( analysis_link ) :
"""
Return the API url related to the provided analysis link .
: param str analysis_link : a link
: rtype : str
: return : the API url
"""
parsed_uri = parse . urlparse ( analysis_link )
return " {uri.scheme} :// {uri.netloc} /papi " . format ( uri = parsed_uri )
class InvalidArgument ( Exception ) :
""" Error raised invalid. """
@ -135,141 +142,13 @@ class ApiError(Error):
return " {} {} " . format ( self . error_msg , error_code )
class LastlineResultBaseParser ( object ) :
"""
This is a parser to extract * basic * information from a Lastline result dictionary .
Note : this is a * version 0 * : the information we extract is merely related to the behaviors and
the HTTP connections . Further iterations will include host activity such as files , mutexes ,
registry keys , strings , etc .
"""
def __init__ ( self ) :
""" Constructor. """
self . misp_event = None
@staticmethod
def _get_mitre_techniques ( result ) :
return [
" misp-galaxy:mitre-attack-pattern= \" {} - {} \" " . format ( w [ 0 ] , w [ 1 ] )
for w in sorted ( set ( [
( y [ " id " ] , y [ " name " ] )
for x in result . get ( " malicious_activity " , [ ] )
for y in result . get ( " activity_to_mitre_techniques " , { } ) . get ( x , [ ] )
] ) )
]
def parse ( self , analysis_link , result ) :
"""
Parse the analysis result into a MISP event .
: param str analysis_link : the analysis link
: param dict [ str , any ] result : the JSON returned by the analysis client .
: rtype : MISPEvent
: return : some results that can be consumed by MIPS .
"""
self . misp_event = pymisp . MISPEvent ( )
# Add analysis subject info
if " url " in result [ " analysis_subject " ] :
o = pymisp . MISPObject ( " url " )
o . add_attribute ( " url " , result [ " analysis_subject " ] [ " url " ] )
else :
o = pymisp . MISPObject ( " file " )
o . add_attribute ( " md5 " , type = " md5 " , value = result [ " analysis_subject " ] [ " md5 " ] )
o . add_attribute ( " sha1 " , type = " sha1 " , value = result [ " analysis_subject " ] [ " sha1 " ] )
o . add_attribute ( " sha256 " , type = " sha256 " , value = result [ " analysis_subject " ] [ " sha256 " ] )
o . add_attribute (
" mimetype " ,
type = " mime-type " ,
value = result [ " analysis_subject " ] [ " mime_type " ]
)
self . misp_event . add_object ( o )
# Add HTTP requests from url analyses
network_dict = result . get ( " report " , { } ) . get ( " analysis " , { } ) . get ( " network " , { } )
for request in network_dict . get ( " requests " , [ ] ) :
parsed_uri = parse . urlparse ( request [ " url " ] )
o = pymisp . MISPObject ( name = ' http-request ' )
o . add_attribute ( ' host ' , parsed_uri . netloc )
o . add_attribute ( ' method ' , " GET " )
o . add_attribute ( ' uri ' , request [ " url " ] )
o . add_attribute ( " ip " , request [ " ip " ] )
self . misp_event . add_object ( o )
# Add network behaviors from files
for subject in result . get ( " report " , { } ) . get ( " analysis_subjects " , [ ] ) :
# Add DNS requests
for dns_query in subject . get ( " dns_queries " , [ ] ) :
hostname = dns_query . get ( " hostname " )
# Skip if it is an IP address
try :
if hostname == " wpad " :
continue
_ = ipaddress . ip_address ( hostname )
continue
except ValueError :
pass
o = pymisp . MISPObject ( name = ' dns-record ' )
o . add_attribute ( ' queried-domain ' , hostname )
self . misp_event . add_object ( o )
# Add HTTP conversations (as network connection and as http request)
for http_conversation in subject . get ( " http_conversations " , [ ] ) :
o = pymisp . MISPObject ( name = " network-connection " )
o . add_attribute ( " ip-src " , http_conversation [ " src_ip " ] )
o . add_attribute ( " ip-dst " , http_conversation [ " dst_ip " ] )
o . add_attribute ( " src-port " , http_conversation [ " src_port " ] )
o . add_attribute ( " dst-port " , http_conversation [ " dst_port " ] )
o . add_attribute ( " hostname-dst " , http_conversation [ " dst_host " ] )
o . add_attribute ( " layer3-protocol " , " IP " )
o . add_attribute ( " layer4-protocol " , " TCP " )
o . add_attribute ( " layer7-protocol " , " HTTP " )
self . misp_event . add_object ( o )
method , path , http_version = http_conversation [ " url " ] . split ( " " )
if http_conversation [ " dst_port " ] == 80 :
uri = " http:// {} {} " . format ( http_conversation [ " dst_host " ] , path )
else :
uri = " http:// {} : {} {} " . format (
http_conversation [ " dst_host " ] ,
http_conversation [ " dst_port " ] ,
path
)
o = pymisp . MISPObject ( name = ' http-request ' )
o . add_attribute ( ' host ' , http_conversation [ " dst_host " ] )
o . add_attribute ( ' method ' , method )
o . add_attribute ( ' uri ' , uri )
o . add_attribute ( ' ip ' , http_conversation [ " dst_ip " ] )
self . misp_event . add_object ( o )
# Add sandbox info like score and sandbox type
o = pymisp . MISPObject ( name = " sandbox-report " )
sandbox_type = " saas " if is_analysis_hosted ( analysis_link ) else " on-premise "
o . add_attribute ( " score " , result [ " score " ] )
o . add_attribute ( " sandbox-type " , sandbox_type )
o . add_attribute ( " {} -sandbox " . format ( sandbox_type ) , " lastline " )
o . add_attribute ( " permalink " , analysis_link )
self . misp_event . add_object ( o )
# Add behaviors
o = pymisp . MISPObject ( name = " sb-signature " )
o . add_attribute ( " software " , " Lastline " )
for activity in result . get ( " malicious_activity " , [ ] ) :
a = pymisp . MISPAttribute ( )
a . from_dict ( type = " text " , value = activity )
o . add_attribute ( " signature " , * * a )
self . misp_event . add_object ( o )
# Add mitre techniques
for technique in self . _get_mitre_techniques ( result ) :
self . misp_event . add_tag ( technique )
class LastlineAbstractClient ( abc . ABC ) :
""" " A very basic HTTP client providing basic functionality. """
__metaclass__ = abc . ABCMeta
class LastlineCommunityHTTPClient ( object ) :
""" " A very basic HTTP client providing basic functionality. """
SUB_APIS = ( ' analysis ' , ' authentication ' , ' knowledgebase ' , ' login ' )
FORMATS = [ " json " , " xml " ]
@classmethod
def sanitize_login_params ( cls , api_key , api_token , username , password ) :
@ -286,7 +165,7 @@ class LastlineCommunityHTTPClient(object):
"""
if api_key and api_token :
return {
" api_ key" : api_key ,
" key " : api_key ,
" api_token " : api_token ,
}
elif username and password :
@ -297,6 +176,21 @@ class LastlineCommunityHTTPClient(object):
else :
raise InvalidArgument ( " Arguments provided do not contain valid data " )
@classmethod
def get_login_params_from_dict ( cls , d ) :
"""
Get the module configuration from a ConfigParser object .
: param dict [ str , str ] d : the dictionary
: rtype : dict [ str , str ]
: return : the parsed configuration
"""
api_key = d . get ( " key " )
api_token = d . get ( " api_token " )
username = d . get ( " username " )
password = d . get ( " password " )
return cls . sanitize_login_params ( api_key , api_token , username , password )
@classmethod
def get_login_params_from_conf ( cls , conf , section_name ) :
"""
@ -307,26 +201,24 @@ class LastlineCommunityHTTPClient(object):
: rtype : dict [ str , str ]
: return : the parsed configuration
"""
api_key = conf . get ( section_name , " api_ key" , fallback = None )
api_key = conf . get ( section_name , " key " , fallback = None )
api_token = conf . get ( section_name , " api_token " , fallback = None )
username = conf . get ( section_name , " username " , fallback = None )
password = conf . get ( section_name , " password " , fallback = None )
return cls . sanitize_login_params ( api_key , api_token , username , password )
@classmethod
def get_login_params_from_request ( cls , request ) :
def load_from_conf ( cls , conf , section_name ) :
"""
Get the module configuration from a ConfigParser object .
Load client from a ConfigParser object .
: param dict [ str , any ] request : the request object
: rtype : dict [ str , str ]
: return : the parsed configuration
: param ConfigParser conf : the conf object
: param str section_name : the section name
: rtype : T < - LastlineAbstractClient
: return : the loaded client
"""
api_key = request . get ( " config " , { } ) . get ( " api_key " )
api_token = request . get ( " config " , { } ) . get ( " api_token " )
username = request . get ( " config " , { } ) . get ( " username " )
password = request . get ( " config " , { } ) . get ( " password " )
return cls . sanitize_login_params ( api_key , api_token , username , password )
url = conf . get ( section_name , " url " )
return cls ( url , cls . get_login_params_from_conf ( conf , section_name ) )
def __init__ ( self , api_url , login_params , timeout = 60 , verify_ssl = True ) :
"""
@ -337,39 +229,23 @@ class LastlineCommunityHTTPClient(object):
: param int timeout : the timeout
: param boolean verify_ssl : whether to verify the SSL certificate
"""
self . __ url = api_url
self . __ login_params = login_params
self . __ timeout = timeout
self . __ verify_ssl = verify_ssl
self . __ session = None
self . __ logger = logging . getLogger ( __name__ )
self . _url = api_url
self . _login_params = login_params
self . _timeout = timeout
self . _verify_ssl = verify_ssl
self . _session = None
self . _logger = logging . getLogger ( __name__ )
def __login ( self ) :
@abc . abstractmethod
def _login ( self ) :
""" Login using account-based or key-based methods. """
if self . __session is None :
self . __session = requests . session ( )
login_url = " / " . join ( [ self . __url , " login " ] )
try :
response = self . __session . request (
method = " POST " ,
url = login_url ,
data = self . __login_params ,
verify = self . __verify_ssl ,
timeout = self . __timeout ,
proxies = None ,
)
except requests . RequestException as e :
raise CommunicationError ( e )
self . __handle_response ( response )
def __is_logged_in ( self ) :
def _is_logged_in ( self ) :
""" Return whether we have an active session. """
return self . __ session is not None
return self . _session is not None
@staticmethod
def __ parse_response ( response ) :
def _parse_response ( response ) :
"""
Parse the response .
@ -394,7 +270,7 @@ class LastlineCommunityHTTPClient(object):
except ValueError as e :
return None , Error ( " Response not json {} " . format ( e ) )
def __ handle_response ( self , response , raw = False ) :
def _handle_response ( self , response , raw = False ) :
"""
Check a response for issues and parse the return .
@ -408,7 +284,7 @@ class LastlineCommunityHTTPClient(object):
try :
response . raise_for_status ( )
except requests . RequestException as e :
_ , err = self . __ parse_response ( response )
_ , err = self . _parse_response ( response )
if isinstance ( err , ApiError ) :
err_msg = " {} : {} " . format ( e , err . error_msg )
else :
@ -418,21 +294,57 @@ class LastlineCommunityHTTPClient(object):
# Otherwise return the data (either parsed or not) but reraise if we have an API error
if raw :
return response . content
data , err = self . __ parse_response ( response )
data , err = self . _parse_response ( response )
if err :
raise err
return data
def _build_url ( self , sub_api , parts , requested_format = " json " ) :
if sub_api not in self . SUB_APIS :
raise InvalidArgument ( sub_api )
if requested_format not in self . FORMATS :
raise InvalidArgument ( requested_format )
num_parts = 2 + len ( parts )
pattern = " / " . join ( [ " %s " ] * num_parts ) + " . %s "
params = [ self . _url , sub_api ] + parts + [ requested_format ]
return pattern % tuple ( params )
def post ( self , module , function , params = None , data = None , files = None , fmt = " json " ) :
if isinstance ( function , list ) :
functions = function
else :
functions = [ function ] if function else [ ]
url = self . _build_url ( module , functions , requested_format = fmt )
return self . do_request (
url = url ,
method = " POST " ,
params = params ,
data = data ,
files = files ,
fmt = fmt ,
)
def get ( self , module , function , params = None , fmt = " json " ) :
if isinstance ( function , list ) :
functions = function
else :
functions = [ function ] if function else [ ]
url = self . _build_url ( module , functions , requested_format = fmt )
return self . do_request (
url = url ,
method = " GET " ,
params = params ,
fmt = fmt ,
)
def do_request (
self ,
method ,
module ,
function ,
url ,
params = None ,
data = None ,
files = None ,
url = None ,
fmt = " JSON " ,
fmt = " json " ,
raw = False ,
raw_response = False ,
headers = None ,
@ -443,7 +355,7 @@ class LastlineCommunityHTTPClient(object):
if fmt :
fmt = fmt . lower ( ) . strip ( )
if fmt not in [ " json " , " xml " , " html " , " pdf " ] :
if fmt not in self . FORMATS :
raise InvalidArgument ( " Only json, xml, html and pdf supported " )
elif not raw :
raise InvalidArgument ( " Unformatted response requires raw=True " )
@ -454,35 +366,19 @@ class LastlineCommunityHTTPClient(object):
if method not in [ " POST " , " GET " ] :
raise InvalidArgument ( " Only POST and GET supported " )
function = function . strip ( " / " )
if not function :
raise InvalidArgument ( " No function provided " )
# Login after we verified that all arguments are fine
if not self . __is_logged_in ( ) :
self . __login ( )
url_parts = [ url or self . __url ]
module = module . strip ( " / " )
if module :
url_parts . append ( module )
if fmt :
function_part = " %s . %s " % ( function , fmt )
else :
function_part = function
url_parts . append ( function_part )
url = " / " . join ( url_parts )
if not self . _is_logged_in ( ) :
self . _login ( )
try :
try :
response = self . __ session . request (
response = self . _session . request (
method = method ,
url = url ,
data = data ,
params = params ,
files = files ,
verify = self . __ verify_ssl ,
timeout = self . __ timeout ,
verify = self . _verify_ssl ,
timeout = self . _timeout ,
stream = stream_response ,
headers = headers ,
)
@ -491,7 +387,7 @@ class LastlineCommunityHTTPClient(object):
if raw_response :
return response
return self . __ handle_response ( response , raw )
return self . _handle_response ( response , raw )
except Error as e :
raise e
@ -500,38 +396,177 @@ class LastlineCommunityHTTPClient(object):
raise e
class LastlineCommunityAPIClient ( object ) :
""" " A very basic API client providing basic functionality. """
class AnalysisClient ( LastlineAbstractClient ) :
def __init__ ( self , api_url , login_params ) :
def _login ( self ) :
"""
Instantiate the API client .
Creates auth session for malscape - service .
: param str api_url : the URL to the API server
: param dict [ str , str ] login_params : the login parameters
Credentials are ' key ' and ' api_token ' .
"""
self . _client = LastlineCommunityHTTPClient ( api_url , login_params )
self . _logger = logging . getLogger ( __name__ )
if self . _session is None :
self . _session = requests . session ( )
url = self . _build_url ( " authentication " , [ " login " ] )
self . do_request ( " POST " , url , params = purge_none ( self . _login_params ) )
def _post ( self , module , function , params = None , data = None , files = None , fmt = " JSON " ) :
return self . _client . do_request (
method = " POST " ,
module = module ,
function = function ,
params = params ,
data = data ,
files = files ,
fmt = fmt ,
)
def get_progress ( self , uuid ) :
"""
Get the completion progress of a given task .
: param str uuid : the unique identifier of the submitted task
: rtype : dict [ str , int ]
: return : a dictionary like the the following :
{
" completed " : 1 ,
" progress " : 100
}
"""
url = self . _build_url ( ' analysis ' , [ ' get_progress ' ] )
params = { ' uuid ' : uuid }
return self . do_request ( " POST " , url , params = params )
def _get ( self , module , function , params = None , fmt = " JSON " ) :
return self . _client . do_request (
method = " GET " ,
module = module ,
function = function ,
params = params ,
fmt = fmt ,
)
def get_result ( self , uuid ) :
"""
Get report results for a given task .
: param str uuid : the unique identifier of the submitted task
: rtype : dict [ str , any ]
: return : a dictionary like the the following :
{
" completed " : 1 ,
" progress " : 100
}
"""
# better: use 'get_results()' but that would break
# backwards-compatibility
url = self . _build_url ( ' analysis ' , [ ' get ' ] )
params = { ' uuid ' : uuid }
return self . do_request ( " GET " , url , params = params )
def submit_file (
self ,
file_data ,
file_name = None ,
password = None ,
analysis_env = None ,
allow_network_traffic = True ,
analysis_timeout = None ,
bypass_cache = False ,
) :
"""
Upload a file to be analyzed .
: param bytes file_data : the data as a byte sequence
: param str | None file_name : if set , represents the name of the file to submit
: param str | None password : if set , use it to extract the sample
: param str | None analysis_env : if set , e . g windowsxp
: param boolean allow_network_traffic : if set to False , deny network connections
: param int | None analysis_timeout : if set limit the duration of the analysis
: param boolean bypass_cache : whether to re - process a file ( requires special permissions )
: rtype : dict [ str , any ]
: return : a dictionary in the following form if the analysis is already available :
{
" submission " : " 2019-11-17 09:33:23 " ,
" child_tasks " : [ . . . ] ,
" reports " : [ . . . ] ,
" submission_timestamp " : " 2019-11-18 16:11:04 " ,
" task_uuid " : " 86097fb8e4cd00100464cb001b97ecbe " ,
" score " : 0 ,
" analysis_subject " : {
" url " : " https://www.google.com "
} ,
" last_submission_timestamp " : " 2019-11-18 16:11:04 "
}
OR the following if the analysis is still pending :
{
" submission_timestamp " : " 2019-11-18 13:59:25 " ,
" task_uuid " : " f3c0ae115d51001017ff8da768fa6049 " ,
}
"""
file_stream = io . BytesIO ( file_data )
api_url = self . _build_url ( " analysis " , [ " submit " , " file " ] )
params = purge_none ( {
" bypass_cache " : bypass_cache and 1 or None ,
" analysis_timeout " : analysis_timeout ,
" analysis_env " : analysis_env ,
" allow_network_traffic " : allow_network_traffic and 1 or None ,
" filename " : file_name ,
" password " : password ,
" full_report_score " : - 1 ,
} )
files = purge_none ( {
# If an explicit filename was provided, we can pass it down to
# python-requests to use it in the multipart/form-data. This avoids
# having python-requests trying to guess the filename based on stream
# attributes.
#
# The problem with this is that, if the filename is not ASCII, then
# this triggers a bug in flask/werkzeug which means the file is
# thrown away. Thus, we just force an ASCII name
" file " : ( ' dummy-ascii-name-for-file-param ' , file_stream ) ,
} )
return self . do_request ( " POST " , api_url , params = params , files = files )
def submit_url (
self ,
url ,
referer = None ,
user_agent = None ,
bypass_cache = False ,
) :
"""
Upload an URL to be analyzed .
: param str url : the url to analyze
: param str | None referer : the referer
: param str | None user_agent : the user agent
: param boolean bypass_cache : bypass_cache
: rtype : dict [ str , any ]
: return : a dictionary like the following if the analysis is already available :
{
" submission " : " 2019-11-17 09:33:23 " ,
" child_tasks " : [ . . . ] ,
" reports " : [ . . . ] ,
" submission_timestamp " : " 2019-11-18 16:11:04 " ,
" task_uuid " : " 86097fb8e4cd00100464cb001b97ecbe " ,
" score " : 0 ,
" analysis_subject " : {
" url " : " https://www.google.com "
} ,
" last_submission_timestamp " : " 2019-11-18 16:11:04 "
}
OR the following if the analysis is still pending :
{
" submission_timestamp " : " 2019-11-18 13:59:25 " ,
" task_uuid " : " f3c0ae115d51001017ff8da768fa6049 " ,
}
"""
api_url = self . _build_url ( " analysis " , [ " submit " , " url " ] )
params = purge_none ( {
" url " : url ,
" referer " : referer ,
" bypass_cache " : bypass_cache and 1 or None ,
" user_agent " : user_agent or None ,
} )
return self . do_request ( " POST " , api_url , params = params )
class PortalClient ( LastlineAbstractClient ) :
def _login ( self ) :
"""
Login using account - based or key - based methods .
Credentials are ' username ' and ' password '
"""
if self . _session is None :
self . _session = requests . session ( )
self . post ( " login " , function = None , data = self . _login_params )
def get_progress ( self , uuid , analysis_instance = None ) :
"""
@ -547,7 +582,7 @@ class LastlineCommunityAPIClient(object):
}
"""
params = purge_none ( { " uuid " : uuid , " analysis_instance " : analysis_instance } )
return self . _ get( " analysis " , " get_progress " , params = params )
return self . get ( " analysis " , " get_progress " , params = params )
def get_result ( self , uuid , analysis_instance = None ) :
"""
@ -569,7 +604,7 @@ class LastlineCommunityAPIClient(object):
" report_format " : " json " ,
}
)
return self . _ get( " analysis " , " get_result " , params = params )
return self . get ( " analysis " , " get_result " , params = params )
def submit_url (
self ,
@ -615,7 +650,7 @@ class LastlineCommunityAPIClient(object):
" user_agent " : user_agent
}
)
return self . _ post( module = " analysis " , function = " submit_url " , params = params )
return self . post ( " analysis " , " submit_url " , params = params )
def submit_file (
self ,
@ -670,4 +705,137 @@ class LastlineCommunityAPIClient(object):
}
)
files = { " file " : ( file_name , file_data , " application/octet-stream " ) }
return self . _post ( module = " analysis " , function = " submit_file " , params = params , files = files )
return self . post ( " analysis " , " submit_file " , params = params , files = files )
class LastlineResultBaseParser ( object ) :
"""
This is a parser to extract * basic * information from a Lastline result dictionary .
Note : this is a * version 0 * : the information we extract is merely related to the behaviors and
the HTTP connections . Further iterations will include host activity such as files , mutexes ,
registry keys , strings , etc .
"""
def __init__ ( self ) :
""" Constructor. """
self . misp_event = None
@staticmethod
def _get_mitre_techniques ( result ) :
return [
" misp-galaxy:mitre-attack-pattern= \" {} - {} \" " . format ( w [ 0 ] , w [ 1 ] )
for w in sorted ( set ( [
( y [ " id " ] , y [ " name " ] )
for x in result . get ( " malicious_activity " , [ ] )
for y in result . get ( " activity_to_mitre_techniques " , { } ) . get ( x , [ ] )
] ) )
]
def parse ( self , analysis_link , result ) :
"""
Parse the analysis result into a MISP event .
: param str analysis_link : the analysis link
: param dict [ str , any ] result : the JSON returned by the analysis client .
: rtype : MISPEvent
: return : some results that can be consumed by MIPS .
"""
self . misp_event = pymisp . MISPEvent ( )
# Add analysis subject info
if " url " in result [ " analysis_subject " ] :
o = pymisp . MISPObject ( " url " )
o . add_attribute ( " url " , result [ " analysis_subject " ] [ " url " ] )
else :
o = pymisp . MISPObject ( " file " )
o . add_attribute ( " md5 " , type = " md5 " , value = result [ " analysis_subject " ] [ " md5 " ] )
o . add_attribute ( " sha1 " , type = " sha1 " , value = result [ " analysis_subject " ] [ " sha1 " ] )
o . add_attribute ( " sha256 " , type = " sha256 " , value = result [ " analysis_subject " ] [ " sha256 " ] )
o . add_attribute (
" mimetype " ,
type = " mime-type " ,
value = result [ " analysis_subject " ] [ " mime_type " ]
)
self . misp_event . add_object ( o )
# Add HTTP requests from url analyses
network_dict = result . get ( " report " , { } ) . get ( " analysis " , { } ) . get ( " network " , { } )
for request in network_dict . get ( " requests " , [ ] ) :
parsed_uri = parse . urlparse ( request [ " url " ] )
o = pymisp . MISPObject ( name = ' http-request ' )
o . add_attribute ( ' host ' , parsed_uri . netloc )
o . add_attribute ( ' method ' , " GET " )
o . add_attribute ( ' uri ' , request [ " url " ] )
o . add_attribute ( " ip " , request [ " ip " ] )
self . misp_event . add_object ( o )
# Add network behaviors from files
for subject in result . get ( " report " , { } ) . get ( " analysis_subjects " , [ ] ) :
# Add DNS requests
for dns_query in subject . get ( " dns_queries " , [ ] ) :
hostname = dns_query . get ( " hostname " )
# Skip if it is an IP address
try :
if hostname == " wpad " :
continue
_ = ipaddress . ip_address ( hostname )
continue
except ValueError :
pass
o = pymisp . MISPObject ( name = ' dns-record ' )
o . add_attribute ( ' queried-domain ' , hostname )
self . misp_event . add_object ( o )
# Add HTTP conversations (as network connection and as http request)
for http_conversation in subject . get ( " http_conversations " , [ ] ) :
o = pymisp . MISPObject ( name = " network-connection " )
o . add_attribute ( " ip-src " , http_conversation [ " src_ip " ] )
o . add_attribute ( " ip-dst " , http_conversation [ " dst_ip " ] )
o . add_attribute ( " src-port " , http_conversation [ " src_port " ] )
o . add_attribute ( " dst-port " , http_conversation [ " dst_port " ] )
o . add_attribute ( " hostname-dst " , http_conversation [ " dst_host " ] )
o . add_attribute ( " layer3-protocol " , " IP " )
o . add_attribute ( " layer4-protocol " , " TCP " )
o . add_attribute ( " layer7-protocol " , " HTTP " )
self . misp_event . add_object ( o )
method , path , http_version = http_conversation [ " url " ] . split ( " " )
if http_conversation [ " dst_port " ] == 80 :
uri = " http:// {} {} " . format ( http_conversation [ " dst_host " ] , path )
else :
uri = " http:// {} : {} {} " . format (
http_conversation [ " dst_host " ] ,
http_conversation [ " dst_port " ] ,
path
)
o = pymisp . MISPObject ( name = ' http-request ' )
o . add_attribute ( ' host ' , http_conversation [ " dst_host " ] )
o . add_attribute ( ' method ' , method )
o . add_attribute ( ' uri ' , uri )
o . add_attribute ( ' ip ' , http_conversation [ " dst_ip " ] )
self . misp_event . add_object ( o )
# Add sandbox info like score and sandbox type
o = pymisp . MISPObject ( name = " sandbox-report " )
sandbox_type = " saas " if is_task_hosted ( analysis_link ) else " on-premise "
o . add_attribute ( " score " , result [ " score " ] )
o . add_attribute ( " sandbox-type " , sandbox_type )
o . add_attribute ( " {} -sandbox " . format ( sandbox_type ) , " lastline " )
o . add_attribute ( " permalink " , analysis_link )
self . misp_event . add_object ( o )
# Add behaviors
o = pymisp . MISPObject ( name = " sb-signature " )
o . add_attribute ( " software " , " Lastline " )
for activity in result . get ( " malicious_activity " , [ ] ) :
a = pymisp . MISPAttribute ( )
a . from_dict ( type = " text " , value = activity )
o . add_attribute ( " signature " , * * a )
self . misp_event . add_object ( o )
# Add mitre techniques
for technique in self . _get_mitre_techniques ( result ) :
self . misp_event . add_tag ( technique )
chrisr3d
3 years ago