MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: Upload of STIX document with non-ascii characters 

Due to: https://github.com/psf/requests/issues/5560

TL;DR: a variable of type str passed to data in a POST request will be
silently re-encoded to ISO-8859-1, making MISP barf on the other side.
pull/786/head
Raphaël Vinot 2021-09-09 16:58:23 +02:00
parent eafbb76441
commit 253730759a
3 changed files with 131 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pymisp/api.py
View File

@ -3027,9 +3027,6 @@ class PyMISP:
else:
raise MISPServerError("please fill path or data parameter")
if isinstance(to_post, bytes):
to_post = to_post.decode()
if str(version) == '1':
url = urljoin(self.root_url, 'events/upload_stix')
response = self._prepare_request('POST', url, data=to_post, output_type='xml', content_type='xml') # type: ignore
@ -3505,21 +3502,20 @@ class PyMISP:
def __repr__(self):
return f'<{self.__class__.__name__}(url={self.root_url})'
def _prepare_request(self, request_type: str, url: str, data: Union[str, Iterable, Mapping, AbstractMISP] = {}, params: Mapping = {},
def _prepare_request(self, request_type: str, url: str, data: Union[Iterable, Mapping, AbstractMISP, bytes] = {}, params: Mapping = {},
kw_params: Mapping = {}, output_type: str = 'json', content_type: str = 'json') -> requests.Response:
'''Prepare a request for python-requests'''
if url[0] == '/':
# strip it: it will fail if MISP is in a sub directory
url = url[1:]
url = urljoin(self.root_url, url)
if data == {} or isinstance(data, str):
if data == {} or isinstance(data, bytes):
d = data
elif data:
if not isinstance(data, str): # Else, we already have a text blob to send
if isinstance(data, dict): # Else, we can directly json encode.
# Remove None values.
data = {k: v for k, v in data.items() if v is not None}
d = json.dumps(data, default=pymisp_json_default)
if isinstance(data, dict): # Else, we can directly json encode.
# Remove None values.
data = {k: v for k, v in data.items() if v is not None}
d = json.dumps(data, default=pymisp_json_default)
logger.debug(f'{request_type} - {url}')
if d is not None:

110
tests/stix1.xml-utf8 Normal file
View File

@ -0,0 +1,110 @@
<stix:STIX_Package
xmlns:cyboxCommon="http://cybox.mitre.org/common-2"
xmlns:cybox="http://cybox.mitre.org/cybox-2"
xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2"
xmlns:AccountObj="http://cybox.mitre.org/objects#AccountObject-2"
xmlns:ArtifactObj="http://cybox.mitre.org/objects#ArtifactObject-2"
xmlns:ASObj="http://cybox.mitre.org/objects#ASObject-1"
xmlns:AddressObj="http://cybox.mitre.org/objects#AddressObject-2"
xmlns:PortObj="http://cybox.mitre.org/objects#PortObject-2"
xmlns:DomainNameObj="http://cybox.mitre.org/objects#DomainNameObject-1"
xmlns:EmailMessageObj="http://cybox.mitre.org/objects#EmailMessageObject-2"
xmlns:FileObj="http://cybox.mitre.org/objects#FileObject-2"
xmlns:HTTPSessionObj="http://cybox.mitre.org/objects#HTTPSessionObject-2"
xmlns:HostnameObj="http://cybox.mitre.org/objects#HostnameObject-1"
xmlns:MutexObj="http://cybox.mitre.org/objects#MutexObject-2"
xmlns:PipeObj="http://cybox.mitre.org/objects#PipeObject-2"
xmlns:URIObj="http://cybox.mitre.org/objects#URIObject-2"
xmlns:WinRegistryKeyObj="http://cybox.mitre.org/objects#WinRegistryKeyObject-2"
xmlns:WinServiceObj="http://cybox.mitre.org/objects#WinServiceObject-2"
xmlns:NetworkConnectionObj="http://cybox.mitre.org/objects#NetworkConnectionObject-2"
xmlns:NetworkSocketObj="http://cybox.mitre.org/objects#NetworkSocketObject-2"
xmlns:SocketAddressObj="http://cybox.mitre.org/objects#SocketAddressObject-1"
xmlns:SystemObj="http://cybox.mitre.org/objects#SystemObject-2"
xmlns:ProcessObj="http://cybox.mitre.org/objects#ProcessObject-2"
xmlns:X509CertificateObj="http://cybox.mitre.org/objects#X509CertificateObject-2"
xmlns:WhoisObj="http://cybox.mitre.org/objects#WhoisObject-2"
xmlns:WinExecutableFileObj="http://cybox.mitre.org/objects#WinExecutableFileObject-2"
xmlns:UnixUserAccountObj="http://cybox.mitre.org/objects#UnixUserAccountObject-2"
xmlns:UserAccountObj="http://cybox.mitre.org/objects#UserAccountObject-2"
xmlns:WinUserAccountObj="http://cybox.mitre.org/objects#WinUserAccountObject-2"
xmlns:CustomObj="http://cybox.mitre.org/objects#CustomObject-1"
xmlns:marking="http://data-marking.mitre.org/Marking-1"
xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1"
xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1"
xmlns:et="http://stix.mitre.org/ExploitTarget-1"
xmlns:incident="http://stix.mitre.org/Incident-1"
xmlns:indicator="http://stix.mitre.org/Indicator-2"
xmlns:coa="http://stix.mitre.org/CourseOfAction-1"
xmlns:ttp="http://stix.mitre.org/TTP-1"
xmlns:ta="http://stix.mitre.org/ThreatActor-1"
xmlns:stixCommon="http://stix.mitre.org/common-1"
xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1"
xmlns:stix-ciqidentity="http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1"
xmlns:snortTM="http://stix.mitre.org/extensions/TestMechanism#Snort-1"
xmlns:stix="http://stix.mitre.org/stix-1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:xal="urn:oasis:names:tc:ciq:xal:3"
xmlns:xnl="urn:oasis:names:tc:ciq:xnl:3"
xmlns:xpil="urn:oasis:names:tc:ciq:xpil:3"
xmlns:ORGNAME="https://localhost:8443"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
xmlns:xs="http://www.w3.org/2001/XMLSchema"
id="ORGNAME:Package-fbe98511-c726-4550-bb2f-46752f4e47c7" version="1.1.1" timestamp="2021-08-24T12:53:38.320654">
<stix:STIX_Header>
<stix:Title>Export from ORGNAME MISP</stix:Title>
<stix:Package_Intent xsi:type="stixVocabs:PackageIntentVocab-1.0">Threat Report</stix:Package_Intent>
</stix:STIX_Header>
<stix:Related_Packages>
<stix:Related_Package>
<stix:Package id="ORGNAME:STIXPackage-f90bb8c1-8505-4d74-af34-3dcffec6b6d4" version="1.1.1" timestamp="2021-08-24T10:53:13">
<stix:STIX_Header>
<stix:Title>Export from ORGNAME MISP © YADA YADA</stix:Title>
<stix:Package_Intent xsi:type="stixVocabs:PackageIntentVocab-1.0">Threat Report</stix:Package_Intent>
</stix:STIX_Header>
<stix:Incidents>
<stix:Incident id="ORGNAME:Incident-f90bb8c1-8505-4d74-af34-3dcffec6b6d4" timestamp="2021-08-24T10:53:28" xsi:type='incident:IncidentType'>
<incident:Title>Test Stix</incident:Title>
<incident:External_ID source="MISP Event">612</incident:External_ID>
<incident:Time>
<incident:Incident_Discovery precision="second">2021-08-24T00:00:00</incident:Incident_Discovery>
<incident:Incident_Reported precision="second">2021-08-24T10:53:28</incident:Incident_Reported>
</incident:Time>
<incident:Reporter>
<stixCommon:Identity>
<stixCommon:Name>ORGNAME</stixCommon:Name>
</stixCommon:Identity>
</incident:Reporter>
<incident:Status xsi:type="stixVocabs:IncidentStatusVocab-1.0">New</incident:Status>
<incident:Related_Observables>
<incident:Related_Observable>
<stixCommon:Relationship>Network activity</stixCommon:Relationship>
<stixCommon:Observable id="ORGNAME:Address-0853d51f-0fe7-4d35-b3cb-b96bdbc1f0ee">
<cybox:Object id="ORGNAME:AddressObject-0853d51f-0fe7-4d35-b3cb-b96bdbc1f0ee">
<cybox:Properties xsi:type="AddressObj:AddressObjectType" category="ipv4-addr" is_source="true" is_destination="false">
<AddressObj:Address_Value condition="Equals">8.8.8.8</AddressObj:Address_Value>
</cybox:Properties>
</cybox:Object>
</stixCommon:Observable>
</incident:Related_Observable>
</incident:Related_Observables>
<incident:History>
<incident:History_Item>
<incident:Journal_Entry time_precision="second">Event Threat Level: High</incident:Journal_Entry>
</incident:History_Item>
<incident:History_Item>
<incident:Journal_Entry time_precision="second">MISP Tag: misp:tool="misp2stix"</incident:Journal_Entry>
</incident:History_Item>
</incident:History>
<incident:Information_Source>
<stixCommon:Identity>
<stixCommon:Name>ORGNAME</stixCommon:Name>
</stixCommon:Identity>
</incident:Information_Source>
</stix:Incident>
</stix:Incidents>
</stix:Package>
</stix:Related_Package>
</stix:Related_Packages>
</stix:STIX_Package>

24
tests/testlive_comprehensive.py
View File

@ -955,8 +955,9 @@ class TestComprehensive(unittest.TestCase):
response = self.user_misp_connector.add_event(event, metadata=True)
self.assertEqual(len(response.attributes), 0) # response should contains zero attributes
event.info = "New name"
event.info = "New name ©"
response = self.user_misp_connector.update_event(event, metadata=True)
self.assertEqual(response.info, event.info)
self.assertEqual(len(response.attributes), 0) # response should contains zero attributes
finally: # cleanup
self.admin_misp_connector.delete_event(event)
@ -2512,11 +2513,15 @@ class TestComprehensive(unittest.TestCase):
def test_upload_stix(self):
# FIXME https://github.com/MISP/MISP/issues/4892
try:
# r1 = self.user_misp_connector.upload_stix('tests/stix1.xml', version='1')
# print('stix 1', r1.json())
# event_stix_one = MISPEvent()
# event_stix_one.load(r.)
r1 = self.user_misp_connector.upload_stix('tests/stix1.xml-utf8', version='1')
print(r1.text)
event_stix_one = MISPEvent()
event_stix_one.load(r1.json())
# self.assertEqual(event_stix_one.attributes[0], '8.8.8.8')
self.admin_misp_connector.delete_event(event_stix_one)
bl = self.admin_misp_connector.delete_event_blocklist(event_stix_one.uuid)
self.assertTrue(bl['success'])
r2 = self.user_misp_connector.upload_stix('tests/stix2.json', version='2')
print(json.dumps(r2.json(), indent=2))
event_stix_two = MISPEvent()
@ -2528,10 +2533,11 @@ class TestComprehensive(unittest.TestCase):
bl = self.admin_misp_connector.delete_event_blocklist(event_stix_two.uuid)
self.assertTrue(bl['success'])
finally:
# try:
# self.admin_misp_connector.delete_event(event_stix_one)
# except Exception:
# pass
try:
self.admin_misp_connector.delete_event(event_stix_one)
self.admin_misp_connector.delete_event_blocklist(event_stix_one.uuid)
except Exception:
pass
try:
self.admin_misp_connector.delete_event(event_stix_two)
self.admin_misp_connector.delete_event_blocklist(event_stix_two.uuid)