Merge branch 'master' into sightingAPI

.gitignore

@@ -3,8 +3,11 @@
 *.pyc
 examples/keys.py
 examples/cudeso.py
-examples/feed-generator/output/*.json
+examples/feed-generator/output/*\.json
+examples/feed-generator/output/hashes\.csv
+examples/feed-generator/settings\.py
 build/*
 dist/*
 pymisp.egg-info/*
 .idea
+

examples/add_named_attribute.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json', debug=True)
+    return PyMISP(url, key, misp_verifycert, 'json', debug=True)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Add an attribute to an event')

examples/add_user.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json')
+    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Add a new user by setting the mandory fields.')

examples/add_user_json.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json')
+    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Add  the user described in the given json. If no file is provided, returns a json listing all the fields used to describe a user.')

examples/create_events.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json', debug=True)
+    return PyMISP(url, key, misp_verifycert, 'json', debug=True)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Create an event on MISP.')

examples/delete_user.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json')
+    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Delete the user with the given id. Keep in mind that disabling users (by setting the disabled flag via an edit) is always prefered to keep user associations to events intact.')

examples/edit_user.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json')
+    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Edit the email of the user designed by the user_id.')

examples/edit_user_json.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json')
+    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Edit the user designed by the user_id. If no file is provided, returns a json listing all the fields used to describe a user.')

examples/et2misp.py

@@ -9,14 +9,14 @@
 
 import sys, json, time, requests
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 
 et_url = 'https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt'
 et_str = 'Emerging Threats '
 
 def init_misp():
     global mymisp
-    mymisp = PyMISP(misp_url, misp_key)
+    mymisp = PyMISP(misp_url, misp_key, misp_verifycert)
 
 def load_misp_event(eid):
     global et_attr

examples/feed-generator/README.md

@@ -5,9 +5,26 @@ This python script can be used to generate a MISP feed based on an existing MISP
 # Installation
 
 ````
-git clone https://github.com/CIRCL/PyMISP
+git clone https://github.com/MISP/PyMISP.git
 cd examples/feed-generator
 cp settings-default.py settings.py
 vi settings.py #adjust your settings
 python3 generate.py
 ````
+
+# Output
+
+The generated feed will be stored in your `outputdir`.
+It contains the files:
+- `manifest.json` - containing the feed manifest (generic event information)
+- `hashes.csv` - listing the hashes of the attribute values
+- `*.json` - a large amount of `json` files 
+
+
+# Importing in MISP
+
+To import this feed into your MISP instance:
+- Sync Actions > List Feeds > Add feed
+- Fill in the form while ensuring the 'source format' is set to 'MISP Feed'
+
+For more information about feeds please read: https://misp.gitbooks.io/misp-book/content/managing-feeds/

examples/feed-generator/generate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import sys
@@ -79,15 +79,17 @@ valid_attribute_distributions = []
 
 attributeHashes = []
 
+
 def init():
     # If we have an old settings.py file then this variable won't exist
     global valid_attribute_distributions
     try:
         valid_attribute_distributions = valid_attribute_distribution_levels
-    except:
+    except Exception:
         valid_attribute_distributions = ['0', '1', '2', '3', '4', '5']
     return PyMISP(url, key, ssl)
 
+
 def recursiveExtract(container, containerType, leaf, eventUuid):
     temp = {}
     if containerType in ['Attribute', 'Object']:
@@ -118,8 +120,8 @@ def recursiveExtract(container, containerType, leaf, eventUuid):
                         temp[childType].append(processed)
     return temp
 
+
 def saveEvent(misp, uuid):
-    result = {}
     event = misp.get_event(uuid)
     if not event.get('Event'):
         print('Error while fetching event: {}'.format(event['message']))
@@ -130,11 +132,13 @@ def saveEvent(misp, uuid):
     eventFile.write(event)
     eventFile.close()
 
+
 def __blockByDistribution(element):
     if element['distribution'] not in valid_attribute_distributions:
         return True
     return False
 
+
 def saveHashes():
     if not attributeHashes:
         return False
@@ -148,7 +152,6 @@ def saveHashes():
         sys.exit('Could not create the quick hash lookup file.')
 
 
-
 def saveManifest(manifest):
     try:
         manifestFile = open(os.path.join(outputdir, 'manifest.json'), 'w')

examples/fetch_events_feed.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 from pymisp import PyMISP
 
@@ -12,7 +12,7 @@ except NameError:
     pass
     
 def init(url, key):
-    return PyMISP(url, key, False, 'json', debug=False)
+    return PyMISP(url, key, misp_verifycert, 'json', debug=False)
 
 
 if __name__ == '__main__':

examples/freetext.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 from io import open
@@ -15,7 +15,7 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
-    pymisp = PyMISP(misp_url, misp_key)
+    pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
 
     with open(args.input, 'r') as f:
         result = pymisp.freetext(args.event, f.read())

examples/misp2cef.py

@@ -7,7 +7,7 @@
 import sys
 import datetime
 from pymisp import PyMISP, MISPAttribute
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 
 cefconfig  = {"Default_Severity":1, "Device_Vendor":"MISP", "Device_Product":"MISP", "Device_Version":1}
 
@@ -45,7 +45,7 @@ def make_cef(event):
 
 def init_misp():
   global mymisp
-  mymisp = PyMISP(misp_url, misp_key)
+  mymisp = PyMISP(misp_url, misp_key, misp_verifycert)
 
 
 def echeck(r):

examples/misp2clamav.py

@@ -6,12 +6,12 @@
 
 import sys
 from pymisp import PyMISP, MISPAttribute
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 
 
 def init_misp():
     global mymisp
-    mymisp = PyMISP(misp_url, misp_key)
+    mymisp = PyMISP(misp_url, misp_key, misp_verifycert)
 
 
 def echeck(r):

examples/sharing_groups.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json')
+    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Get a list of the sharing groups from the MISP instance.')

examples/sighting.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json')
+    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Add sighting.')

examples/suricata.py

@@ -2,12 +2,12 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 
 def init(url, key):
-    return PyMISP(url, key, True)
+    return PyMISP(url, key, misp_verifycert)
 
 
 def fetch(m, all_events, event):

examples/tags.py

@@ -2,13 +2,13 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 import json
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json', True)
+    return PyMISP(url, key, misp_verifycert, 'json', True)
 
 
 def get_tags(m):

examples/up.py

@@ -2,13 +2,13 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 from io import open
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json', debug=True)
+    return PyMISP(url, key, misp_verifycert, 'json', debug=True)
 
 def up_event(m, event, content):
     with open(content, 'r') as f:

examples/users_list.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from pymisp import PyMISP
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 import argparse
 
 # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one
@@ -13,7 +13,7 @@ except NameError:
 
 
 def init(url, key):
-    return PyMISP(url, key, True, 'json')
+    return PyMISP(url, key, misp_verifycert, 'json')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Get a list of the sharing groups from the MISP instance.')

examples/warninglists.py

@@ -4,7 +4,7 @@
 from pymisp import PyMISP
 from pymisp.tools import load_warninglists
 import argparse
-from keys import misp_url, misp_key
+from keys import misp_url, misp_key, misp_verifycert
 
 
 if __name__ == '__main__':
@@ -18,5 +18,5 @@ if __name__ == '__main__':
     if args.package:
         print(load_warninglists.from_package())
     elif args.remote:
-        pm = PyMISP(misp_url, misp_key)
+        pm = PyMISP(misp_url, misp_key, misp_verifycert)
         print(load_warninglists.from_instance(pm))

tests/mispevent_testfiles/event_obj_attr_tag.json

@@ -51,7 +51,7 @@
         "name": "url",
         "sharing_group_id": "0",
         "template_uuid": "60efb77b-40b5-4c46-871b-ed1ed999fce5",
-        "template_version": "6",
+        "template_version": "7",
         "uuid": "b"
       }
     ]

tests/mispevent_testfiles/malware_exist.json

@@ -39,7 +39,7 @@
                 "meta-category": "file",
                 "description": "File object describing a file with meta-information",
                 "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
-                "template_version": "7",
+                "template_version": "6",
                 "event_id": "6719",
                 "uuid": "5a4e4ffe-4cb8-48b1-bd5c-48fb950d210f",
                 "timestamp": "1515081726",

tests/mispevent_testfiles/shadow.json

@@ -112,7 +112,7 @@
         "name": "file",
         "sharing_group_id": "0",
         "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
-        "template_version": "7",
+        "template_version": "8",
         "timestamp": "1514975928",
         "uuid": "5a4cb2b8-7958-4323-852c-4d2a950d210f"
       }

tests/testlive_comprehensive.py

@@ -836,8 +836,10 @@ class TestComprehensive(unittest.TestCase):
 
     def test_taxonomies(self):
         # Make sure we're up-to-date
-        self.admin_misp_connector.update_taxonomies()
         r = self.admin_misp_connector.update_taxonomies()
+        print(r)
+        r = self.admin_misp_connector.update_taxonomies()
+        print(r)
         self.assertEqual(r['name'], 'All taxonomy libraries are up to date already.')
         # Get list
         taxonomies = self.admin_misp_connector.get_taxonomies_list()