Merge branch 'master' of github.com:MISP/PyMISP

.gitchangelog.rc

@@ -0,0 +1,192 @@
+##
+## Format
+##
+##   ACTION: [AUDIENCE:] COMMIT_MSG [!TAG ...]
+##
+## Description
+##
+##   ACTION is one of 'chg', 'fix', 'new'
+##
+##       Is WHAT the change is about.
+##
+##       'chg' is for refactor, small improvement, cosmetic changes...
+##       'fix' is for bug fixes
+##       'new' is for new features, big improvement
+##
+##   AUDIENCE is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'
+##
+##       Is WHO is concerned by the change.
+##
+##       'dev'  is for developpers (API changes, refactors...)
+##       'usr'  is for final users (UI changes)
+##       'pkg'  is for packagers   (packaging changes)
+##       'test' is for testers     (test only related changes)
+##       'doc'  is for doc guys    (doc only changes)
+##
+##   COMMIT_MSG is ... well ... the commit message itself.
+##
+##   TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic'
+##
+##       They are preceded with a '!' or a '@' (prefer the former, as the
+##       latter is wrongly interpreted in github.) Commonly used tags are:
+##
+##       'refactor' is obviously for refactoring code only
+##       'minor' is for a very meaningless change (a typo, adding a comment)
+##       'cosmetic' is for cosmetic driven change (re-indentation, 80-col...)
+##       'wip' is for partial functionality but complete subfunctionality.
+##
+## Example:
+##
+##   new: usr: support of bazaar implemented
+##   chg: re-indentend some lines !cosmetic
+##   new: dev: updated code to be compatible with last version of killer lib.
+##   fix: pkg: updated year of licence coverage.
+##   new: test: added a bunch of test around user usability of feature X.
+##   fix: typo in spelling my name in comment. !minor
+##
+##   Please note that multi-line commit message are supported, and only the
+##   first line will be considered as the "summary" of the commit message. So
+##   tags, and other rules only applies to the summary.  The body of the commit
+##   message will be displayed in the changelog without reformatting.
+
+
+##
+## ``ignore_regexps`` is a line of regexps
+##
+## Any commit having its full commit message matching any regexp listed here
+## will be ignored and won't be reported in the changelog.
+##
+ignore_regexps = [
+        r'@minor', r'!minor',
+        r'@cosmetic', r'!cosmetic',
+        r'@refactor', r'!refactor',
+        r'@wip', r'!wip',
+        r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[p|P]kg:',
+        r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[d|D]ev:',
+        r'^(.{3,3}\s*:)?\s*[fF]irst commit.?\s*$',
+  ]
+
+
+## ``section_regexps`` is a list of 2-tuples associating a string label and a
+## list of regexp
+##
+## Commit messages will be classified in sections thanks to this. Section
+## titles are the label, and a commit is classified under this section if any
+## of the regexps associated is matching.
+##
+section_regexps = [
+    ('New', [
+        r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+     ]),
+    ('Changes', [
+        r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+     ]),
+    ('Fix', [
+        r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+     ]),
+
+    ('Other', None ## Match all lines
+     ),
+
+]
+
+
+## ``body_process`` is a callable
+##
+## This callable will be given the original body and result will
+## be used in the changelog.
+##
+## Available constructs are:
+##
+##   - any python callable that take one txt argument and return txt argument.
+##
+##   - ReSub(pattern, replacement): will apply regexp substitution.
+##
+##   - Indent(chars="  "): will indent the text with the prefix
+##     Please remember that template engines gets also to modify the text and
+##     will usually indent themselves the text if needed.
+##
+##   - Wrap(regexp=r"\n\n"): re-wrap text in separate paragraph to fill 80-Columns
+##
+##   - noop: do nothing
+##
+##   - ucfirst: ensure the first letter is uppercase.
+##     (usually used in the ``subject_process`` pipeline)
+##
+##   - final_dot: ensure text finishes with a dot
+##     (usually used in the ``subject_process`` pipeline)
+##
+##   - strip: remove any spaces before or after the content of the string
+##
+## Additionally, you can `pipe` the provided filters, for instance:
+#body_process = Wrap(regexp=r'\n(?=\w+\s*:)') | Indent(chars="  ")
+#body_process = Wrap(regexp=r'\n(?=\w+\s*:)')
+#body_process = noop
+body_process = ReSub(r'((^|\n)[A-Z]\w+(-\w+)*: .*(\n\s+.*)*)+$', r'') | strip
+
+
+## ``subject_process`` is a callable
+##
+## This callable will be given the original subject and result will
+## be used in the changelog.
+##
+## Available constructs are those listed in ``body_process`` doc.
+subject_process = (strip |
+    ReSub(r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$', r'\4') |
+    ucfirst | final_dot)
+
+
+## ``tag_filter_regexp`` is a regexp
+##
+## Tags that will be used for the changelog must match this regexp.
+##
+tag_filter_regexp = r'^v[0-9]+\.[0-9]+\.[0-9]+$'
+
+
+## ``unreleased_version_label`` is a string
+##
+## This label will be used as the changelog Title of the last set of changes
+## between last valid tag and HEAD if any.
+unreleased_version_label = "%%version%% (unreleased)"
+
+
+## ``output_engine`` is a callable
+##
+## This will change the output format of the generated changelog file
+##
+## Available choices are:
+##
+##   - rest_py
+##
+##        Legacy pure python engine, outputs ReSTructured text.
+##        This is the default.
+##
+##   - mustache(<template_name>)
+##
+##        Template name could be any of the available templates in
+##        ``templates/mustache/*.tpl``.
+##        Requires python package ``pystache``.
+##        Examples:
+##           - mustache("markdown")
+##           - mustache("restructuredtext")
+##
+##   - makotemplate(<template_name>)
+##
+##        Template name could be any of the available templates in
+##        ``templates/mako/*.tpl``.
+##        Requires python package ``mako``.
+##        Examples:
+##           - makotemplate("restructuredtext")
+##
+output_engine = rest_py
+#output_engine = mustache("restructuredtext")
+#output_engine = mustache("markdown")
+#output_engine = makotemplate("restructuredtext")
+
+
+## ``include_merge`` is a boolean
+##
+## This option tells git-log whether to include merge commits in the log.
+## The default is to include them.
+include_merge = True
+

pymisp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.4.68'
+__version__ = '2.4.71'
 
 from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey
 from .api import PyMISP

pymisp/api.py

@@ -411,6 +411,8 @@ class PyMISP(object):
                 eventID_to_update = e.uuid
         if eventID_to_update is None:
             raise PyMISPError("Unable to find the ID of the event to update")
+        if not attributes:
+            return {'error': 'No attributes.'}
         for a in attributes:
             if proposal:
                 response = self.proposal_add(eventID_to_update, a)
@@ -795,22 +797,21 @@ class PyMISP(object):
                        'analysis': ["0", "1", "2", "!0", "!1", "!2"]}
         buildup_url = "events/index"
 
+        to_post = {}
         for rule in allowed.keys():
-            if allowed[rule] is not None:
+            if allowed.get(rule) is None:
-                if not isinstance(allowed[rule], list):
+                continue
-                    allowed[rule] = [allowed[rule]]
+            param = allowed[rule]
-                allowed[rule] = [x for x in map(str, allowed[rule])]
+            if not isinstance(param, list):
-                if rule in rule_levels:
+                param = [param]
-                    if not set(allowed[rule]).issubset(rule_levels[rule]):
+            param = [x for x in map(str, param)]
-                        raise SearchError('Values in your {} are invalid, has to be in {}'.format(rule, ', '.join(str(x) for x in rule_levels[rule])))
+            if rule in rule_levels:
-                if type(allowed[rule]) == list:
+                if not set(param).issubset(rule_levels[rule]):
-                    joined = '|'.join(str(x) for x in allowed[rule])
+                    raise SearchError('Values in your {} are invalid, has to be in {}'.format(rule, ', '.join(str(x) for x in rule_levels[rule])))
-                    buildup_url += '/search{}:{}'.format(rule, joined)
+            to_post[rule] = '|'.join(str(x) for x in param)
-                else:
-                    buildup_url += '/search{}:{}'.format(rule, allowed[rule])
         session = self.__prepare_session()
         url = urljoin(self.root_url, buildup_url)
-        response = session.get(url)
+        response = session.post(url, data=json.dumps(to_post))
         return self._check_response(response)
 
     def search_all(self, value):
@@ -1135,31 +1136,31 @@ class PyMISP(object):
             user['role_id'] = kwargs.get('role_id')
         if kwargs.get('password'):
             user['password'] = kwargs.get('password')
-        if kwargs.get('external_auth_required'):
+        if kwargs.get('external_auth_required') is not None:
             user['external_auth_required'] = kwargs.get('external_auth_required')
         if kwargs.get('external_auth_key'):
             user['external_auth_key'] = kwargs.get('external_auth_key')
-        if kwargs.get('enable_password'):
+        if kwargs.get('enable_password') is not None:
             user['enable_password'] = kwargs.get('enable_password')
         if kwargs.get('nids_sid'):
             user['nids_sid'] = kwargs.get('nids_sid')
-        if kwargs.get('server_id'):
+        if kwargs.get('server_id') is not None:
             user['server_id'] = kwargs.get('server_id')
         if kwargs.get('gpgkey'):
             user['gpgkey'] = kwargs.get('gpgkey')
         if kwargs.get('certif_public'):
             user['certif_public'] = kwargs.get('certif_public')
-        if kwargs.get('autoalert'):
+        if kwargs.get('autoalert') is not None:
             user['autoalert'] = kwargs.get('autoalert')
-        if kwargs.get('contactalert'):
+        if kwargs.get('contactalert') is not None:
             user['contactalert'] = kwargs.get('contactalert')
-        if kwargs.get('disabled'):
+        if kwargs.get('disabled') is not None:
             user['disabled'] = kwargs.get('disabled')
-        if kwargs.get('change_pw'):
+        if kwargs.get('change_pw') is not None:
             user['change_pw'] = kwargs.get('change_pw')
-        if kwargs.get('termsaccepted'):
+        if kwargs.get('termsaccepted') is not None:
             user['termsaccepted'] = kwargs.get('termsaccepted')
-        if kwargs.get('newsread'):
+        if kwargs.get('newsread') is not None:
             user['newsread'] = kwargs.get('newsread')
         if kwargs.get('authkey'):
             user['authkey'] = kwargs.get('authkey')
@@ -1225,8 +1226,6 @@ class PyMISP(object):
         organisation = {}
         if kwargs.get('name'):
             organisation['name'] = kwargs.get('name')
-        if kwargs.get('anonymise'):
-            organisation['anonymise'] = kwargs.get('anonymise')
         if kwargs.get('description'):
             organisation['description'] = kwargs.get('description')
         if kwargs.get('type'):
@@ -1239,13 +1238,13 @@ class PyMISP(object):
             organisation['uuid'] = kwargs.get('uuid')
         if kwargs.get('contacts'):
             organisation['contacts'] = kwargs.get('contacts')
-        if kwargs.get('local'):
+        if kwargs.get('local') is not None:
             organisation['local'] = kwargs.get('local')
         return organisation
 
     def get_organisations_list(self, scope="local"):
         session = self.__prepare_session()
-        scope=scope.lower()
+        scope = scope.lower()
         if scope not in ["local", "external", "all"]:
             raise ValueError("Authorized fields are 'local','external' or 'all'")
         url = urljoin(self.root_url, 'organisations/index/scope:{}'.format(scope))
@@ -1261,8 +1260,8 @@ class PyMISP(object):
     def add_organisation(self, name, **kwargs):
         new_org = self._set_organisation_parameters(**dict(name=name, **kwargs))
         session = self.__prepare_session()
-        if local in new_org:
+        if 'local' in new_org:
-            if new_org.get('local') == False:
+            if new_org.get('local') is False:
                 if 'uuid' not in new_org:
                     raise PyMISPError('A remote org MUST have a valid uuid')
         url = urljoin(self.root_url, 'admin/organisations/add/')
@@ -1310,13 +1309,13 @@ class PyMISP(object):
         if organisation is None:
             raise PyMISPError('Need a valid organisation as argument, create it before if needed')
         if 'Organisation' in organisation:
-            organisation=organisation.get('Organisation')
+            organisation = organisation.get('Organisation')
         if 'local' not in organisation:
             raise PyMISPError('Need a valid organisation as argument. "local" value have not been set in this organisation')
         if 'id' not in organisation:
             raise PyMISPError('Need a valid organisation as argument. "id" value doesn\'t exist in provided organisation')
-        # Local organisation is '0' and remote organisation is '1'. These values are extracted from web interface of MISP
+
-        if organisation.get('local') == True:
+        if organisation.get('local'):  # Local organisation is '0' and remote organisation is '1'. These values are extracted from web interface of MISP
             organisation_type = 0
         else:
             organisation_type = 1

pymisp/data/describeTypes.json

@@ -117,6 +117,10 @@
         "default_category": "Payload installation",
         "to_ids": 1
       },
+      "sigma": {
+        "default_category": "Payload installation",
+        "to_ids": 1
+      },
       "vulnerability": {
         "default_category": "External analysis",
         "to_ids": 0
@@ -141,6 +145,10 @@
         "default_category": "Other",
         "to_ids": 0
       },
+      "hex": {
+        "default_category": "Other",
+        "to_ids": 0
+      },
       "other": {
         "default_category": "Other",
         "to_ids": 0
@@ -245,6 +253,10 @@
         "default_category": "Payload delivery",
         "to_ids": 1
       },
+      "impfuzzy": {
+        "default_category": "Payload delivery",
+        "to_ids": 1
+      },
       "sha224": {
         "default_category": "Payload delivery",
         "to_ids": 1
@@ -281,6 +293,10 @@
         "default_category": "Payload delivery",
         "to_ids": 1
       },
+      "filename|impfuzzy": {
+        "default_category": "Payload delivery",
+        "to_ids": 1
+      },
       "filename|pehash": {
         "default_category": "Payload delivery",
         "to_ids": 1
@@ -560,12 +576,14 @@
       "pattern-in-traffic",
       "pattern-in-memory",
       "yara",
+      "sigma",
       "vulnerability",
       "attachment",
       "malware-sample",
       "link",
       "comment",
       "text",
+      "hex",
       "other",
       "named pipe",
       "mutex",
@@ -592,6 +610,7 @@
       "ssdeep",
       "imphash",
       "pehash",
+      "impfuzzy",
       "sha224",
       "sha384",
       "sha512",
@@ -601,6 +620,7 @@
       "filename|authentihash",
       "filename|ssdeep",
       "filename|imphash",
+      "filename|impfuzzy",
       "filename|pehash",
       "filename|sha224",
       "filename|sha384",
@@ -687,7 +707,8 @@
         "text",
         "link",
         "comment",
-        "other"
+        "other",
+        "hex"
       ],
       "Targeting data": [
         "target-user",
@@ -702,6 +723,7 @@
         "link",
         "comment",
         "text",
+        "hex",
         "attachment",
         "other"
       ],
@@ -716,6 +738,7 @@
         "sha512/256",
         "ssdeep",
         "imphash",
+        "impfuzzy",
         "authentihash",
         "pehash",
         "tlsh",
@@ -732,6 +755,7 @@
         "filename|ssdeep",
         "filename|tlsh",
         "filename|imphash",
+        "filename|impfuzzy",
         "filename|pehash",
         "ip-src",
         "ip-dst",
@@ -749,12 +773,14 @@
         "pattern-in-file",
         "pattern-in-traffic",
         "yara",
+        "sigma",
         "attachment",
         "malware-sample",
         "link",
         "malware-type",
         "comment",
         "text",
+        "hex",
         "vulnerability",
         "x509-fingerprint-sha1",
         "other",
@@ -782,6 +808,7 @@
         "sha512/256",
         "ssdeep",
         "imphash",
+        "impfuzzy",
         "authentihash",
         "filename",
         "filename|md5",
@@ -796,6 +823,7 @@
         "filename|ssdeep",
         "filename|tlsh",
         "filename|imphash",
+        "filename|impfuzzy",
         "filename|pehash",
         "regkey",
         "regkey|value",
@@ -803,6 +831,7 @@
         "pattern-in-memory",
         "pdb",
         "yara",
+        "sigma",
         "attachment",
         "malware-sample",
         "named pipe",
@@ -812,6 +841,7 @@
         "windows-service-displayname",
         "comment",
         "text",
+        "hex",
         "x509-fingerprint-sha1",
         "other"
       ],
@@ -826,6 +856,7 @@
         "sha512/256",
         "ssdeep",
         "imphash",
+        "impfuzzy",
         "authentihash",
         "pehash",
         "tlsh",
@@ -842,17 +873,20 @@
         "filename|ssdeep",
         "filename|tlsh",
         "filename|imphash",
+        "filename|impfuzzy",
         "filename|pehash",
         "pattern-in-file",
         "pattern-in-traffic",
         "pattern-in-memory",
         "yara",
+        "sigma",
         "vulnerability",
         "attachment",
         "malware-sample",
         "malware-type",
         "comment",
         "text",
+        "hex",
         "x509-fingerprint-sha1",
         "mobile-application-id",
         "other"
@@ -863,7 +897,8 @@
         "regkey|value",
         "comment",
         "text",
-        "other"
+        "other",
+        "hex"
       ],
       "Network activity": [
         "ip-src",
@@ -886,7 +921,8 @@
         "comment",
         "text",
         "x509-fingerprint-sha1",
-        "other"
+        "other",
+        "hex"
       ],
       "Payload type": [
         "comment",
@@ -952,7 +988,8 @@
         "prtn",
         "comment",
         "text",
-        "other"
+        "other",
+        "hex"
       ],
       "Support Tool": [
         "link",
@@ -960,7 +997,8 @@
         "attachment",
         "comment",
         "text",
-        "other"
+        "other",
+        "hex"
       ],
       "Social network": [
         "github-username",
@@ -1011,7 +1049,8 @@
         "datetime",
         "cpe",
         "port",
-        "float"
+        "float",
+        "hex"
       ]
     }
   }

tests/test.py

@@ -49,7 +49,6 @@ class TestBasic(unittest.TestCase):
                                u'Orgc': {u'name': u'ORGNAME'},
                                u'Galaxy': [],
                                u'threat_level_id': u'1'}}
-        print(event)
         self.assertEqual(event, to_check, 'Failed at creating a new Event')
         return int(event_id)
 
@@ -135,6 +134,93 @@ class TestBasic(unittest.TestCase):
         event = self.misp.add_event(event)
         print(event)
 
+    def add_user(self):
+        email = 'test@misp.local'
+        role_id = '5'
+        org_id = '1'
+        password = 'Password1234!'
+        external_auth_required = False
+        external_auth_key = ''
+        enable_password = False
+        nids_sid = '1238717'
+        server_id = '1'
+        gpgkey = ''
+        certif_public = ''
+        autoalert = False
+        contactalert = False
+        disabled = False
+        change_pw = '0'
+        termsaccepted = False
+        newsread = '0'
+        authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+        to_check = {'User': {'email': email, 'org_id': org_id, 'role_id': role_id,
+                             'password': password, 'external_auth_required': external_auth_required,
+                             'external_auth_key': external_auth_key, 'enable_password': enable_password,
+                             'nids_sid': nids_sid, 'server_id': server_id, 'gpgkey': gpgkey,
+                             'certif_public': certif_public, 'autoalert': autoalert,
+                             'contactalert': contactalert, 'disabled': disabled,
+                             'change_pw': change_pw, 'termsaccepted': termsaccepted,
+                             'newsread': newsread, 'authkey': authkey}}
+        user = self.misp.add_user(email=email,
+                                  role_id=role_id,
+                                  org_id=org_id,
+                                  password=password,
+                                  external_auth_required=external_auth_required,
+                                  external_auth_key=external_auth_key,
+                                  enable_password=enable_password,
+                                  nids_sid=nids_sid,
+                                  server_id=server_id,
+                                  gpgkey=gpgkey,
+                                  certif_public=certif_public,
+                                  autoalert=autoalert,
+                                  contactalert=contactalert,
+                                  disabled=disabled,
+                                  change_pw=change_pw,
+                                  termsaccepted=termsaccepted,
+                                  newsread=newsread,
+                                  authkey=authkey)
+        # delete user to allow reuse of test
+        uid = user.get('User').get('id')
+        self.misp.delete_user(uid)
+        # ----------------------------------
+        # test interesting keys only (some keys are modified(password) and some keys are added (lastlogin)
+        tested_keys = ['email', 'org_id', 'role_id', 'server_id', 'autoalert',
+                       'authkey', 'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted',
+                       'newsread', 'contactalert', 'disabled']
+        for k in tested_keys:
+            self.assertEqual(user.get('User').get(k), to_check.get('User').get(k), "Failed to match input with output on key: {}".format(k))
+
+    def add_organisation(self):
+        name = 'Organisation tests'
+        description = 'This is a test organisation'
+        orgtype = 'Type is a string'
+        nationality = 'French'
+        sector = 'Bank sector'
+        uuid = '16fd2706-8baf-433b-82eb-8c7fada847da'
+        contacts = 'Text field with no limitations'
+        local = False
+        to_check = {'Organisation': {'name': name, 'description': description,
+                                     'type': orgtype, 'nationality': nationality,
+                                     'sector': sector, 'uuid': uuid, 'contacts': contacts,
+                                     'local': local}}
+        org = self.misp.add_organisation(name=name,
+                                         description=description,
+                                         type=orgtype,
+                                         nationality=nationality,
+                                         sector=sector,
+                                         uuid=uuid,
+                                         contacts=contacts,
+                                         local=local,
+                                         )
+        # delete organisation to allow reuse of test
+        oid = org.get('Organisation').get('id')
+        self.misp.delete_organisation(oid)
+        # ----------------------------------
+        tested_keys = ['anonymise', 'contacts', 'description', 'local', 'name',
+                       'nationality', 'sector', 'type', 'uuid']
+        for k in tested_keys:
+            self.assertEqual(org.get('Organisation').get(k), to_check.get('Organisation').get(k), "Failed to match input with output on key: {}".format(k))
+
     def test_create_event(self):
         eventid = self.new_event()
         time.sleep(1)
@@ -163,5 +249,11 @@ class TestBasic(unittest.TestCase):
         self.assertEqual(self.misp._one_or_more(1), (1,))
         self.assertEqual(self.misp._one_or_more([1]), [1])
 
+    def test_create_user(self):
+        self.add_user()
+
+    def test_create_organisation(self):
+        self.add_organisation()
+
 if __name__ == '__main__':
     unittest.main()

tests/test_offline.py

@@ -47,8 +47,7 @@ class TestOffline(unittest.TestCase):
         m.register_uri('DELETE', self.domain + 'events/2', json={'message': 'Event deleted.'})
         m.register_uri('DELETE', self.domain + 'events/3', json={'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'})
         m.register_uri('DELETE', self.domain + 'attributes/2', json={'message': 'Attribute deleted.'})
-        m.register_uri('GET', self.domain + 'events/index/searchtag:1', json=self.search_index_result)
+        m.register_uri('POST', self.domain + 'events/index', json=self.search_index_result)
-        m.register_uri('GET', self.domain + 'events/index/searchtag:ecsirt:malicious-code=%22ransomware%22', json=self.search_index_result)
 
     def test_getEvent(self, m):
         self.initURI(m)