diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..c78dbf7 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,12 @@ +version: 2 + +python: + version: 3.6 + install: + - method: pip + path: . + extra_requirements: + - docs + +build: + image: latest diff --git a/.travis.yml b/.travis.yml index addba8f..9bbfd22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,17 +16,15 @@ python: - "3.6-dev" install: - - pip install -U nose pip setuptools - - pip install coveralls codecov requests-mock - - pip install git+https://github.com/kbandla/pydeep.git - - pip install .[fileobjects,neo,openioc,virustotal] + - pip install pipenv + - pipenv install --dev - pushd tests - git clone https://github.com/viper-framework/viper-test-files.git - popd script: - - nosetests --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py + - pipenv run nosetests --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py after_success: - - codecov - - coveralls + - pipenv run codecov + - pipenv run coveralls diff --git a/CHANGELOG.txt b/CHANGELOG.txt index d8c1dc2..8ac8e25 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -2,12 +2,171 @@ Changelog ========= +v2.4.102 (2019-02-03) +--------------------- + +New +~~~ +- Add test cases for stix export. [Raphaël Vinot] + +Changes +~~~~~~~ +- Bump misp-objects. [Raphaël Vinot] +- [datamodel] new anonymised type added. [Alexandre Dulaunoy] +- [data] types updated to include the new zeek type. [Alexandre + Dulaunoy] + +Other +~~~~~ +- Merge pull request #328 from kirzaks/master. [Raphaël Vinot] + + Check if IOC values are in warninglist +- Check if IOC values are in warninglist. [Armīns Palms] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] + + +v2.4.101 (2019-01-28) +--------------------- + +New +~~~ +- Add missing test case for NOT on attribute level, update Pipfile. + [Raphaël Vinot] +- Add support for unix timestamp in set_date. [Raphaël Vinot] + + fix #302 +- Add test for references when adding/updating a full event. [Raphaël + Vinot] +- Bump describe types. [Raphaël Vinot] + + fix #317 +- [usersStats] Possibility to fetch users/statistics data for all + context (usage, org, tags, ...) [Sami Mokaddem] + +Changes +~~~~~~~ +- Bump Changelog. [Raphaël Vinot] +- Bump version, misp-objects. [Raphaël Vinot] +- Force to_ids to be a boolean, as MISP is expecting. [Raphaël Vinot] + + fix #320 +- Add support for sharing group filter for search_index. [Tom King] +- Support for Payload Delivery > Other attribute as PyMISP function. + [Tom King] +- Add Pipfile config. [Raphaël Vinot] +- [data] fix describeTypes. [Alexandre Dulaunoy] +- [data] new types added (hassh-md5 and hasshserver-md5) [Alexandre + Dulaunoy] +- Bump misp-objects. [Raphaël Vinot] +- [misp-objects] templates updated to the latest version. [Alexandre + Dulaunoy] +- [data] describeTypes updated (grabbed from MISP HEAD) [Alexandre + Dulaunoy] +- [data] ja3-fingerprint-md5 type added. [Alexandre Dulaunoy] +- [test] set a default distribution for massive event creation. + [Alexandre Dulaunoy] +- [data] describeTypes.json updated to the latest version. [Alexandre + Dulaunoy] +- More flexibility when loading an object from python dict. [Raphaël + Vinot] +- Pass all parameters to the search API. [Raphaël Vinot] +- Remove compat for MISP 2.4.52, cleanup. [Raphaël Vinot] +- Set verifycert to false in tests. [Raphaël Vinot] +- [tests] Added verifycert option in case of using self-signed cert. + [Steve Clement] + +Fix +~~~ +- Remove jupyter & ipython from dev install so it works with python2. + [Raphaël Vinot] +- Wrong variable name in MISPEvent. [Raphaël Vinot] +- Documentation error fix #278. [Raphaël Vinot] +- Attempt to fix memory footprint in MISPAttribute. [Raphaël Vinot] +- Still support simple event dict content. [Raphaël Vinot] +- Don't modify event passed to the add_attribute methods. [Raphaël + Vinot] + + fix #321 +- The wrong class name was used when there is an error at Event + creation. [Raphaël Vinot] +- Use new API in get_csv.py. [Raphaël Vinot] + + Fix #314 +- Test case was broken. [Raphaël Vinot] +- Create massive event using ExpandedPyMISP. [Raphaël Vinot] +- Error vs errors key. [Raphaël Vinot] +- Typo. [Raphaël Vinot] +- Get_object_template_id. [Christophe Vandeplas] + +Other +~~~~~ +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #325 from winklerrr/patch-1. [Raphaël Vinot] + + Update pymisp tutorial +- Update pymisp tutorial. [Sandro Winkler] + + Extract the "response" field from the json result returned by misp.search_index +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #306 from tomking2/master. [Raphaël Vinot] + + chg: Support for Payload Delivery > Other attribute as PyMISP function & search_index sharinggroup filter +- Merge remote-tracking branch 'upstream/master' [Tom King] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #318 from cvandeplas/master. [Alexandre Dulaunoy] + + sort describeTypes.json output +- Sort describeTypes.json output. [Christophe Vandeplas] + + This is needed for the compatibility with the gen_misp_types_categories.py script. Data was sorted using the order_dict function of the gen_misp_types_categories script. +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #257 from mokaddem/usersStatistics. [Raphaël Vinot] + + new: [usersStats] Possibility to fetch users/statistics +- Merge pull request #311 from cvandeplas/master. [Raphaël Vinot] + + Bugfixes + + v2.4.99 (2018-12-06) -------------------- New ~~~ - Auto generate doc for PyMISPExpanded. [Raphaël Vinot] + +Changes +~~~~~~~ +- Bump Changelog, again. [Raphaël Vinot] +- Bump Changelog. [Raphaël Vinot] +- Bump version. [Raphaël Vinot] +- Bump misp-objects & describeTypes. [Raphaël Vinot] + +Fix +~~~ +- Auto generate doc for PyMISPExpanded. [Raphaël Vinot] + +Other +~~~~~ +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] +- Merge pull request #310 from DragonDev1906/master. [Raphaël Vinot] + + Added get_object & get_attribute (by ID) +- Dded get_object & get_attribute. [DragonDev1906] +- Merge pull request #307 from garanews/patch-1. [Raphaël Vinot] + + fix for last pymisp version +- Fix for last pymisp version. [garanews] +- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] + + +v2.4.98 (2018-12-03) +-------------------- + +New +~~~ - Search_index in ExpandedPyMISP, cleanup, update jupyter. [Raphaël Vinot] - Add log search. [Raphaël Vinot] @@ -22,9 +181,6 @@ New Changes ~~~~~~~ - Bump Changelog. [Raphaël Vinot] -- Bump version. [Raphaël Vinot] -- Bump misp-objects & describeTypes. [Raphaël Vinot] -- Bump Changelog. [Raphaël Vinot] - Version bump. [Raphaël Vinot] - Bump misp-objects. [Raphaël Vinot] - Add test cases for default distribution levels. [Raphaël Vinot] @@ -43,7 +199,6 @@ Changes Fix ~~~ -- Auto generate doc for PyMISPExpanded. [Raphaël Vinot] - Test failing on travis... [Raphaël Vinot] - Properly handle errors on event creation/update. [Raphaël Vinot] - Test case. [Raphaël Vinot] @@ -58,19 +213,10 @@ Fix align python path to readme specifying python3 - Feed-generator gitignore. [Christophe Vandeplas] - Test cases. [Raphaël Vinot] +- Test cases sample files. [Raphaël Vinot] Other ~~~~~ -- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] -- Merge pull request #310 from DragonDev1906/master. [Raphaël Vinot] - - Added get_object & get_attribute (by ID) -- Dded get_object & get_attribute. [DragonDev1906] -- Merge pull request #307 from garanews/patch-1. [Raphaël Vinot] - - fix for last pymisp version -- Fix for last pymisp version. [garanews] -- Merge branch 'master' of github.com:MISP/PyMISP. [Raphaël Vinot] - Merge pull request #305 from dawid- czarnecki/feature/include_proposals. [Raphaël Vinot] @@ -130,7 +276,6 @@ Changes Fix ~~~ -- Test cases sample files. [Raphaël Vinot] - Prevent checking length on a integer. [Sami Mokaddem] - Direct call & add example. [Raphaël Vinot] - Disable test for travis, take 2. [Raphaël Vinot] diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..801ad01 --- /dev/null +++ b/Pipfile @@ -0,0 +1,18 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +nose = "*" +coveralls = "*" +codecov = "*" +requests-mock = "*" + +[packages] +pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal"],path = "."} +pydeep = {editable = true,git = "https://github.com/kbandla/pydeep.git"} +pymispwarninglists = {editable = true,git = "https://github.com/MISP/PyMISPWarningLists.git"} + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..1258507 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,411 @@ +{ + "_meta": { + "hash": { + "sha256": "2551d32f7430eba34eac975cc1b28eca13fe9faff7197d83f312d7de8df187da" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "asn1crypto": { + "hashes": [ + "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", + "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" + ], + "version": "==0.24.0" + }, + "beautifulsoup4": { + "hashes": [ + "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", + "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", + "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" + ], + "version": "==4.7.1" + }, + "certifi": { + "hashes": [ + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + ], + "version": "==2018.11.29" + }, + "cffi": { + "hashes": [ + "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", + "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", + "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", + "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", + "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", + "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", + "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", + "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", + "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", + "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", + "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", + "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", + "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", + "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", + "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", + "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", + "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", + "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", + "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", + "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", + "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", + "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", + "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", + "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", + "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", + "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", + "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", + "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", + "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", + "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", + "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", + "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" + ], + "version": "==1.11.5" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "colorama": { + "hashes": [ + "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", + "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" + ], + "version": "==0.4.1" + }, + "cryptography": { + "hashes": [ + "sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af", + "sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e", + "sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2", + "sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7", + "sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079", + "sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063", + "sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401", + "sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695", + "sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85", + "sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3", + "sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad", + "sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca", + "sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd", + "sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f", + "sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159", + "sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0", + "sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e", + "sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3", + "sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00" + ], + "version": "==2.5" + }, + "decorator": { + "hashes": [ + "sha256:33cd704aea07b4c28b3eb2c97d288a06918275dac0ecebdaf1bc8a48d98adb9e", + "sha256:cabb249f4710888a2fc0e13e9a16c343d932033718ff62e1e9bc93a9d3a9122b" + ], + "version": "==4.3.2" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "ipaddress": { + "hashes": [ + "sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794", + "sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c" + ], + "version": "==1.0.22" + }, + "jsonschema": { + "hashes": [ + "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", + "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02" + ], + "version": "==2.6.0" + }, + "lief": { + "hashes": [ + "sha256:c95974006a6b8a767eea8b35e6c63e2b20939730063ac472894b53ab9855a0b5" + ], + "version": "==0.9.0" + }, + "neobolt": { + "hashes": [ + "sha256:f70df7422568f3f92f065482237dabe3b96cd49a921c5e17feb1c9e68fdd0357" + ], + "version": "==1.7.3" + }, + "neotime": { + "hashes": [ + "sha256:4e0477ba0f24e004de2fa79a3236de2bd941f20de0b5db8d976c52a86d7363eb" + ], + "version": "==1.7.4" + }, + "prompt-toolkit": { + "hashes": [ + "sha256:88002cc618cacfda8760c4539e76c3b3f148ecdb7035a3d422c7ecdc90c2a3ba", + "sha256:c6655a12e9b08edb8cf5aeab4815fd1e1bdea4ad73d3bbf269cf2e0c4eb75d5e", + "sha256:df5835fb8f417aa55e5cafadbaeb0cf630a1e824aad16989f9f0493e679ec010" + ], + "version": "==2.0.8" + }, + "py2neo": { + "hashes": [ + "sha256:c25d24a1504bbfaf61e862e29953f17ad67a4810d55531b1436ad0c7664d85fd" + ], + "version": "==4.2.0" + }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "pydeep": { + "editable": true, + "git": "https://github.com/kbandla/pydeep.git", + "ref": "bc0d33bff4b45718b4c5f2c79d4715d92a427eda" + }, + "pygments": { + "hashes": [ + "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", + "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" + ], + "version": "==2.3.1" + }, + "pymisp": { + "editable": true, + "extras": [ + "fileobjects", + "neo", + "openioc", + "virustotal" + ], + "path": "." + }, + "pymispwarninglists": { + "editable": true, + "git": "https://github.com/MISP/PyMISPWarningLists.git", + "ref": "d512ca91ae0635407754933099d6f3dd654dbcfe" + }, + "pyopenssl": { + "hashes": [ + "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200", + "sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6" + ], + "version": "==19.0.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "version": "==2.8.0" + }, + "python-magic": { + "hashes": [ + "sha256:f2674dcfad52ae6c49d4803fa027809540b130db1dec928cfbb9240316831375", + "sha256:f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5" + ], + "version": "==0.4.15" + }, + "pytz": { + "hashes": [ + "sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9", + "sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c" + ], + "version": "==2018.9" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "version": "==2.21.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "soupsieve": { + "hashes": [ + "sha256:466910df7561796a60748826781ebe9a888f7a1668a636ae86783f44d10aae73", + "sha256:87db12ae79194f0ff9808d2b1641c4f031ae39ffa3cab6b907ea7c1e5e5ed445" + ], + "version": "==1.7.3" + }, + "urllib3": { + "extras": [ + "secure" + ], + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + }, + "validators": { + "hashes": [ + "sha256:68e4b74889aac1270d83636cb1dbcce3d2271e291ab14023cf95e7dbfbbce09d" + ], + "version": "==0.12.4" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + } + }, + "develop": { + "certifi": { + "hashes": [ + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + ], + "version": "==2018.11.29" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "codecov": { + "hashes": [ + "sha256:8ed8b7c6791010d359baed66f84f061bba5bd41174bf324c31311e8737602788", + "sha256:ae00d68e18d8a20e9c3288ba3875ae03db3a8e892115bf9b83ef20507732bed4" + ], + "index": "pypi", + "version": "==2.0.15" + }, + "coverage": { + "hashes": [ + "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", + "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", + "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", + "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", + "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", + "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", + "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", + "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", + "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", + "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", + "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", + "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", + "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", + "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", + "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", + "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", + "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", + "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", + "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", + "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", + "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", + "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", + "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", + "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", + "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", + "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", + "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", + "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", + "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", + "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", + "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + ], + "version": "==4.5.2" + }, + "coveralls": { + "hashes": [ + "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d", + "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0" + ], + "index": "pypi", + "version": "==1.5.1" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "nose": { + "hashes": [ + "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", + "sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a", + "sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98" + ], + "index": "pypi", + "version": "==1.3.7" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "version": "==2.21.0" + }, + "requests-mock": { + "hashes": [ + "sha256:7a5fa99db5e3a2a961b6f20ed40ee6baeff73503cf0a553cc4d679409e6170fb", + "sha256:8ca0628dc66d3f212878932fd741b02aa197ad53fd2228164800a169a4a826af" + ], + "index": "pypi", + "version": "==1.5.2" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "urllib3": { + "extras": [ + "secure" + ], + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + } + } +} diff --git a/README.md b/README.md index de41e1a..68a087d 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,10 @@ Documentation can be generated with epydoc: epydoc --url https://github.com/MISP/PyMISP --graph all --name PyMISP --pdf pymisp -o doc ``` +### Jupyter notebook + +A series of [Jupyter notebooks for PyMISP tutorial](https://github.com/MISP/PyMISP/tree/master/docs/tutorial) are available in the repository. + ## Everything is a Mutable Mapping ... or at least everything that can be imported/exported from/to a json blob diff --git a/docs/source/conf.py b/docs/source/conf.py index 78715f6..10a7cea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,6 +30,7 @@ from recommonmark.parser import CommonMarkParser # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx_autodoc_typehints', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', @@ -255,21 +256,21 @@ htmlhelp_basename = 'PyMISPdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 5fc2210..39843e4 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -17,7 +17,7 @@ PyMISP PyMISPExpanded (Python 3.6+ only) --------------------------------- -.. autoclass:: PyMISPExpanded +.. autoclass:: ExpandedPyMISP :members: MISPAbstract diff --git a/docs/tutorial/PyMISP_tutorial.ipynb b/docs/tutorial/PyMISP_tutorial.ipynb index 7eab177..c12a217 100644 --- a/docs/tutorial/PyMISP_tutorial.ipynb +++ b/docs/tutorial/PyMISP_tutorial.ipynb @@ -315,9 +315,11 @@ "metadata": {}, "outputs": [], "source": [ - "results = misp.search_index(eventinfo='notebook')\n", + "result = misp.search_index(eventinfo='notebook')\n", + "events = result['response']\n", "\n", - "for event in results:\n", + "print('Found ', len(events), ' events!')\n", + "for event in events:\n", " print(event['id'], ':', event['info'])" ] }, diff --git a/examples/events/create_massive_dummy_events.py b/examples/events/create_massive_dummy_events.py index 12a2826..7829ad6 100755 --- a/examples/events/create_massive_dummy_events.py +++ b/examples/events/create_massive_dummy_events.py @@ -1,8 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pymisp import PyMISP -from keys import url, key +from pymisp import ExpandedPyMISP +try: + from keys import url, key +except ImportError: + url = 'http://localhost:8080' + key = '8h0gHbhS0fv6JUOlTED0AznLXFbf83TYtQrCycqb' import argparse import tools @@ -13,7 +17,7 @@ if __name__ == '__main__': parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)") args = parser.parse_args() - misp = PyMISP(url, key, True, 'json') + misp = ExpandedPyMISP(url, key, True) if args.limit is None: args.limit = 1 diff --git a/examples/events/tools.py b/examples/events/tools.py index 9d0e3f5..94f5d91 100644 --- a/examples/events/tools.py +++ b/examples/events/tools.py @@ -4,6 +4,7 @@ import random from random import randint import string +from pymisp import MISPEvent def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits): @@ -63,12 +64,16 @@ def create_dummy_event(misp): def create_massive_dummy_events(misp, nbattribute): - event = misp.new_event(0, 4, 0, 'massive dummy event') - eventid = event['Event']['id'] + event = MISPEvent() + event.info = 'massive dummy event' + event = misp.add_event(event) + print(event) + eventid = event.id + distribution = '0' functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment] for i in range(nbattribute): choice = randint(0, 5) if choice == 5: - floodattachment(misp, eventid, event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) + floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event.info, event.analysis, event.threat_level_id) else: functions[choice](misp, event) diff --git a/examples/get_csv.py b/examples/get_csv.py index 33baf62..5921e53 100755 --- a/examples/get_csv.py +++ b/examples/get_csv.py @@ -1,9 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse -from pymisp import PyMISP +from pymisp import ExpandedPyMISP from keys import misp_url, misp_key, misp_verifycert @@ -14,12 +14,20 @@ if __name__ == '__main__': parser.add_argument("-o", "--object_attribute", nargs='+', help="Object attribute column names") parser.add_argument("-t", "--misp_types", nargs='+', help="MISP types to fetch (ip-src, hostname, ...)") parser.add_argument("-c", "--context", action='store_true', help="Add event level context (tags...)") - parser.add_argument("-i", "--ignore", action='store_true', help="Returns the attributes even if the event isn't published, or the attribute doesn't have the to_ids flag") parser.add_argument("-f", "--outfile", help="Output file to write the CSV.") args = parser.parse_args() - pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) - response = pymisp.get_csv(args.event_id, args.attribute, args.object_attribute, args.misp_types, args.context, args.ignore) + pymisp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, debug=True) + attr = [] + if args.attribute: + attr += args.attribute + if args.object_attribute: + attr += args.object_attribute + if not attr: + attr = None + print(args.context) + response = pymisp.search(return_format='csv', eventid=args.event_id, requested_attributes=attr, + type_attribute=args.misp_types, include_context=args.context) if args.outfile: with open(args.outfile, 'w') as f: diff --git a/pymisp/__init__.py b/pymisp/__init__.py index 09a3d52..73062e7 100644 --- a/pymisp/__init__.py +++ b/pymisp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.99' +__version__ = '2.4.102' import logging import functools import warnings @@ -32,7 +32,7 @@ def deprecated(func): try: - from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse # noqa + from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .api import PyMISP # noqa from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog # noqa diff --git a/pymisp/abstract.py b/pymisp/abstract.py index 55b504b..fedaac5 100644 --- a/pymisp/abstract.py +++ b/pymisp/abstract.py @@ -1,13 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import abc import sys import datetime import json from json import JSONEncoder import collections -import six # Remove that import when discarding python2 support. import logging from enum import Enum @@ -16,7 +14,7 @@ from .exceptions import PyMISPInvalidFormat logger = logging.getLogger('pymisp') -if six.PY2: +if sys.version_info < (3, 0): logger.warning("You're using python 2, it is strongly recommended to use python >=3.6") # This is required because Python 2 is a pain. @@ -77,7 +75,6 @@ class MISPEncode(JSONEncoder): return JSONEncoder.default(self, obj) -@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. class AbstractMISP(collections.MutableMapping): __not_jsonable = [] diff --git a/pymisp/api.py b/pymisp/api.py index 8e81ca8..fbc4b80 100644 --- a/pymisp/api.py +++ b/pymisp/api.py @@ -3,6 +3,7 @@ """Python API using the REST interface of MISP""" +import copy import sys import json import datetime @@ -15,7 +16,7 @@ from io import BytesIO, open import zipfile from . import __version__, deprecated -from .exceptions import PyMISPError, SearchError, NoURL, NoKey +from .exceptions import PyMISPError, SearchError, NoURL, NoKey, PyMISPEmptyResponse from .mispevent import MISPEvent, MISPAttribute, MISPUser, MISPOrganisation, MISPSighting, MISPFeed, MISPObject from .abstract import AbstractMISP, MISPEncode @@ -28,7 +29,7 @@ try: unicode = str except ImportError: from urlparse import urljoin - logger.warning("You're using python 2, it is strongly recommended to use python >=3.5") + logger.warning("You're using python 2, it is strongly recommended to use python >=3.6") try: import requests @@ -157,6 +158,11 @@ class PyMISP(object): if data is None: req = requests.Request(request_type, url) else: + if not isinstance(data, str): + if isinstance(data, dict): + # Remove None values. + data = {k: v for k, v in data.items() if v is not None} + data = json.dumps(data) req = requests.Request(request_type, url, data=data) if self.asynch and background_callback is not None: local_session = FuturesSession @@ -227,6 +233,8 @@ class PyMISP(object): json_response = response.json() except ValueError: # If the server didn't return a JSON blob, we've a problem. + if not len(response.text): + raise PyMISPEmptyResponse('The server returned an empty response. \n{}\n{}\n'.format(response.request.headers, response.request.body)) raise PyMISPError(everything_broken.format(response.request.headers, response.request.body, response.text)) errors = [] @@ -271,7 +279,7 @@ class PyMISP(object): """Transform a Json MISP event into a MISPEvent""" if not isinstance(event, MISPEvent): e = MISPEvent(self.describe_types) - e.load(event) + e.load(copy.copy(event)) else: e = event return e @@ -300,7 +308,7 @@ class PyMISP(object): :param uuid: an uuid """ - regex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z', re.I) + regex = re.compile(r'^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z', re.I) match = regex.match(uuid) return bool(match) @@ -589,12 +597,14 @@ class PyMISP(object): elif isinstance(event, int) or (isinstance(event, str) and (event.isdigit() or self._valid_uuid(event))): event_id = event else: - e = MISPEvent(describe_types=self.describe_types) - e.load(event) - if hasattr(e, 'id'): - event_id = e.id - elif hasattr(e, 'uuid'): - event_id = e.uuid + if 'Event' in event: + e = event['Event'] + else: + e = event + if 'id' in e: + event_id = e['id'] + elif 'uuid' in e: + event_id = e['uuid'] return event_id def add_named_attribute(self, event, type_value, value, category=None, to_ids=False, comment=None, distribution=None, proposal=False, **kwargs): @@ -1058,7 +1068,7 @@ class PyMISP(object): def search_index(self, published=None, eventid=None, tag=None, datefrom=None, dateuntil=None, eventinfo=None, threatlevel=None, distribution=None, analysis=None, attribute=None, org=None, async_callback=None, normalize=False, - timestamp=None): + timestamp=None, sharinggroup=None): """Search only at the index level. Use ! infront of value as NOT, default OR If using async, give a callback that takes 2 args, session and response: basic usage is @@ -1077,11 +1087,12 @@ class PyMISP(object): :param async_callback: Function to call when the request returns (if running async) :param normalize: Normalize output | True or False :param timestamp: Interval since last update (in second, or 1d, 1h, ...) + :param sharinggroup: The sharing group value """ allowed = {'published': published, 'eventid': eventid, 'tag': tag, 'dateuntil': dateuntil, 'datefrom': datefrom, 'eventinfo': eventinfo, 'threatlevel': threatlevel, 'distribution': distribution, 'analysis': analysis, 'attribute': attribute, - 'org': org, 'timestamp': timestamp} + 'org': org, 'timestamp': timestamp, 'sharinggroup': sharinggroup} rule_levels = {'distribution': ["0", "1", "2", "3", "!0", "!1", "!2", "!3"], 'threatlevel': ["1", "2", "3", "4", "!1", "!2", "!3", "!4"], 'analysis': ["0", "1", "2", "!0", "!1", "!2"]} @@ -1120,7 +1131,7 @@ class PyMISP(object): def search_all(self, value): """Search a value in the whole database""" query = {'value': value, 'searchall': 1} - return self.__query('restSearch/download', query) + return self.__query('restSearch', query) def __prepare_rest_search(self, values, not_values): """Prepare a search, generate the chain processed by the server @@ -1209,6 +1220,15 @@ class PyMISP(object): else: return {'error': 'You must enter a valid uuid.'} + returnFormat = kwargs.pop('returnFormat', None) + if returnFormat: + if returnFormat in ['json', 'openioc', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix', 'stix2']: + query['returnFormat'] = returnFormat + else: + return {'error': 'You must enter a valid returnFormat - json, openioc, xml, suricata, snort, text, rpz, csv, stix, stix2 or cache'} + else: + query['returnFormat'] = 'json' + query['publish_timestamp'] = kwargs.pop('publish_timestamp', None) query['timestamp'] = kwargs.pop('timestamp', None) query['enforceWarninglist'] = kwargs.pop('enforceWarninglist', None) @@ -1224,14 +1244,16 @@ class PyMISP(object): query['event_timestamp'] = kwargs.pop('event_timestamp', None) query['includeProposals'] = kwargs.pop('includeProposals', None) + if kwargs: + logger.info('Some unknown parameters are in kwargs. appending as-is: {}'.format(', '.join(kwargs.keys()))) + # Add all other keys as-is. + query.update({k: v for k, v in kwargs.items()}) + # Cleanup query = {k: v for k, v in query.items() if v is not None} - if kwargs: - raise SearchError('Unused parameter: {}'.format(', '.join(kwargs.keys()))) - # Create a session, make it async if and only if we have a callback - return self.__query('restSearch/download', query, controller, async_callback) + return self.__query('restSearch', query, controller, async_callback) def get_attachment(self, attribute_id): """Get an attachement (not a malware sample) by attribute ID. @@ -1422,6 +1444,15 @@ class PyMISP(object): response = self._prepare_request('GET', url) return self._check_response(response) + def get_users_statistics(self, context='data'): + """Get users statistics from the MISP instance""" + availables_contexts = ['data', 'orgs', 'users', 'tags', 'attributehistogram', 'sightings', 'attackMatrix'] + if context not in availables_contexts: + context = 'data' + url = urljoin(self.root_url, 'users/statistics/{}.json'.format(context)) + response = self._prepare_request('GET', url) + return self._check_response(response) + # ############## Sightings ################## def sighting_per_id(self, attribute_id): @@ -1475,7 +1506,7 @@ class PyMISP(object): :type element_id: int :param scope: could be attribute or event :return: A json list of sighting corresponding to the search - :rtype: list + :rtype: dict :Example: @@ -1515,11 +1546,11 @@ class PyMISP(object): :Example: - >>> misp.search_sightings({'publish_timestamp': '30d'}) # search sightings for the last 30 days on the instance + >>> misp.search_sightings(**{'publish_timestamp': '30d'}) # search sightings for the last 30 days on the instance [ ... ] - >>> misp.search_sightings('attribute', {'id': 6, 'include_attribute': 1}) # return list of sighting for attribute 6 along with the attribute itself + >>> misp.search_sightings('attribute', context_id=6, include_attribute=1) # return list of sighting for attribute 6 along with the attribute itself [ ... ] - >>> misp.search_sightings('event', {'id': 17, 'include_event': 1, 'org_id': 2}) # return list of sighting for event 17 filtered with org id 2 + >>> misp.search_sightings('event', **{'context_id': 17, 'include_event': 1, 'org_id': 2}) # return list of sighting for event 17 filtered with org id 2 """ if context not in ['', 'attribute', 'event']: raise Exception('Context parameter must be empty, "attribute" or "event"') @@ -1921,6 +1952,12 @@ class PyMISP(object): """Disable a warninglist by id.""" return self.toggle_warninglist(warninglist_id=warninglist_id, force_enable=False) + def check_warninglist(self, value): + """Check if IOC values are in warninglist""" + url = urljoin(self.root_url, '/warninglists/checkValue') + response = self._prepare_request('POST', url, json.dumps(value)) + return self._check_response(response) + # ############## NoticeLists ################## def get_noticelists(self): diff --git a/pymisp/aping.py b/pymisp/aping.py index 9f59273..5f697f5 100644 --- a/pymisp/aping.py +++ b/pymisp/aping.py @@ -4,9 +4,8 @@ from .exceptions import MISPServerError, NewEventError, UpdateEventError, UpdateAttributeError, PyMISPNotImplementedYet from .api import PyMISP, everything_broken from .mispevent import MISPEvent, MISPAttribute, MISPSighting, MISPLog -from typing import TypeVar, Optional, Tuple, List, Dict +from typing import TypeVar, Optional, Tuple, List, Dict, Union from datetime import date, datetime -import json import csv import logging @@ -18,6 +17,7 @@ SearchParameterTypes = TypeVar('SearchParameterTypes', str, List[SearchType], Di DateTypes = TypeVar('DateTypes', datetime, date, SearchType, float) DateInterval = TypeVar('DateInterval', DateTypes, Tuple[DateTypes, DateTypes]) +ToIDSType = TypeVar('ToIDSType', str, int, bool) logger = logging.getLogger('pymisp') @@ -70,7 +70,7 @@ class ExpandedPyMISP(PyMISP): # The server returns a json message with the error details error_message = response.json() logger.error(f'Something went wrong ({response.status_code}): {error_message}') - return {'errors': [(response.status_code, error_message)]} + return {'errors': (response.status_code, error_message)} # At this point, we had no error. @@ -80,11 +80,15 @@ class ExpandedPyMISP(PyMISP): logger.debug(response) if isinstance(response, dict) and response.get('response') is not None: # Cleanup. - return response.get('response') + response = response['response'] return response except Exception: if logger.isEnabledFor(logging.DEBUG): logger.debug(response.text) + if not len(response.content): + # Empty response + logger.error('Got an empty response.') + return {'errors': 'The response is empty.'} return response.text def get_event(self, event_id: int): @@ -178,13 +182,10 @@ class ExpandedPyMISP(PyMISP): query['includeEvent'] = include_event_meta url = urljoin(self.root_url, url_path) - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) - if isinstance(normalized_response, str) or (isinstance(normalized_response, dict) and - normalized_response.get('errors')): + if isinstance(normalized_response, str) or (isinstance(normalized_response, dict) + and normalized_response.get('errors')): return normalized_response elif pythonify: to_return = [] @@ -227,7 +228,7 @@ class ExpandedPyMISP(PyMISP): timestamp: Optional[DateInterval]=None, published: Optional[bool]=None, enforce_warninglist: Optional[bool]=None, enforceWarninglist: Optional[bool]=None, - to_ids: Optional[str]=None, + to_ids: Optional[Union[ToIDSType, List[ToIDSType]]]=None, deleted: Optional[str]=None, include_event_uuid: Optional[str]=None, includeEventUuid: Optional[str]=None, event_timestamp: Optional[DateTypes]=None, @@ -260,7 +261,7 @@ class ExpandedPyMISP(PyMISP): :param timestamp: Restrict the results by the timestamp (last edit). Any event with a timestamp newer than the given timestamp will be returned. In case you are dealing with /attributes as scope, the attribute's timestamp will be used for the lookup. :param published: Set whether published or unpublished events should be returned. Do not set the parameter if you want both. :param enforce_warninglist: Remove any attributes from the result that would cause a hit on a warninglist entry. - :param to_ids: By default (0) all attributes are returned that match the other filter parameters, irregardless of their to_ids setting. To restrict the returned data set to to_ids only attributes set this parameter to 1. You can only use the special "exclude" setting to only return attributes that have the to_ids flag disabled. + :param to_ids: By default all attributes are returned that match the other filter parameters, irregardless of their to_ids setting. To restrict the returned data set to to_ids only attributes set this parameter to 1. 0 for the ones with to_ids set to False. :param deleted: If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using "only" as a parameter it will limit the returned data set to soft-deleted data only. :param include_event_uuid: Instead of just including the event ID, also include the event UUID in each of the attributes. :param event_timestamp: Only return attributes from events that have received a modification after the given timestamp. @@ -283,7 +284,7 @@ class ExpandedPyMISP(PyMISP): ''' - return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache'] + return_formats = ['openioc', 'json', 'xml', 'suricata', 'snort', 'text', 'rpz', 'csv', 'cache', 'stix', 'stix2'] if controller not in ['events', 'attributes', 'objects', 'sightings']: raise ValueError('controller has to be in {}'.format(', '.join(['events', 'attributes', 'objects']))) @@ -337,8 +338,8 @@ class ExpandedPyMISP(PyMISP): query['published'] = published query['enforceWarninglist'] = enforce_warninglist if to_ids is not None: - if str(to_ids) not in ['0', '1', 'exclude']: - raise ValueError('to_ids has to be in {}'.format(', '.join(['0', '1', 'exclude']))) + if int(to_ids) not in [0, 1]: + raise ValueError('to_ids has to be in {}'.format(', '.join([0, 1]))) query['to_ids'] = to_ids query['deleted'] = deleted query['includeEventUuid'] = include_event_uuid @@ -354,15 +355,12 @@ class ExpandedPyMISP(PyMISP): query['includeContext'] = include_context query['headerless'] = headerless url = urljoin(self.root_url, f'{controller}/restSearch') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if return_format == 'csv' and pythonify and not headerless: return self._csv_to_dict(normalized_response) - elif isinstance(normalized_response, str) or (isinstance(normalized_response, dict) and - normalized_response.get('errors')): + elif isinstance(normalized_response, str) or (isinstance(normalized_response, dict) + and normalized_response.get('errors')): return normalized_response elif return_format == 'json' and pythonify: # The response is in json, we can convert it to a list of pythonic MISP objects @@ -426,10 +424,7 @@ class ExpandedPyMISP(PyMISP): query['id'] = query.pop('log_id') url = urljoin(self.root_url, 'admin/logs/index') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if not pythonify: return normalized_response @@ -483,10 +478,7 @@ class ExpandedPyMISP(PyMISP): query['timestamp'] = self.make_timestamp(timestamp) url = urljoin(self.root_url, 'events/index') - # Remove None values. - # TODO: put that in self._prepare_request - query = {k: v for k, v in query.items() if v is not None} - response = self._prepare_request('POST', url, data=json.dumps(query)) + response = self._prepare_request('POST', url, data=query) normalized_response = self._check_response(response) if not pythonify: diff --git a/pymisp/data/describeTypes.json b/pymisp/data/describeTypes.json index cb90814..71f4fea 100644 --- a/pymisp/data/describeTypes.json +++ b/pymisp/data/describeTypes.json @@ -1,503 +1,49 @@ { "result": { - "categories": [ - "Internal reference", - "Targeting data", - "Antivirus detection", - "Payload delivery", - "Artifacts dropped", - "Payload installation", - "Persistence mechanism", - "Network activity", - "Payload type", - "Attribution", - "External analysis", - "Financial fraud", - "Support Tool", - "Social network", - "Person", - "Other" - ], - "category_type_mappings": { - "Antivirus detection": [ - "link", - "comment", - "text", - "hex", - "attachment", - "other" - ], - "Artifacts dropped": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "regkey", - "regkey|value", - "pattern-in-file", - "pattern-in-memory", - "pdb", - "stix2-pattern", - "yara", - "sigma", - "attachment", - "malware-sample", - "named pipe", - "mutex", - "windows-scheduled-task", - "windows-service-name", - "windows-service-displayname", - "comment", - "text", - "hex", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "other", - "cookie", - "gene", - "mime-type" - ], - "Attribution": [ - "threat-actor", - "campaign-name", - "campaign-id", - "whois-registrant-phone", - "whois-registrant-email", - "whois-registrant-name", - "whois-registrant-org", - "whois-registrar", - "whois-creation-date", - "comment", - "text", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "other", - "dns-soa-email" - ], - "External analysis": [ - "md5", - "sha1", - "sha256", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha256", - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "mac-address", - "mac-eui-64", - "hostname", - "domain", - "domain|ip", - "url", - "user-agent", - "regkey", - "regkey|value", - "AS", - "snort", - "bro", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "vulnerability", - "attachment", - "malware-sample", - "link", - "comment", - "text", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "github-repository", - "other", - "cortex" - ], - "Financial fraud": [ - "btc", - "xmr", - "iban", - "bic", - "bank-account-nr", - "aba-rtn", - "bin", - "cc-number", - "prtn", - "phone-number", - "comment", - "text", - "other", - "hex" - ], - "Internal reference": [ - "text", - "link", - "comment", - "other", - "hex" - ], - "Network activity": [ - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "port", - "hostname", - "domain", - "domain|ip", - "mac-address", - "mac-eui-64", - "email-dst", - "url", - "uri", - "user-agent", - "http-method", - "AS", - "snort", - "pattern-in-file", - "stix2-pattern", - "pattern-in-traffic", - "attachment", - "comment", - "text", - "x509-fingerprint-md5", - "x509-fingerprint-sha1", - "x509-fingerprint-sha256", - "other", - "hex", - "cookie", - "hostname|port", - "bro" - ], - "Other": [ - "comment", - "text", - "other", - "size-in-bytes", - "counter", - "datetime", - "cpe", - "port", - "float", - "hex", - "phone-number", - "boolean" - ], - "Payload delivery": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "pehash", - "tlsh", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "mac-address", - "mac-eui-64", - "ip-src", - "ip-dst", - "ip-dst|port", - "ip-src|port", - "hostname", - "domain", - "email-src", - "email-dst", - "email-subject", - "email-attachment", - "email-body", - "url", - "user-agent", - "AS", - "pattern-in-file", - "pattern-in-traffic", - "stix2-pattern", - "yara", - "sigma", - "mime-type", - "attachment", - "malware-sample", - "link", - "malware-type", - "comment", - "text", - "hex", - "vulnerability", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "other", - "hostname|port", - "email-dst-display-name", - "email-src-display-name", - "email-header", - "email-reply-to", - "email-x-mailer", - "email-mime-boundary", - "email-thread-index", - "email-message-id", - "mobile-application-id", - "whois-registrant-email" - ], - "Payload installation": [ - "md5", - "sha1", - "sha224", - "sha256", - "sha384", - "sha512", - "sha512/224", - "sha512/256", - "ssdeep", - "imphash", - "impfuzzy", - "authentihash", - "pehash", - "tlsh", - "filename", - "filename|md5", - "filename|sha1", - "filename|sha224", - "filename|sha256", - "filename|sha384", - "filename|sha512", - "filename|sha512/224", - "filename|sha512/256", - "filename|authentihash", - "filename|ssdeep", - "filename|tlsh", - "filename|imphash", - "filename|impfuzzy", - "filename|pehash", - "pattern-in-file", - "pattern-in-traffic", - "pattern-in-memory", - "stix2-pattern", - "yara", - "sigma", - "vulnerability", - "attachment", - "malware-sample", - "malware-type", - "comment", - "text", - "hex", - "x509-fingerprint-sha1", - "x509-fingerprint-md5", - "x509-fingerprint-sha256", - "mobile-application-id", - "other", - "mime-type" - ], - "Payload type": [ - "comment", - "text", - "other" - ], - "Persistence mechanism": [ - "filename", - "regkey", - "regkey|value", - "comment", - "text", - "other", - "hex" - ], - "Person": [ - "first-name", - "middle-name", - "last-name", - "date-of-birth", - "place-of-birth", - "gender", - "passport-number", - "passport-country", - "passport-expiration", - "redress-number", - "nationality", - "visa-number", - "issue-date-of-the-visa", - "primary-residence", - "country-of-residence", - "special-service-request", - "frequent-flyer-number", - "travel-details", - "payment-details", - "place-port-of-original-embarkation", - "place-port-of-clearance", - "place-port-of-onward-foreign-destination", - "passenger-name-record-locator-number", - "comment", - "text", - "other", - "phone-number", - "identity-card-number" - ], - "Social network": [ - "github-username", - "github-repository", - "github-organisation", - "jabber-id", - "twitter-id", - "email-src", - "email-dst", - "comment", - "text", - "other", - "whois-registrant-email" - ], - "Support Tool": [ - "link", - "text", - "attachment", - "comment", - "other", - "hex" - ], - "Targeting data": [ - "target-user", - "target-email", - "target-machine", - "target-org", - "target-location", - "target-external", - "comment" - ] - }, "sane_defaults": { - "AS": { - "default_category": "Network activity", - "to_ids": 0 - }, - "aba-rtn": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "attachment": { - "default_category": "External analysis", - "to_ids": 0 - }, - "authentihash": { + "md5": { "default_category": "Payload delivery", "to_ids": 1 }, - "bank-account-nr": { - "default_category": "Financial fraud", + "sha1": { + "default_category": "Payload delivery", "to_ids": 1 }, - "bic": { - "default_category": "Financial fraud", + "sha256": { + "default_category": "Payload delivery", "to_ids": 1 }, - "bin": { - "default_category": "Financial fraud", + "filename": { + "default_category": "Payload delivery", "to_ids": 1 }, - "boolean": { - "default_category": "Other", + "pdb": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "bro": { + "filename|md5": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha1": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ip-src": { "default_category": "Network activity", "to_ids": 1 }, - "btc": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "campaign-id": { - "default_category": "Attribution", - "to_ids": 0 - }, - "campaign-name": { - "default_category": "Attribution", - "to_ids": 0 - }, - "cc-number": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "comment": { - "default_category": "Other", - "to_ids": 0 - }, - "cookie": { + "ip-dst": { "default_category": "Network activity", - "to_ids": 0 + "to_ids": 1 }, - "cortex": { - "default_category": "External analysis", - "to_ids": 0 - }, - "counter": { - "default_category": "Other", - "to_ids": 0 - }, - "country-of-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "cpe": { - "default_category": "Other", - "to_ids": 0 - }, - "date-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "datetime": { - "default_category": "Other", - "to_ids": 0 - }, - "dns-soa-email": { - "default_category": "Attribution", - "to_ids": 0 + "hostname": { + "default_category": "Network activity", + "to_ids": 1 }, "domain": { "default_category": "Network activity", @@ -507,6 +53,18 @@ "default_category": "Network activity", "to_ids": 1 }, + "email-src": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "email-dst": { + "default_category": "Network activity", + "to_ids": 1 + }, + "email-subject": { + "default_category": "Payload delivery", + "to_ids": 0 + }, "email-attachment": { "default_category": "Payload delivery", "to_ids": 1 @@ -515,151 +73,11 @@ "default_category": "Payload delivery", "to_ids": 0 }, - "email-dst": { - "default_category": "Network activity", - "to_ids": 1 - }, - "email-dst-display-name": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-header": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-message-id": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-mime-boundary": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-reply-to": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-src": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "email-src-display-name": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-subject": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-thread-index": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "email-x-mailer": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "filename": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|authentihash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|impfuzzy": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|imphash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|pehash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha1": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha384": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512/224": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|sha512/256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|ssdeep": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "filename|tlsh": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "first-name": { - "default_category": "Person", - "to_ids": 0 - }, "float": { "default_category": "Other", "to_ids": 0 }, - "frequent-flyer-number": { - "default_category": "Person", - "to_ids": 0 - }, - "gender": { - "default_category": "Person", - "to_ids": 0 - }, - "gene": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "github-organisation": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-repository": { - "default_category": "Social network", - "to_ids": 0 - }, - "github-username": { - "default_category": "Social network", - "to_ids": 0 - }, - "hex": { - "default_category": "Other", - "to_ids": 0 - }, - "hostname": { - "default_category": "Network activity", - "to_ids": 1 - }, - "hostname|port": { + "url": { "default_category": "Network activity", "to_ids": 1 }, @@ -667,177 +85,21 @@ "default_category": "Network activity", "to_ids": 0 }, - "iban": { - "default_category": "Financial fraud", - "to_ids": 1 - }, - "identity-card-number": { - "default_category": "Person", - "to_ids": 0 - }, - "impfuzzy": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "imphash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "ip-dst": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-dst|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-src": { - "default_category": "Network activity", - "to_ids": 1 - }, - "ip-src|port": { - "default_category": "Network activity", - "to_ids": 1 - }, - "issue-date-of-the-visa": { - "default_category": "Person", - "to_ids": 0 - }, - "jabber-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "last-name": { - "default_category": "Person", - "to_ids": 0 - }, - "link": { - "default_category": "External analysis", - "to_ids": 0 - }, - "mac-address": { + "user-agent": { "default_category": "Network activity", "to_ids": 0 }, - "mac-eui-64": { - "default_category": "Network activity", - "to_ids": 0 - }, - "malware-sample": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "malware-type": { - "default_category": "Payload delivery", - "to_ids": 0 - }, - "md5": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "middle-name": { - "default_category": "Person", - "to_ids": 0 - }, - "mime-type": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "mobile-application-id": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "mutex": { - "default_category": "Artifacts dropped", - "to_ids": 1 - }, - "named pipe": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "nationality": { - "default_category": "Person", - "to_ids": 0 - }, - "other": { - "default_category": "Other", - "to_ids": 0 - }, - "passenger-name-record-locator-number": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-country": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-expiration": { - "default_category": "Person", - "to_ids": 0 - }, - "passport-number": { - "default_category": "Person", - "to_ids": 0 - }, - "pattern-in-file": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-memory": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "pattern-in-traffic": { + "ja3-fingerprint-md5": { "default_category": "Network activity", "to_ids": 1 }, - "payment-details": { - "default_category": "Person", - "to_ids": 0 - }, - "pdb": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "pehash": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "phone-number": { - "default_category": "Person", - "to_ids": 0 - }, - "place-of-birth": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-clearance": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-onward-foreign-destination": { - "default_category": "Person", - "to_ids": 0 - }, - "place-port-of-original-embarkation": { - "default_category": "Person", - "to_ids": 0 - }, - "port": { + "hassh-md5": { "default_category": "Network activity", - "to_ids": 0 - }, - "primary-residence": { - "default_category": "Person", - "to_ids": 0 - }, - "prtn": { - "default_category": "Financial fraud", "to_ids": 1 }, - "redress-number": { - "default_category": "Person", - "to_ids": 0 + "hasshserver-md5": { + "default_category": "Network activity", + "to_ids": 1 }, "regkey": { "default_category": "Persistence mechanism", @@ -847,7 +109,203 @@ "default_category": "Persistence mechanism", "to_ids": 1 }, - "sha1": { + "AS": { + "default_category": "Network activity", + "to_ids": 0 + }, + "snort": { + "default_category": "Network activity", + "to_ids": 1 + }, + "bro": { + "default_category": "Network activity", + "to_ids": 1 + }, + "zeek": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-file": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "pattern-in-traffic": { + "default_category": "Network activity", + "to_ids": 1 + }, + "pattern-in-memory": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "yara": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "stix2-pattern": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "sigma": { + "default_category": "Payload installation", + "to_ids": 1 + }, + "gene": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mime-type": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "identity-card-number": { + "default_category": "Person", + "to_ids": 0 + }, + "cookie": { + "default_category": "Network activity", + "to_ids": 0 + }, + "vulnerability": { + "default_category": "External analysis", + "to_ids": 0 + }, + "attachment": { + "default_category": "External analysis", + "to_ids": 0 + }, + "malware-sample": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "link": { + "default_category": "External analysis", + "to_ids": 0 + }, + "comment": { + "default_category": "Other", + "to_ids": 0 + }, + "text": { + "default_category": "Other", + "to_ids": 0 + }, + "hex": { + "default_category": "Other", + "to_ids": 0 + }, + "other": { + "default_category": "Other", + "to_ids": 0 + }, + "named pipe": { + "default_category": "Artifacts dropped", + "to_ids": 0 + }, + "mutex": { + "default_category": "Artifacts dropped", + "to_ids": 1 + }, + "target-user": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-email": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-machine": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-org": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-location": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "target-external": { + "default_category": "Targeting data", + "to_ids": 0 + }, + "btc": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "xmr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "iban": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bic": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bank-account-nr": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "aba-rtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "bin": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "cc-number": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "prtn": { + "default_category": "Financial fraud", + "to_ids": 1 + }, + "phone-number": { + "default_category": "Person", + "to_ids": 0 + }, + "threat-actor": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-name": { + "default_category": "Attribution", + "to_ids": 0 + }, + "campaign-id": { + "default_category": "Attribution", + "to_ids": 0 + }, + "malware-type": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "uri": { + "default_category": "Network activity", + "to_ids": 1 + }, + "authentihash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "impfuzzy": { "default_category": "Payload delivery", "to_ids": 1 }, @@ -855,10 +313,6 @@ "default_category": "Payload delivery", "to_ids": 1 }, - "sha256": { - "default_category": "Payload delivery", - "to_ids": 1 - }, "sha384": { "default_category": "Payload delivery", "to_ids": 1 @@ -875,102 +329,78 @@ "default_category": "Payload delivery", "to_ids": 1 }, - "sigma": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "size-in-bytes": { - "default_category": "Other", - "to_ids": 0 - }, - "snort": { - "default_category": "Network activity", - "to_ids": 1 - }, - "special-service-request": { - "default_category": "Person", - "to_ids": 0 - }, - "ssdeep": { - "default_category": "Payload delivery", - "to_ids": 1 - }, - "stix2-pattern": { - "default_category": "Payload installation", - "to_ids": 1 - }, - "target-email": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-external": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-location": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-machine": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-org": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "target-user": { - "default_category": "Targeting data", - "to_ids": 0 - }, - "text": { - "default_category": "Other", - "to_ids": 0 - }, - "threat-actor": { - "default_category": "Attribution", - "to_ids": 0 - }, "tlsh": { "default_category": "Payload delivery", "to_ids": 1 }, - "travel-details": { - "default_category": "Person", - "to_ids": 0 - }, - "twitter-id": { - "default_category": "Social network", - "to_ids": 0 - }, - "uri": { - "default_category": "Network activity", + "cdhash": { + "default_category": "Payload delivery", "to_ids": 1 }, - "url": { - "default_category": "Network activity", + "filename|authentihash": { + "default_category": "Payload delivery", "to_ids": 1 }, - "user-agent": { - "default_category": "Network activity", + "filename|ssdeep": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|imphash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|impfuzzy": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|pehash": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha384": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/224": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|sha512/256": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "filename|tlsh": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "windows-scheduled-task": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "visa-number": { - "default_category": "Person", + "windows-service-name": { + "default_category": "Artifacts dropped", "to_ids": 0 }, - "vulnerability": { - "default_category": "External analysis", - "to_ids": 0 - }, - "whois-creation-date": { - "default_category": "Attribution", + "windows-service-displayname": { + "default_category": "Artifacts dropped", "to_ids": 0 }, "whois-registrant-email": { "default_category": "Attribution", "to_ids": 0 }, + "whois-registrant-phone": { + "default_category": "Attribution", + "to_ids": 0 + }, "whois-registrant-name": { "default_category": "Attribution", "to_ids": 0 @@ -979,31 +409,19 @@ "default_category": "Attribution", "to_ids": 0 }, - "whois-registrant-phone": { - "default_category": "Attribution", - "to_ids": 0 - }, "whois-registrar": { "default_category": "Attribution", "to_ids": 0 }, - "windows-scheduled-task": { - "default_category": "Artifacts dropped", + "whois-creation-date": { + "default_category": "Attribution", "to_ids": 0 }, - "windows-service-displayname": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "windows-service-name": { - "default_category": "Artifacts dropped", - "to_ids": 0 - }, - "x509-fingerprint-md5": { + "x509-fingerprint-sha1": { "default_category": "Network activity", "to_ids": 1 }, - "x509-fingerprint-sha1": { + "x509-fingerprint-md5": { "default_category": "Network activity", "to_ids": 1 }, @@ -1011,13 +429,209 @@ "default_category": "Network activity", "to_ids": 1 }, - "xmr": { - "default_category": "Financial fraud", + "dns-soa-email": { + "default_category": "Attribution", + "to_ids": 0 + }, + "size-in-bytes": { + "default_category": "Other", + "to_ids": 0 + }, + "counter": { + "default_category": "Other", + "to_ids": 0 + }, + "datetime": { + "default_category": "Other", + "to_ids": 0 + }, + "cpe": { + "default_category": "Other", + "to_ids": 0 + }, + "port": { + "default_category": "Network activity", + "to_ids": 0 + }, + "ip-dst|port": { + "default_category": "Network activity", "to_ids": 1 }, - "yara": { - "default_category": "Payload installation", + "ip-src|port": { + "default_category": "Network activity", "to_ids": 1 + }, + "hostname|port": { + "default_category": "Network activity", + "to_ids": 1 + }, + "mac-address": { + "default_category": "Network activity", + "to_ids": 0 + }, + "mac-eui-64": { + "default_category": "Network activity", + "to_ids": 0 + }, + "email-dst-display-name": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-src-display-name": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-header": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-reply-to": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-x-mailer": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-mime-boundary": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-thread-index": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "email-message-id": { + "default_category": "Payload delivery", + "to_ids": 0 + }, + "github-username": { + "default_category": "Social network", + "to_ids": 0 + }, + "github-repository": { + "default_category": "Social network", + "to_ids": 0 + }, + "github-organisation": { + "default_category": "Social network", + "to_ids": 0 + }, + "jabber-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "twitter-id": { + "default_category": "Social network", + "to_ids": 0 + }, + "first-name": { + "default_category": "Person", + "to_ids": 0 + }, + "middle-name": { + "default_category": "Person", + "to_ids": 0 + }, + "last-name": { + "default_category": "Person", + "to_ids": 0 + }, + "date-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "place-of-birth": { + "default_category": "Person", + "to_ids": 0 + }, + "gender": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-number": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-country": { + "default_category": "Person", + "to_ids": 0 + }, + "passport-expiration": { + "default_category": "Person", + "to_ids": 0 + }, + "redress-number": { + "default_category": "Person", + "to_ids": 0 + }, + "nationality": { + "default_category": "Person", + "to_ids": 0 + }, + "visa-number": { + "default_category": "Person", + "to_ids": 0 + }, + "issue-date-of-the-visa": { + "default_category": "Person", + "to_ids": 0 + }, + "primary-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "country-of-residence": { + "default_category": "Person", + "to_ids": 0 + }, + "special-service-request": { + "default_category": "Person", + "to_ids": 0 + }, + "frequent-flyer-number": { + "default_category": "Person", + "to_ids": 0 + }, + "travel-details": { + "default_category": "Person", + "to_ids": 0 + }, + "payment-details": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-original-embarkation": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-clearance": { + "default_category": "Person", + "to_ids": 0 + }, + "place-port-of-onward-foreign-destination": { + "default_category": "Person", + "to_ids": 0 + }, + "passenger-name-record-locator-number": { + "default_category": "Person", + "to_ids": 0 + }, + "mobile-application-id": { + "default_category": "Payload delivery", + "to_ids": 1 + }, + "cortex": { + "default_category": "External analysis", + "to_ids": 0 + }, + "boolean": { + "default_category": "Other", + "to_ids": 0 + }, + "anonymised": { + "default_category": "Other", + "to_ids": 0 } }, "types": [ @@ -1043,11 +657,15 @@ "url", "http-method", "user-agent", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", "regkey", "regkey|value", "AS", "snort", "bro", + "zeek", "pattern-in-file", "pattern-in-traffic", "pattern-in-memory", @@ -1100,6 +718,7 @@ "sha512/224", "sha512/256", "tlsh", + "cdhash", "filename|authentihash", "filename|ssdeep", "filename|imphash", @@ -1172,7 +791,448 @@ "passenger-name-record-locator-number", "mobile-application-id", "cortex", - "boolean" - ] + "boolean", + "anonymised" + ], + "categories": [ + "Internal reference", + "Targeting data", + "Antivirus detection", + "Payload delivery", + "Artifacts dropped", + "Payload installation", + "Persistence mechanism", + "Network activity", + "Payload type", + "Attribution", + "External analysis", + "Financial fraud", + "Support Tool", + "Social network", + "Person", + "Other" + ], + "category_type_mappings": { + "Internal reference": [ + "text", + "link", + "comment", + "other", + "hex", + "anonymised" + ], + "Targeting data": [ + "target-user", + "target-email", + "target-machine", + "target-org", + "target-location", + "target-external", + "comment", + "anonymised" + ], + "Antivirus detection": [ + "link", + "comment", + "text", + "hex", + "attachment", + "other", + "anonymised" + ], + "Payload delivery": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "pehash", + "tlsh", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "mac-address", + "mac-eui-64", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "hostname", + "domain", + "email-src", + "email-dst", + "email-subject", + "email-attachment", + "email-body", + "url", + "user-agent", + "AS", + "pattern-in-file", + "pattern-in-traffic", + "stix2-pattern", + "yara", + "sigma", + "mime-type", + "attachment", + "malware-sample", + "link", + "malware-type", + "comment", + "text", + "hex", + "vulnerability", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", + "other", + "hostname|port", + "email-dst-display-name", + "email-src-display-name", + "email-header", + "email-reply-to", + "email-x-mailer", + "email-mime-boundary", + "email-thread-index", + "email-message-id", + "mobile-application-id", + "whois-registrant-email", + "anonymised" + ], + "Artifacts dropped": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "regkey", + "regkey|value", + "pattern-in-file", + "pattern-in-memory", + "pdb", + "stix2-pattern", + "yara", + "sigma", + "attachment", + "malware-sample", + "named pipe", + "mutex", + "windows-scheduled-task", + "windows-service-name", + "windows-service-displayname", + "comment", + "text", + "hex", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "other", + "cookie", + "gene", + "mime-type", + "anonymised" + ], + "Payload installation": [ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha512/224", + "sha512/256", + "ssdeep", + "imphash", + "impfuzzy", + "authentihash", + "pehash", + "tlsh", + "cdhash", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha224", + "filename|sha256", + "filename|sha384", + "filename|sha512", + "filename|sha512/224", + "filename|sha512/256", + "filename|authentihash", + "filename|ssdeep", + "filename|tlsh", + "filename|imphash", + "filename|impfuzzy", + "filename|pehash", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "stix2-pattern", + "yara", + "sigma", + "vulnerability", + "attachment", + "malware-sample", + "malware-type", + "comment", + "text", + "hex", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "mobile-application-id", + "other", + "mime-type", + "anonymised" + ], + "Persistence mechanism": [ + "filename", + "regkey", + "regkey|value", + "comment", + "text", + "other", + "hex", + "anonymised" + ], + "Network activity": [ + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "port", + "hostname", + "domain", + "domain|ip", + "mac-address", + "mac-eui-64", + "email-dst", + "url", + "uri", + "user-agent", + "http-method", + "AS", + "snort", + "pattern-in-file", + "stix2-pattern", + "pattern-in-traffic", + "attachment", + "comment", + "text", + "x509-fingerprint-md5", + "x509-fingerprint-sha1", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", + "other", + "hex", + "cookie", + "hostname|port", + "bro", + "zeek", + "anonymised" + ], + "Payload type": [ + "comment", + "text", + "other", + "anonymised" + ], + "Attribution": [ + "threat-actor", + "campaign-name", + "campaign-id", + "whois-registrant-phone", + "whois-registrant-email", + "whois-registrant-name", + "whois-registrant-org", + "whois-registrar", + "whois-creation-date", + "comment", + "text", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "other", + "dns-soa-email", + "anonymised" + ], + "External analysis": [ + "md5", + "sha1", + "sha256", + "filename", + "filename|md5", + "filename|sha1", + "filename|sha256", + "ip-src", + "ip-dst", + "ip-dst|port", + "ip-src|port", + "mac-address", + "mac-eui-64", + "hostname", + "domain", + "domain|ip", + "url", + "user-agent", + "regkey", + "regkey|value", + "AS", + "snort", + "bro", + "zeek", + "pattern-in-file", + "pattern-in-traffic", + "pattern-in-memory", + "vulnerability", + "attachment", + "malware-sample", + "link", + "comment", + "text", + "x509-fingerprint-sha1", + "x509-fingerprint-md5", + "x509-fingerprint-sha256", + "ja3-fingerprint-md5", + "hassh-md5", + "hasshserver-md5", + "github-repository", + "other", + "cortex", + "anonymised" + ], + "Financial fraud": [ + "btc", + "xmr", + "iban", + "bic", + "bank-account-nr", + "aba-rtn", + "bin", + "cc-number", + "prtn", + "phone-number", + "comment", + "text", + "other", + "hex", + "anonymised" + ], + "Support Tool": [ + "link", + "text", + "attachment", + "comment", + "other", + "hex", + "anonymised" + ], + "Social network": [ + "github-username", + "github-repository", + "github-organisation", + "jabber-id", + "twitter-id", + "email-src", + "email-dst", + "comment", + "text", + "other", + "whois-registrant-email", + "anonymised" + ], + "Person": [ + "first-name", + "middle-name", + "last-name", + "date-of-birth", + "place-of-birth", + "gender", + "passport-number", + "passport-country", + "passport-expiration", + "redress-number", + "nationality", + "visa-number", + "issue-date-of-the-visa", + "primary-residence", + "country-of-residence", + "special-service-request", + "frequent-flyer-number", + "travel-details", + "payment-details", + "place-port-of-original-embarkation", + "place-port-of-clearance", + "place-port-of-onward-foreign-destination", + "passenger-name-record-locator-number", + "comment", + "text", + "other", + "phone-number", + "identity-card-number", + "anonymised" + ], + "Other": [ + "comment", + "text", + "other", + "size-in-bytes", + "counter", + "datetime", + "cpe", + "port", + "float", + "hex", + "phone-number", + "boolean", + "anonymised" + ] + } } } diff --git a/pymisp/data/misp-objects b/pymisp/data/misp-objects index 11a462e..75ae30f 160000 --- a/pymisp/data/misp-objects +++ b/pymisp/data/misp-objects @@ -1 +1 @@ -Subproject commit 11a462e79b02428a08b11698d45aa8aa5ab6887d +Subproject commit 75ae30f44df997280255eec60b981b9f376c5ac4 diff --git a/pymisp/exceptions.py b/pymisp/exceptions.py index 481720b..1d7663f 100644 --- a/pymisp/exceptions.py +++ b/pymisp/exceptions.py @@ -67,3 +67,7 @@ class PyMISPNotImplementedYet(PyMISPError): class PyMISPUnexpectedResponse(PyMISPError): pass + + +class PyMISPEmptyResponse(PyMISPError): + pass diff --git a/pymisp/mispevent.py b/pymisp/mispevent.py index 68263b7..a1cfa8b 100644 --- a/pymisp/mispevent.py +++ b/pymisp/mispevent.py @@ -15,13 +15,11 @@ from . import deprecated from .abstract import AbstractMISP from .exceptions import UnknownMISPObjectTemplate, InvalidMISPObject, PyMISPError, NewEventError, NewAttributeError -import six # Remove that import when discarding python2 support. - import logging logger = logging.getLogger('pymisp') -if six.PY2: +if sys.version_info < (3, 0): logger.warning("You're using python 2, it is strongly recommended to use python >=3.6") # This is required because Python 2 is a pain. @@ -372,15 +370,18 @@ class MISPEvent(AbstractMISP): self.__json_schema = json.loads(f.read().decode()) else: self.__json_schema = json.load(f) - if not describe_types: + if describe_types: + # This variable is used in add_attribute in order to avoid duplicating the structure + self._describe_types = describe_types + else: with open(os.path.join(ressources_path, 'describeTypes.json'), 'rb') as f: if OLD_PY3: t = json.loads(f.read().decode()) else: t = json.load(f) - describe_types = t['result'] + self._describe_types = t['result'] - self._types = describe_types['types'] + self._types = self._describe_types['types'] self.Attribute = [] self.Object = [] self.RelatedEvent = [] @@ -464,12 +465,7 @@ class MISPEvent(AbstractMISP): event = json_event if not event: raise PyMISPError('Invalid event') - # Invalid event created by MISP up to 2.4.52 (attribute_count is none instead of '0') - if (event.get('Event') and - 'attribute_count' in event.get('Event') and - event.get('Event').get('attribute_count') is None): - event['Event']['attribute_count'] = '0' - self.from_dict(**event['Event']) + self.from_dict(**event) if validate: jsonschema.validate(json.loads(self.to_json()), self.__json_schema) @@ -477,6 +473,8 @@ class MISPEvent(AbstractMISP): """Set a date for the event (string, datetime, or date object)""" if isinstance(date, basestring) or isinstance(date, unicode): self.date = parse(date).date() + elif isinstance(date, int): + self.date = datetime.datetime.utcfromtimestamp(date).date() elif isinstance(date, datetime.datetime): self.date = date.date() elif isinstance(date, datetime.date): @@ -488,17 +486,19 @@ class MISPEvent(AbstractMISP): raise NewEventError('Invalid format for the date: {} - {}'.format(date, type(date))) def from_dict(self, **kwargs): + if kwargs.get('Event'): + kwargs = kwargs.get('Event') # Required value self.info = kwargs.pop('info', None) if self.info is None: - raise NewAttributeError('The info field of the new event is required.') + raise NewEventError('The info field of the new event is required.') # Default values for a valid event to send to a MISP instance self.distribution = kwargs.pop('distribution', None) if self.distribution is not None: self.distribution = int(self.distribution) if self.distribution not in [0, 1, 2, 3, 4]: - raise NewAttributeError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) + raise NewEventError('{} is invalid, the distribution has to be in 0, 1, 2, 3, 4'.format(self.distribution)) if kwargs.get('threat_level_id') is not None: self.threat_level_id = int(kwargs.pop('threat_level_id')) @@ -599,10 +599,10 @@ class MISPEvent(AbstractMISP): ''' tags = [] for a in self.attributes + [attribute for o in self.objects for attribute in o.attributes]: - if ((hasattr(a, 'id') and a.id == attribute_identifier) or - (hasattr(a, 'uuid') and a.uuid == attribute_identifier) or - (hasattr(a, 'value') and attribute_identifier == a.value or - attribute_identifier in a.value.split('|'))): + if ((hasattr(a, 'id') and a.id == attribute_identifier) + or (hasattr(a, 'uuid') and a.uuid == attribute_identifier) + or (hasattr(a, 'value') and attribute_identifier == a.value + or attribute_identifier in a.value.split('|'))): tags += a.tags return tags @@ -613,10 +613,10 @@ class MISPEvent(AbstractMISP): ''' attributes = [] for a in self.attributes + [attribute for o in self.objects for attribute in o.attributes]: - if ((hasattr(a, 'id') and a.id == attribute_identifier) or - (hasattr(a, 'uuid') and a.uuid == attribute_identifier) or - (hasattr(a, 'value') and attribute_identifier == a.value or - attribute_identifier in a.value.split('|'))): + if ((hasattr(a, 'id') and a.id == attribute_identifier) + or (hasattr(a, 'uuid') and a.uuid == attribute_identifier) + or (hasattr(a, 'value') and attribute_identifier == a.value + or attribute_identifier in a.value.split('|'))): a.add_tag(tag) attributes.append(a) @@ -637,8 +637,8 @@ class MISPEvent(AbstractMISP): """Delete an attribute, you can search by ID or UUID""" found = False for a in self.attributes: - if ((hasattr(a, 'id') and a.id == attribute_id) or - (hasattr(a, 'uuid') and a.uuid == attribute_id)): + if ((hasattr(a, 'id') and a.id == attribute_id) + or (hasattr(a, 'uuid') and a.uuid == attribute_id)): a.delete() found = True break @@ -652,7 +652,7 @@ class MISPEvent(AbstractMISP): if isinstance(value, list): attr_list = [self.add_attribute(type=type, value=a, **kwargs) for a in value] else: - attribute = MISPAttribute() + attribute = MISPAttribute(describe_types=self._describe_types) attribute.from_dict(type=type, value=value, **kwargs) self.attributes.append(attribute) self.edited = True @@ -1004,6 +1004,8 @@ class MISPObject(AbstractMISP): raise PyMISPError('All the attributes have to be of type MISPObjectReference.') def from_dict(self, **kwargs): + if kwargs.get('Object'): + kwargs = kwargs.get('Object') if self._known_template: if kwargs.get('template_uuid') and kwargs['template_uuid'] != self.template_uuid: if self._strict: diff --git a/pymisp/tools/abstractgenerator.py b/pymisp/tools/abstractgenerator.py index dc4c592..71520db 100644 --- a/pymisp/tools/abstractgenerator.py +++ b/pymisp/tools/abstractgenerator.py @@ -1,16 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import abc -import six from .. import MISPObject from ..exceptions import InvalidMISPObject from datetime import datetime, date from dateutil.parser import parse -@six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. -# Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): class AbstractMISPObjectGenerator(MISPObject): def _detect_epoch(self, timestamp): diff --git a/pymisp/tools/create_misp_object.py b/pymisp/tools/create_misp_object.py index 4f6e745..ffbfb46 100644 --- a/pymisp/tools/create_misp_object.py +++ b/pymisp/tools/create_misp_object.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import six +import sys from . import FileObject, PEObject, ELFObject, MachOObject from ..exceptions import MISPObjectException @@ -57,7 +57,7 @@ def make_binary_objects(filepath=None, pseudofile=None, filename=None, standalon if filepath: lief_parsed = lief.parse(filepath=filepath) else: - if six.PY2: + if sys.version_info < (3, 0): logger.critical('Pseudofile is not supported in python2. Just update.') lief_parsed = None else: diff --git a/setup.py b/setup.py index fb39a0d..a546fb2 100644 --- a/setup.py +++ b/setup.py @@ -39,17 +39,18 @@ setup( 'Topic :: Security', 'Topic :: Internet', ], - install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'setuptools>=36.4', 'python-dateutil', 'enum34;python_version<"3.4"'], + install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', + 'python-dateutil', 'enum34;python_version<"3.4"', + 'functools32;python_version<"3.0"'], extras_require={'fileobjects': ['lief>=0.8', 'python-magic'], 'neo': ['py2neo'], 'openioc': ['beautifulsoup4'], 'virustotal': ['validators'], - 'warninglists': ['pymispwarninglists']}, + 'docs': ['sphinx-autodoc-typehints']}, tests_require=[ 'jsonschema', 'python-magic', - 'requests-mock', - 'six' + 'requests-mock' ], test_suite="tests.test_offline", include_package_data=True, diff --git a/tests/test_offline.py b/tests/test_offline.py index 485443f..2a5178f 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -5,7 +5,6 @@ import unittest import requests_mock import json import os -import six import sys from io import BytesIO @@ -308,7 +307,7 @@ class TestOffline(unittest.TestCase): return json.dumps(to_return, cls=MISPEncode) def test_objects_pseudofile(self, m): - if six.PY2: + if sys.version_info < (3, 0): return unittest.SkipTest() paths = ['cmd.exe', 'tmux', 'MachO-OSX-x64-ls'] try: diff --git a/tests/testlive_comprehensive.py b/tests/testlive_comprehensive.py index f646e26..4c7e673 100644 --- a/tests/testlive_comprehensive.py +++ b/tests/testlive_comprehensive.py @@ -1,25 +1,43 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import sys + + import unittest -from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject +from pymisp.tools import make_binary_objects from datetime import datetime, timedelta, date from io import BytesIO +import re +import json import time +from uuid import uuid4 + +import logging +logging.disable(logging.CRITICAL) + +try: + from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation, MISPUser, Distribution, ThreatLevel, Analysis, MISPObject +except ImportError: + if sys.version_info < (3, 6): + print('This test suite requires Python 3.6+, breaking.') + sys.exit(0) + else: + raise try: from keys import url, key + verifycert = False travis_run = True except ImportError as e: print(e) url = 'http://localhost:8080' - key = 'LBelWqKY9SQyG0huZzAMqiEBl6FODxpgRRXMsZFu' + key = 'E36iZ8hr31XIcWfqU4NGniwhMuvYRngip6O1dC9T' + verifycert = False travis_run = False -from uuid import uuid4 - class TestComprehensive(unittest.TestCase): @@ -27,7 +45,7 @@ class TestComprehensive(unittest.TestCase): def setUpClass(cls): cls.maxDiff = None # Connect as admin - cls.admin_misp_connector = ExpandedPyMISP(url, key, debug=False) + cls.admin_misp_connector = ExpandedPyMISP(url, key, verifycert, debug=False) # Creates an org org = cls.admin_misp_connector.add_organisation(name='Test Org') cls.test_org = MISPOrganisation() @@ -36,12 +54,12 @@ class TestComprehensive(unittest.TestCase): usr = cls.admin_misp_connector.add_user(email='testusr@user.local', org_id=cls.test_org.id, role_id=3) cls.test_usr = MISPUser() cls.test_usr.from_dict(**usr) - cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey) + cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey, verifycert, debug=False) # Creates a publisher pub = cls.admin_misp_connector.add_user(email='testpub@user.local', org_id=cls.test_org.id, role_id=4) cls.test_pub = MISPUser() cls.test_pub.from_dict(**pub) - cls.pub_misp_connector = ExpandedPyMISP(url, cls.test_pub.authkey) + cls.pub_misp_connector = ExpandedPyMISP(url, cls.test_pub.authkey, verifycert) @classmethod def tearDownClass(cls): @@ -83,6 +101,7 @@ class TestComprehensive(unittest.TestCase): second_event.add_attribute('text', str(uuid4())) second_event.attributes[0].add_tag('tlp:white___test') second_event.add_attribute('ip-dst', '1.1.1.1') + second_event.attributes[1].add_tag('tlp:amber___test') # Same value as in first event. second_event.add_attribute('text', first_event.attributes[0].value) @@ -212,9 +231,9 @@ class TestComprehensive(unittest.TestCase): for e in events: self.assertIn(e.id, [first.id, second.id, third.id]) events = self.admin_misp_connector.search(tags='tlp:amber___test', pythonify=True) - self.assertEqual(len(events), 1) + self.assertEqual(len(events), 2) for e in events: - self.assertIn(e.id, [third.id]) + self.assertIn(e.id, [second.id, third.id]) events = self.admin_misp_connector.search(tags='admin_only', pythonify=True) self.assertEqual(len(events), 1) for e in events: @@ -225,9 +244,9 @@ class TestComprehensive(unittest.TestCase): for e in events: self.assertIn(e.id, [second.id, third.id]) events = self.user_misp_connector.search(tags='tlp:amber___test', pythonify=True) - self.assertEqual(len(events), 1) + self.assertEqual(len(events), 2) for e in events: - self.assertIn(e.id, [third.id]) + self.assertIn(e.id, [second.id, third.id]) events = self.user_misp_connector.search(tags='admin_only', pythonify=True) self.assertEqual(events, []) finally: @@ -244,16 +263,19 @@ class TestComprehensive(unittest.TestCase): attributes = self.admin_misp_connector.search(controller='attributes', tags='tlp:white___test', pythonify=True) self.assertEqual(len(attributes), 5) attributes = self.admin_misp_connector.search(controller='attributes', tags='tlp:amber___test', pythonify=True) - self.assertEqual(len(attributes), 2) + self.assertEqual(len(attributes), 3) attributes = self.admin_misp_connector.search(tags='admin_only', pythonify=True) self.assertEqual(len(attributes), 1) # Search as user attributes = self.user_misp_connector.search(controller='attributes', tags='tlp:white___test', pythonify=True) self.assertEqual(len(attributes), 4) attributes = self.user_misp_connector.search(controller='attributes', tags='tlp:amber___test', pythonify=True) - self.assertEqual(len(attributes), 2) + self.assertEqual(len(attributes), 3) attributes = self.user_misp_connector.search(tags='admin_only', pythonify=True) self.assertEqual(attributes, []) + attributes_tags_search = self.admin_misp_connector.build_complex_query(or_parameters=['tlp:amber___test'], not_parameters=['tlp:white___test']) + attributes = self.user_misp_connector.search(controller='attributes', tags=attributes_tags_search, pythonify=True) + self.assertEqual(len(attributes), 1) finally: # Delete event self.admin_misp_connector.delete_event(first.id) @@ -590,10 +612,6 @@ class TestComprehensive(unittest.TestCase): self.assertEqual(len(events), 1) self.assertEqual(events[0].id, second.id) self.assertEqual(len(events[0].attributes), 1) - events = self.user_misp_connector.search(timestamp=timeframe, to_ids='exclude', pythonify=True) - self.assertEqual(len(events), 2) - self.assertEqual(len(events[0].attributes), 1) - self.assertEqual(len(events[1].attributes), 1) # deleted second.attributes[1].delete() @@ -754,7 +772,7 @@ class TestComprehensive(unittest.TestCase): second = self.user_misp_connector.add_event(second) response = self.user_misp_connector.fast_publish(first.id, alert=False) - self.assertEqual(response['errors'][0][1]['message'], 'You do not have permission to use this functionality.') + self.assertEqual(response['errors'][1]['message'], 'You do not have permission to use this functionality.') # Default search, attribute with to_ids == True first.attributes[0].to_ids = True @@ -820,6 +838,22 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.delete_event(first.id) self.admin_misp_connector.delete_event(second.id) + def test_search_stix(self): + first = self.create_simple_event() + first.add_attribute('ip-src', '8.8.8.8') + try: + first = self.user_misp_connector.add_event(first) + if not travis_run: + stix = self.user_misp_connector.search(return_format='stix', eventid=first.id) + found = re.findall('8.8.8.8', stix) + self.assertTrue(found) + stix2 = self.user_misp_connector.search(return_format='stix2', eventid=first.id) + json.dumps(stix2, indent=2) + self.assertEqual(stix2['objects'][-1]['pattern'], "[network-traffic:src_ref.type = 'ipv4-addr' AND network-traffic:src_ref.value = '8.8.8.8']") + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + def test_upload_sample(self): first = self.create_simple_event() second = self.create_simple_event() @@ -880,6 +914,24 @@ class TestComprehensive(unittest.TestCase): self.admin_misp_connector.enable_tag(tag['id']) # FIXME: returns the tag with ID 1 + def test_add_event_with_attachment(self): + first = self.create_simple_event() + try: + first = self.user_misp_connector.add_event(first) + file_obj, bin_obj, sections = make_binary_objects('tests/viper-test-files/test_files/whoami.exe', standalone=False) + first.add_object(file_obj) + first.add_object(bin_obj) + for s in sections: + first.add_object(s) + self.assertEqual(len(first.objects[0].references), 1) + self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in') + first = self.user_misp_connector.update_event(first) + self.assertEqual(len(first.objects[0].references), 1) + self.assertEqual(first.objects[0].references[0].relationship_type, 'included-in') + finally: + # Delete event + self.admin_misp_connector.delete_event(first.id) + def test_taxonomies(self): # Make sure we're up-to-date self.admin_misp_connector.update_taxonomies()