MISP
/
PyMISP
mirror of https://github.com/MISP/PyMISP
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of github.com:MISP/PyMISP

pull/527/head
Alexandre Dulaunoy 2020-01-21 09:46:05 +01:00
parent a5c8f1844a adf97dfeff
commit eabc6481d0
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
28 changed files with 3215 additions and 6037 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
.travis.yml
View File

@ -9,27 +9,13 @@ addons:
- libstdc++6 - libstdc++6
- libfuzzy-dev - libfuzzy-dev
python:
matrix: - "3.6"
include: - "3.6-dev"
- name: "Python 2.7 - legacy" - "3.7"
python: 2.7 - "3.7-dev"
env: LEGACY=true - "3.8"
- name: "Python 3.5" - "3.8-dev"
python: 3.5
dist: xenial
- name: "Python 3.6"
python: 3.6
dist: xenial
- name: "Python 3.6 - Dev"
python: 3.6-dev
dist: xenial
- name: "Python 3.7"
python: 3.7
dist: xenial
- name: "Python 3.7 - Dev"
python: 3.7-dev
dist: xenial
install: install:
- bash travis/install_travis.sh - bash travis/install_travis.sh

66
CHANGELOG.txt
View File

@ -2,6 +2,71 @@ Changelog
========= =========
v2.4.120 (2020-01-17)
---------------------
New
~~~
- [attribute type] kusto-query attribute type. [Alexandre Dulaunoy]
Kusto query is the query language for the Kusto services in Azure used
to search large dataset. It's used in Windows Defender ATP Hunting-Queries
and also Azure Sentinel (Cloud-native SIEM).
- Remove python < 3.6 support. [Raphaël Vinot]
Changes
~~~~~~~
- Bump version. [Raphaël Vinot]
- Bump Changelog. [Raphaël Vinot]
- Bump misp-objects. [Raphaël Vinot]
- Bump dependencies, add debug. [Raphaël Vinot]
- Upate dummy events creator. [Raphaël Vinot]
- Add tests on more version of Python. [Raphaël Vinot]
- Search with the STIX output returns a json STIX. [Raphaël Vinot]
Was XML before.
- Bump dependencies. [Raphaël Vinot]
- Add more typing information. [Raphaël Vinot]
- Add typing markup. [Raphaël Vinot]
- Bump misp-objects. [Raphaël Vinot]
- Bump Dependencies. [Raphaël Vinot]
- Bump misp-objects. [Raphaël Vinot]
Fix
~~~
- Add missing variable in dummy creator. [Raphaël Vinot]
- Et2misp was python2 only. [Raphaël Vinot]
- Feed generator was broken. [Raphaël Vinot]
Fix #506
- Event without hashable attribute. [Raphaël Vinot]
Related #506
Other
~~~~~
- Update api.py. [AaronK]
minor typo, can;t help it noticing those. sorry,
- Fixed TODO, added quarantineFolder/quarantineRule from
messagesBlocked, added some error handling to prevent empty attributes
from trying to be added. [th3jiv3r]
- Scrape proofpoint tap api for messages blocked/delivered & clicks
blocked/permitted and create misp events. [th3jiv3r]
- Add variable for proofpoint tap api auth. [th3jiv3r]
- Update README.md. [AaronK]
minor typo
- Define the number of entries to output. [AndreC10002]
Allow for defining in the settings.py file the number of entries to output
- Update generate.py. [AndreC10002]
- Cleanup of code and 'quick-n-dirty' sanitizing of tags. [Koen Van
Impe]
- Sync. [Koen Van Impe]
- Update README.md. [Raphaël Vinot]
v2.4.119.1 (2019-12-17) v2.4.119.1 (2019-12-17)
----------------------- -----------------------
@ -11,6 +76,7 @@ New
Changes Changes
~~~~~~~ ~~~~~~~
- Bump changelog. [Raphaël Vinot]
- Version bump. [Raphaël Vinot] - Version bump. [Raphaël Vinot]
- Bump test files. [Raphaël Vinot] - Bump test files. [Raphaël Vinot]
- Bump misp-objects. [Raphaël Vinot] - Bump misp-objects. [Raphaël Vinot]

1
Pipfile
View File

@ -11,6 +11,7 @@ requests-mock = "*"
pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal", "pdfexport", "docs"],path = "."} pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal", "pdfexport", "docs"],path = "."}
docutils = "==0.15" docutils = "==0.15"
memory-profiler = "*" memory-profiler = "*"
mypy = "*"
[packages] [packages]
pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."} pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."}

488
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "4be7259a433785d74e1879a4a555bb669d50c5f409d0a094652c1abc9b1227c5" "sha256": "77accb43d4bbba1ff86f29a66cae21eb56bc19ce82e39b096763dfaf65a9d5d8"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -25,11 +25,11 @@
}, },
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a",
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931", "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887",
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae"
], ],
"version": "==4.8.1" "version": "==4.8.2"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
@ -68,11 +68,11 @@
}, },
"importlib-metadata": { "importlib-metadata": {
"hashes": [ "hashes": [
"sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21", "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
"sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742" "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8"
], ],
"markers": "python_version < '3.8'", "markers": "python_version < '3.8'",
"version": "==1.1.0" "version": "==1.4.0"
}, },
"jsonschema": { "jsonschema": {
"hashes": [ "hashes": [
@ -102,45 +102,37 @@
}, },
"more-itertools": { "more-itertools": {
"hashes": [ "hashes": [
"sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39",
"sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288"
], ],
"version": "==8.0.0" "version": "==8.1.0"
}, },
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be",
"sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", "sha256:4d9ed9a64095e031435af120d3c910148067087541131e82b3e8db302f4c8946",
"sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", "sha256:54ebae163e8412aff0b9df1e88adab65788f5f5b58e625dc5c7f51eaf14a6837",
"sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", "sha256:5bfef0b1cdde9f33881c913af14e43db69815c7e8df429ceda4c70a5e529210f",
"sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", "sha256:5f3546ceb08089cedb9e8ff7e3f6a7042bb5b37c2a95d392fb027c3e53a2da00",
"sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", "sha256:5f7ae9126d16194f114435ebb79cc536b5682002a4fa57fa7bb2cbcde65f2f4d",
"sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", "sha256:62a889aeb0a79e50ecf5af272e9e3c164148f4bd9636cc6bcfa182a52c8b0533",
"sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", "sha256:7406f5a9b2fd966e79e6abdaf700585a4522e98d6559ce37fc52e5c955fade0a",
"sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", "sha256:8453f914f4e5a3d828281a6628cf517832abfa13ff50679a4848926dac7c0358",
"sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", "sha256:87269cc6ce1e3dee11f23fa515e4249ae678dbbe2704598a51cee76c52e19cda",
"sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", "sha256:875358310ed7abd5320f21dd97351d62de4929b0426cdb1eaa904b64ac36b435",
"sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", "sha256:8ac6ce7ff3892e5deaab7abaec763538ffd011f74dc1801d93d3c5fc541feee2",
"sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", "sha256:91b710e3353aea6fc758cdb7136d9bbdcb26b53cefe43e2cba953ac3ee1d3313",
"sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", "sha256:9d2ba4ed13af381233e2d810ff3bab84ef9f18430a9b336ab69eaf3cd24299ff",
"sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", "sha256:a62ec5e13e227399be73303ff301f2865bf68657d15ea50b038d25fc41097317",
"sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", "sha256:ab76e5580b0ed647a8d8d2d2daee170e8e9f8aad225ede314f684e297e3643c2",
"sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", "sha256:bf4003aa538af3f4205c5fac56eacaa67a6dd81e454ffd9e9f055fff9f1bc614",
"sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", "sha256:bf598d2e37cf8edb1a2f26ed3fb255191f5232badea4003c16301cb94ac5bdd0",
"sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", "sha256:c18f70dc27cc5d236f10e7834236aff60aadc71346a5bc1f4f83a4b3abee6386",
"sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", "sha256:c5ed816632204a2fc9486d784d8e0d0ae754347aba99c811458d69fcdfd2a2f9",
"sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", "sha256:dc058b7833184970d1248135b8b0ab702e6daa833be14035179f2acb78ff5636",
"sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", "sha256:ff3797f2f16bf9d17d53257612da84dd0758db33935777149b3334c01ff68865"
"sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9",
"sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1",
"sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a",
"sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96",
"sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132",
"sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a",
"sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5",
"sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0"
], ],
"version": "==6.2.1" "version": "==7.0.0"
}, },
"pydeep": { "pydeep": {
"hashes": [ "hashes": [
@ -165,9 +157,9 @@
}, },
"pyrsistent": { "pyrsistent": {
"hashes": [ "hashes": [
"sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b" "sha256:cdc7b5e3ed77bed61270a47d35434a30617b9becdf2478af76ad2c6ade307280"
], ],
"version": "==0.15.6" "version": "==0.15.7"
}, },
"python-dateutil": { "python-dateutil": {
"hashes": [ "hashes": [
@ -185,36 +177,36 @@
}, },
"reportlab": { "reportlab": {
"hashes": [ "hashes": [
"sha256:149f0eeb4ea716441638b05fd6d3667d32f1463f3eac50b63e100a73a5533cdd", "sha256:2a1c4ea2155fd5b6e3f89e36b8aa21b5a14c9bbaf9b44de2787641668bc95edc",
"sha256:1aa9a2e1a87749db265b592ad25e498b39f70fce9f53a012cdf69f74259b6e43", "sha256:2b7469a98df1315d4f52319c4438eaee3fdd17330830edadae775e9312402638",
"sha256:1f5ce489adb2db2862249492e6367539cfa65b781cb06dcf13363dc52219be7e", "sha256:3b556160aac294fa661545245e4bc273328f9226e5110139647f4d4bc0cfc453",
"sha256:23b28ba1784a6c52a926c075abd9f396d03670e71934b24db5ff684f8b870e0f", "sha256:3eb25d2c2bde078815d8f7ea400abbcae16a0c498a4b27ead3c4a620b1f1f980",
"sha256:3d3de0f4facdd7e3c56ecbc55733a958b86c35a8e7ba6066c7b1ba383e282f58", "sha256:3f229c0b2ca27eb5b08777981d3bd0d34e59bfa306627b88d80c3734cd3e26d5",
"sha256:484d346b8f463ba2ddaf6d365c6ac5971cd062528b6d5ba68cac02b9435366c5", "sha256:4695755cc70b7a9308508aa41eafc3f335348be0eadd86e8f92cb87815d6177b",
"sha256:4da2467def21f2e20720b21f6c18e7f7866720a955c716b990e94e3979fe913f", "sha256:4f97b4474e419ae5c441ecdf0db8eceb5f5af0461bdf73e3e5ec05353844045c",
"sha256:5ebdf22daee7d8e630134d94f477fe6abd65a65449d4eec682a7b458b5249604", "sha256:550d2d8516e468192e12be8aeaf80f3bd805dc46dd0a5a4ddf2a3e1cd8149a16",
"sha256:655a1b68be18a73fec5233fb5d81f726b4db32269e487aecf5b6853cca926d86", "sha256:59aa9c4ca80d397f6cabec092b5a6e2304fb1b7ca53e5b650872aae13ebfeb68",
"sha256:6c535a304888dafe50c2c24d4924aeefc11e0542488ee6965f6133d415e86bbc", "sha256:6e4479b75778b9c1e4640dc90efb72cb990471d56089947d6be4ccd9e7a56a3c",
"sha256:7560ef655ac6448bb257fd34bfdfb8d546f9c7c0900ed8963fb8509f75e8ca80", "sha256:6e9434bd0afa6d6fcf9abbc565750cc456b6e60dc49abd7cd2bc7cf414ee079b",
"sha256:7a1c2fa3e6310dbe47efee2020dc0f25be7a75ff09a8fedc4a87d4397f3810c1", "sha256:73e4e30b72da1f9f8caba775ad9cc027957c2340c38ba2d6622a9f2351b12c3a",
"sha256:817c344b9aa53b5bfc2f58ff82111a1e85ca4c8b68d1add088b547360a6ebcfa", "sha256:7c05c2ba8ab32f02b23a56a75a4d136c2bfb7221a04a8306835a938fa6711644",
"sha256:81d950e398d6758aeaeeb267aa1a62940735414c980f77dd0a270cef1782a43d", "sha256:849e4cabce1ed1183e83dc89570810b3bf9bf9cf0d0a605bde854a0baf212124",
"sha256:83ef44936ef4e9c432d62bc2b72ec8d772b87af319d123e827a72e9b6884c851", "sha256:863c6fcf5fc0c8184b6315885429f5468373a3def2eb0c0073d09b79b2161113",
"sha256:9f975adc2c7a236403f0bc91d7a3916e644e47b1f1e3990325f15e73b83581ec", "sha256:8e688df260682038ecd32f106d796024fbcf70e7bf54340b14f991bd5465f97a",
"sha256:a5ca59e2b7e70a856de6db9dadd3e11a1b3b471c999585284d5c1d479c01cf5d", "sha256:9675a26d01ec141cb717091bb139b6227bfb3794f521943101da50327bff4825",
"sha256:ad2cf5a673c05fae9e91e987994b95205c13c5fa55d7393cf8b06f9de6f92990", "sha256:969b0d9663c0c641347d2408d41e6723e84d9f7863babc94438c91295c74f36d",
"sha256:b8c3d76276372f87b7c8ff22065dbc072cca5ffb06ba0267edc298df7acf942d", "sha256:978560732758bf5fca4ec1ed124afe2702d08824f6b0364cca31519bd5e7dadd",
"sha256:b93f7f908e916d9413dd8c04da1ccb3977e446803f59078424decdc0de449133", "sha256:99ea85b47248c6cdbece147bdbd67aed16209bdd95770aa1f151ec3bb8794496",
"sha256:c0ecd0af92c759edec0d24ba92f4a18c28d4a19229ae7c8249f94e82f3d76288", "sha256:9cdc318c37fa959909db5beb05ca0b684d3e2cba8f40af1ce6f332c3f69bd2b8",
"sha256:c9e38eefc90a02c072a87a627ff66b2d67c23f6f82274d2aa7fb28e644e8f409", "sha256:b55c26510ff7f135af8eae1216372028cde7dab22003d918649fce219020eb58",
"sha256:ca2a1592d2e181a04372d0276ee847308ea206dfe7c86fe94769e7ac126e6e85", "sha256:cb301340b4fc1f2b7b25ea4584c5cbde139ced2d4ff01ad5e8fcf7d7822982b0",
"sha256:ce1dfc9beec83e66250ca3afaf5ddf6b9a3ce70a30a9526dec7c6bec3266baf1", "sha256:e7578a573454a5490553fb091374996d32269dff44021a401763080bda1357cf",
"sha256:d3550c90751132b26b72a78954905974f33b1237335fbe0d8be957f9636c376a", "sha256:e84387d35a666aafafda332afca8a75fb04f097cc0a2dc2d04e8c90a83cf7c1b",
"sha256:e35a574f4e5ec0fdd5dc354e74ec143d853abd7f76db435ffe2a57d0161a22eb", "sha256:eb66eff64ea75f028af3ac63a7a2bf1e8733297141a85cbdffd5deaef404fa52",
"sha256:ee5cafca6ef1a38fef8cbf3140dd2198ad1ee82331530b546039216ef94f93cb", "sha256:f5e3afd2cc35a73f34c3084c69fe4653591611da5189e50b58db550bb46e340a",
"sha256:fa1c969176cb3594a785c6818bcb943ebd49453791f702380b13a35fa23b385a" "sha256:f6c10628386bfe0c1f6640c28fb262d0960bb26c249cefabb755fb273323220d"
], ],
"version": "==3.5.32" "version": "==3.5.34"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
@ -225,10 +217,10 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
], ],
"version": "==1.13.0" "version": "==1.14.0"
}, },
"soupsieve": { "soupsieve": {
"hashes": [ "hashes": [
@ -246,9 +238,9 @@
}, },
"validators": { "validators": {
"hashes": [ "hashes": [
"sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" "sha256:0bfe836a1af37bb266d71ec1e98b530c38ce11bc7fbe0c4c96ef7b1532d019e5"
], ],
"version": "==0.14.0" "version": "==0.14.1"
}, },
"wrapt": { "wrapt": {
"hashes": [ "hashes": [
@ -258,10 +250,10 @@
}, },
"zipp": { "zipp": {
"hashes": [ "hashes": [
"sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656",
"sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c"
], ],
"version": "==0.6.0" "version": "==1.0.0"
} }
}, },
"develop": { "develop": {
@ -281,18 +273,18 @@
}, },
"babel": { "babel": {
"hashes": [ "hashes": [
"sha256:af92e6106cb7c55286b25b38ad7695f8b4efb36a90ba483d7f7a6628c46158ab", "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38",
"sha256:e86135ae101e31e2c8ec20a4e0c5220f4eed12487d5cf3f78be7e98d3a57fc28" "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"
], ],
"version": "==2.7.0" "version": "==2.8.0"
}, },
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a",
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931", "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887",
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae"
], ],
"version": "==4.8.1" "version": "==4.8.2"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
@ -325,10 +317,10 @@
}, },
"colorama": { "colorama": {
"hashes": [ "hashes": [
"sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",
"sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"
], ],
"version": "==0.4.1" "version": "==0.4.3"
}, },
"commonmark": { "commonmark": {
"hashes": [ "hashes": [
@ -339,48 +331,47 @@
}, },
"coverage": { "coverage": {
"hashes": [ "hashes": [
"sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", "sha256:15cf13a6896048d6d947bf7d222f36e4809ab926894beb748fc9caa14605d9c3",
"sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", "sha256:1daa3eceed220f9fdb80d5ff950dd95112cd27f70d004c7918ca6dfc6c47054c",
"sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", "sha256:1e44a022500d944d42f94df76727ba3fc0a5c0b672c358b61067abb88caee7a0",
"sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", "sha256:25dbf1110d70bab68a74b4b9d74f30e99b177cde3388e07cc7272f2168bd1477",
"sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", "sha256:3230d1003eec018ad4a472d254991e34241e0bbd513e97a29727c7c2f637bd2a",
"sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", "sha256:3dbb72eaeea5763676a1a1efd9b427a048c97c39ed92e13336e726117d0b72bf",
"sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", "sha256:5012d3b8d5a500834783689a5d2292fe06ec75dc86ee1ccdad04b6f5bf231691",
"sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", "sha256:51bc7710b13a2ae0c726f69756cf7ffd4362f4ac36546e243136187cfcc8aa73",
"sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", "sha256:527b4f316e6bf7755082a783726da20671a0cc388b786a64417780b90565b987",
"sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", "sha256:722e4557c8039aad9592c6a4213db75da08c2cd9945320220634f637251c3894",
"sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", "sha256:76e2057e8ffba5472fd28a3a010431fd9e928885ff480cb278877c6e9943cc2e",
"sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", "sha256:77afca04240c40450c331fa796b3eab6f1e15c5ecf8bf2b8bee9706cd5452fef",
"sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", "sha256:7afad9835e7a651d3551eab18cbc0fdb888f0a6136169fbef0662d9cdc9987cf",
"sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", "sha256:9bea19ac2f08672636350f203db89382121c9c2ade85d945953ef3c8cf9d2a68",
"sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", "sha256:a8b8ac7876bc3598e43e2603f772d2353d9931709345ad6c1149009fd1bc81b8",
"sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", "sha256:b0840b45187699affd4c6588286d429cd79a99d509fe3de0f209594669bb0954",
"sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", "sha256:b26aaf69713e5674efbde4d728fb7124e429c9466aeaf5f4a7e9e699b12c9fe2",
"sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", "sha256:b63dd43f455ba878e5e9f80ba4f748c0a2156dde6e0e6e690310e24d6e8caf40",
"sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", "sha256:be18f4ae5a9e46edae3f329de2191747966a34a3d93046dbdf897319923923bc",
"sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", "sha256:c312e57847db2526bc92b9bfa78266bfbaabac3fdcd751df4d062cd4c23e46dc",
"sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", "sha256:c60097190fe9dc2b329a0eb03393e2e0829156a589bd732e70794c0dd804258e",
"sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", "sha256:c62a2143e1313944bf4a5ab34fd3b4be15367a02e9478b0ce800cb510e3bbb9d",
"sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", "sha256:cc1109f54a14d940b8512ee9f1c3975c181bbb200306c6d8b87d93376538782f",
"sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", "sha256:cd60f507c125ac0ad83f05803063bed27e50fa903b9c2cfee3f8a6867ca600fc",
"sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", "sha256:d513cc3db248e566e07a0da99c230aca3556d9b09ed02f420664e2da97eac301",
"sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", "sha256:d649dc0bcace6fcdb446ae02b98798a856593b19b637c1b9af8edadf2b150bea",
"sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", "sha256:d7008a6796095a79544f4da1ee49418901961c97ca9e9d44904205ff7d6aa8cb",
"sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", "sha256:da93027835164b8223e8e5af2cf902a4c80ed93cb0909417234f4a9df3bcd9af",
"sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", "sha256:e69215621707119c6baf99bda014a45b999d37602cb7043d943c76a59b05bf52",
"sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", "sha256:ea9525e0fef2de9208250d6c5aeeee0138921057cd67fcef90fbed49c4d62d37",
"sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", "sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0"
"sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025"
], ],
"version": "==4.5.4" "version": "==5.0.3"
}, },
"coveralls": { "coveralls": {
"hashes": [ "hashes": [
"sha256:9bc5a1f92682eef59f688a8f280207190d9a6afb84cef8f567fa47631a784060", "sha256:2da39aeaef986757653f0a442ba2bef22a8ec602c8bacbc69d39f468dfae12ec",
"sha256:fb51cddef4bc458de347274116df15d641a735d3f0a580a9472174e2e62f408c" "sha256:906e07a12b2ac04b8ad782d06173975fe5ff815fe9df3bfedd2c099bc5791aec"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.8.2" "version": "==1.10.0"
}, },
"decorator": { "decorator": {
"hashes": [ "hashes": [
@ -419,18 +410,18 @@
}, },
"imagesize": { "imagesize": {
"hashes": [ "hashes": [
"sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1",
"sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"
], ],
"version": "==1.1.0" "version": "==1.2.0"
}, },
"importlib-metadata": { "importlib-metadata": {
"hashes": [ "hashes": [
"sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21", "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
"sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742" "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8"
], ],
"markers": "python_version < '3.8'", "markers": "python_version < '3.8'",
"version": "==1.1.0" "version": "==1.4.0"
}, },
"jinja2": { "jinja2": {
"hashes": [ "hashes": [
@ -500,23 +491,50 @@
}, },
"memory-profiler": { "memory-profiler": {
"hashes": [ "hashes": [
"sha256:5fa47b274c929dd2cbcd9190afb62fec110701251d2ac2d301caaf545c81afc1" "sha256:23b196f91ea9ac9996e30bfab1e82fecc30a4a1d24870e81d1e81625f786a2c3"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.55.0" "version": "==0.57.0"
}, },
"more-itertools": { "more-itertools": {
"hashes": [ "hashes": [
"sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39",
"sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288"
], ],
"version": "==8.0.0" "version": "==8.1.0"
},
"mypy": {
"hashes": [
"sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a",
"sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7",
"sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2",
"sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474",
"sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0",
"sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217",
"sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749",
"sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6",
"sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf",
"sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36",
"sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b",
"sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72",
"sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1",
"sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1"
],
"index": "pypi",
"version": "==0.761"
},
"mypy-extensions": {
"hashes": [
"sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
"sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
],
"version": "==0.4.3"
}, },
"neobolt": { "neobolt": {
"hashes": [ "hashes": [
"sha256:56b86b8b2c3facdd54589e60ecd22e0234d6f40645ab2e2cf87ef0cd79df20af" "sha256:ca4e87679fe3ed39aec23638658e02dbdc6bbc3289a04e826f332e05ab32275d"
], ],
"version": "==1.7.15" "version": "==1.7.16"
}, },
"neotime": { "neotime": {
"hashes": [ "hashes": [
@ -535,45 +553,37 @@
}, },
"packaging": { "packaging": {
"hashes": [ "hashes": [
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb",
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8"
], ],
"version": "==19.2" "version": "==20.0"
}, },
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be",
"sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", "sha256:4d9ed9a64095e031435af120d3c910148067087541131e82b3e8db302f4c8946",
"sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", "sha256:54ebae163e8412aff0b9df1e88adab65788f5f5b58e625dc5c7f51eaf14a6837",
"sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", "sha256:5bfef0b1cdde9f33881c913af14e43db69815c7e8df429ceda4c70a5e529210f",
"sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", "sha256:5f3546ceb08089cedb9e8ff7e3f6a7042bb5b37c2a95d392fb027c3e53a2da00",
"sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", "sha256:5f7ae9126d16194f114435ebb79cc536b5682002a4fa57fa7bb2cbcde65f2f4d",
"sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", "sha256:62a889aeb0a79e50ecf5af272e9e3c164148f4bd9636cc6bcfa182a52c8b0533",
"sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", "sha256:7406f5a9b2fd966e79e6abdaf700585a4522e98d6559ce37fc52e5c955fade0a",
"sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", "sha256:8453f914f4e5a3d828281a6628cf517832abfa13ff50679a4848926dac7c0358",
"sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", "sha256:87269cc6ce1e3dee11f23fa515e4249ae678dbbe2704598a51cee76c52e19cda",
"sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", "sha256:875358310ed7abd5320f21dd97351d62de4929b0426cdb1eaa904b64ac36b435",
"sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", "sha256:8ac6ce7ff3892e5deaab7abaec763538ffd011f74dc1801d93d3c5fc541feee2",
"sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", "sha256:91b710e3353aea6fc758cdb7136d9bbdcb26b53cefe43e2cba953ac3ee1d3313",
"sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", "sha256:9d2ba4ed13af381233e2d810ff3bab84ef9f18430a9b336ab69eaf3cd24299ff",
"sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", "sha256:a62ec5e13e227399be73303ff301f2865bf68657d15ea50b038d25fc41097317",
"sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", "sha256:ab76e5580b0ed647a8d8d2d2daee170e8e9f8aad225ede314f684e297e3643c2",
"sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", "sha256:bf4003aa538af3f4205c5fac56eacaa67a6dd81e454ffd9e9f055fff9f1bc614",
"sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", "sha256:bf598d2e37cf8edb1a2f26ed3fb255191f5232badea4003c16301cb94ac5bdd0",
"sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", "sha256:c18f70dc27cc5d236f10e7834236aff60aadc71346a5bc1f4f83a4b3abee6386",
"sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", "sha256:c5ed816632204a2fc9486d784d8e0d0ae754347aba99c811458d69fcdfd2a2f9",
"sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", "sha256:dc058b7833184970d1248135b8b0ab702e6daa833be14035179f2acb78ff5636",
"sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", "sha256:ff3797f2f16bf9d17d53257612da84dd0758db33935777149b3334c01ff68865"
"sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9",
"sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1",
"sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a",
"sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96",
"sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132",
"sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a",
"sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5",
"sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0"
], ],
"version": "==6.2.1" "version": "==7.0.0"
}, },
"prompt-toolkit": { "prompt-toolkit": {
"hashes": [ "hashes": [
@ -630,16 +640,16 @@
}, },
"pyparsing": { "pyparsing": {
"hashes": [ "hashes": [
"sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
"sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
], ],
"version": "==2.4.5" "version": "==2.4.6"
}, },
"pyrsistent": { "pyrsistent": {
"hashes": [ "hashes": [
"sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b" "sha256:cdc7b5e3ed77bed61270a47d35434a30617b9becdf2478af76ad2c6ade307280"
], ],
"version": "==0.15.6" "version": "==0.15.7"
}, },
"python-dateutil": { "python-dateutil": {
"hashes": [ "hashes": [
@ -671,36 +681,36 @@
}, },
"reportlab": { "reportlab": {
"hashes": [ "hashes": [
"sha256:149f0eeb4ea716441638b05fd6d3667d32f1463f3eac50b63e100a73a5533cdd", "sha256:2a1c4ea2155fd5b6e3f89e36b8aa21b5a14c9bbaf9b44de2787641668bc95edc",
"sha256:1aa9a2e1a87749db265b592ad25e498b39f70fce9f53a012cdf69f74259b6e43", "sha256:2b7469a98df1315d4f52319c4438eaee3fdd17330830edadae775e9312402638",
"sha256:1f5ce489adb2db2862249492e6367539cfa65b781cb06dcf13363dc52219be7e", "sha256:3b556160aac294fa661545245e4bc273328f9226e5110139647f4d4bc0cfc453",
"sha256:23b28ba1784a6c52a926c075abd9f396d03670e71934b24db5ff684f8b870e0f", "sha256:3eb25d2c2bde078815d8f7ea400abbcae16a0c498a4b27ead3c4a620b1f1f980",
"sha256:3d3de0f4facdd7e3c56ecbc55733a958b86c35a8e7ba6066c7b1ba383e282f58", "sha256:3f229c0b2ca27eb5b08777981d3bd0d34e59bfa306627b88d80c3734cd3e26d5",
"sha256:484d346b8f463ba2ddaf6d365c6ac5971cd062528b6d5ba68cac02b9435366c5", "sha256:4695755cc70b7a9308508aa41eafc3f335348be0eadd86e8f92cb87815d6177b",
"sha256:4da2467def21f2e20720b21f6c18e7f7866720a955c716b990e94e3979fe913f", "sha256:4f97b4474e419ae5c441ecdf0db8eceb5f5af0461bdf73e3e5ec05353844045c",
"sha256:5ebdf22daee7d8e630134d94f477fe6abd65a65449d4eec682a7b458b5249604", "sha256:550d2d8516e468192e12be8aeaf80f3bd805dc46dd0a5a4ddf2a3e1cd8149a16",
"sha256:655a1b68be18a73fec5233fb5d81f726b4db32269e487aecf5b6853cca926d86", "sha256:59aa9c4ca80d397f6cabec092b5a6e2304fb1b7ca53e5b650872aae13ebfeb68",
"sha256:6c535a304888dafe50c2c24d4924aeefc11e0542488ee6965f6133d415e86bbc", "sha256:6e4479b75778b9c1e4640dc90efb72cb990471d56089947d6be4ccd9e7a56a3c",
"sha256:7560ef655ac6448bb257fd34bfdfb8d546f9c7c0900ed8963fb8509f75e8ca80", "sha256:6e9434bd0afa6d6fcf9abbc565750cc456b6e60dc49abd7cd2bc7cf414ee079b",
"sha256:7a1c2fa3e6310dbe47efee2020dc0f25be7a75ff09a8fedc4a87d4397f3810c1", "sha256:73e4e30b72da1f9f8caba775ad9cc027957c2340c38ba2d6622a9f2351b12c3a",
"sha256:817c344b9aa53b5bfc2f58ff82111a1e85ca4c8b68d1add088b547360a6ebcfa", "sha256:7c05c2ba8ab32f02b23a56a75a4d136c2bfb7221a04a8306835a938fa6711644",
"sha256:81d950e398d6758aeaeeb267aa1a62940735414c980f77dd0a270cef1782a43d", "sha256:849e4cabce1ed1183e83dc89570810b3bf9bf9cf0d0a605bde854a0baf212124",
"sha256:83ef44936ef4e9c432d62bc2b72ec8d772b87af319d123e827a72e9b6884c851", "sha256:863c6fcf5fc0c8184b6315885429f5468373a3def2eb0c0073d09b79b2161113",
"sha256:9f975adc2c7a236403f0bc91d7a3916e644e47b1f1e3990325f15e73b83581ec", "sha256:8e688df260682038ecd32f106d796024fbcf70e7bf54340b14f991bd5465f97a",
"sha256:a5ca59e2b7e70a856de6db9dadd3e11a1b3b471c999585284d5c1d479c01cf5d", "sha256:9675a26d01ec141cb717091bb139b6227bfb3794f521943101da50327bff4825",
"sha256:ad2cf5a673c05fae9e91e987994b95205c13c5fa55d7393cf8b06f9de6f92990", "sha256:969b0d9663c0c641347d2408d41e6723e84d9f7863babc94438c91295c74f36d",
"sha256:b8c3d76276372f87b7c8ff22065dbc072cca5ffb06ba0267edc298df7acf942d", "sha256:978560732758bf5fca4ec1ed124afe2702d08824f6b0364cca31519bd5e7dadd",
"sha256:b93f7f908e916d9413dd8c04da1ccb3977e446803f59078424decdc0de449133", "sha256:99ea85b47248c6cdbece147bdbd67aed16209bdd95770aa1f151ec3bb8794496",
"sha256:c0ecd0af92c759edec0d24ba92f4a18c28d4a19229ae7c8249f94e82f3d76288", "sha256:9cdc318c37fa959909db5beb05ca0b684d3e2cba8f40af1ce6f332c3f69bd2b8",
"sha256:c9e38eefc90a02c072a87a627ff66b2d67c23f6f82274d2aa7fb28e644e8f409", "sha256:b55c26510ff7f135af8eae1216372028cde7dab22003d918649fce219020eb58",
"sha256:ca2a1592d2e181a04372d0276ee847308ea206dfe7c86fe94769e7ac126e6e85", "sha256:cb301340b4fc1f2b7b25ea4584c5cbde139ced2d4ff01ad5e8fcf7d7822982b0",
"sha256:ce1dfc9beec83e66250ca3afaf5ddf6b9a3ce70a30a9526dec7c6bec3266baf1", "sha256:e7578a573454a5490553fb091374996d32269dff44021a401763080bda1357cf",
"sha256:d3550c90751132b26b72a78954905974f33b1237335fbe0d8be957f9636c376a", "sha256:e84387d35a666aafafda332afca8a75fb04f097cc0a2dc2d04e8c90a83cf7c1b",
"sha256:e35a574f4e5ec0fdd5dc354e74ec143d853abd7f76db435ffe2a57d0161a22eb", "sha256:eb66eff64ea75f028af3ac63a7a2bf1e8733297141a85cbdffd5deaef404fa52",
"sha256:ee5cafca6ef1a38fef8cbf3140dd2198ad1ee82331530b546039216ef94f93cb", "sha256:f5e3afd2cc35a73f34c3084c69fe4653591611da5189e50b58db550bb46e340a",
"sha256:fa1c969176cb3594a785c6818bcb943ebd49453791f702380b13a35fa23b385a" "sha256:f6c10628386bfe0c1f6640c28fb262d0960bb26c249cefabb755fb273323220d"
], ],
"version": "==3.5.32" "version": "==3.5.34"
}, },
"requests": { "requests": {
"hashes": [ "hashes": [
@ -719,10 +729,10 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
], ],
"version": "==1.13.0" "version": "==1.14.0"
}, },
"snowballstemmer": { "snowballstemmer": {
"hashes": [ "hashes": [
@ -740,10 +750,10 @@
}, },
"sphinx": { "sphinx": {
"hashes": [ "hashes": [
"sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd", "sha256:298537cb3234578b2d954ff18c5608468229e116a9757af3b831c2b2b4819159",
"sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79" "sha256:e6e766b74f85f37a5f3e0773a1e1be8db3fcb799deb58ca6d18b70b0b44542a5"
], ],
"version": "==2.2.1" "version": "==2.3.1"
}, },
"sphinx-autodoc-typehints": { "sphinx-autodoc-typehints": {
"hashes": [ "hashes": [
@ -794,6 +804,40 @@
], ],
"version": "==1.1.3" "version": "==1.1.3"
}, },
"typed-ast": {
"hashes": [
"sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
"sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
"sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
"sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
"sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
"sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
"sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
"sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
"sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
"sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
"sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
"sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
"sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
"sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
"sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
"sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
"sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
"sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
"sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
"sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
"sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
],
"version": "==1.4.1"
},
"typing-extensions": {
"hashes": [
"sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2",
"sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d",
"sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"
],
"version": "==3.7.4.1"
},
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
@ -803,16 +847,16 @@
}, },
"validators": { "validators": {
"hashes": [ "hashes": [
"sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" "sha256:0bfe836a1af37bb266d71ec1e98b530c38ce11bc7fbe0c4c96ef7b1532d019e5"
], ],
"version": "==0.14.0" "version": "==0.14.1"
}, },
"wcwidth": { "wcwidth": {
"hashes": [ "hashes": [
"sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603",
"sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"
], ],
"version": "==0.1.7" "version": "==0.1.8"
}, },
"wrapt": { "wrapt": {
"hashes": [ "hashes": [
@ -822,10 +866,10 @@
}, },
"zipp": { "zipp": {
"hashes": [ "hashes": [
"sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656",
"sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c"
], ],
"version": "==0.6.0" "version": "==1.0.0"
} }
} }
} }

6
docs/tutorial/FullOverview.ipynb
View File

@ -192,7 +192,7 @@
"source": [ "source": [
"## Set parameters (inline)\n", "## Set parameters (inline)\n",
"\n", "\n",
"This is the was to pass other parameters" "This is the way to pass other parameters"
] ]
}, },
{ {
@ -603,7 +603,7 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"## Use locally defined objet templates\n", "## Use locally defined object templates\n",
"\n", "\n",
"**Important**: The path you pass as parameter for `misp_objects_path_custom` needs to contain a directory equals to the value of the parameter `name` (same structure as the content of the `misp-object` repository)\n" "**Important**: The path you pass as parameter for `misp_objects_path_custom` needs to contain a directory equals to the value of the parameter `name` (same structure as the content of the `misp-object` repository)\n"
] ]
@ -654,7 +654,7 @@
"source": [ "source": [
"## Use lief to extract indicators out of binaries\n", "## Use lief to extract indicators out of binaries\n",
"\n", "\n",
"An other cool helper: one liner to whom you can pass the path to a binary, if it is supported by `lief` (PE/ELF/Mach-o), you get the the file object, a PE, ELF, or Mach-o object, and the relevant sections.\n", "An other cool helper: one liner to whom you can pass the path to a binary, if it is supported by `lief` (PE/ELF/Mach-o), you get the file object, a PE, ELF, or Mach-o object, and the relevant sections.\n",
"\n", "\n",
"If it is anything else, it will just generate the the file object.\n" "If it is anything else, it will just generate the the file object.\n"
] ]

6
examples/et2misp.py
View File

@ -71,7 +71,7 @@ def update_et_event(name):
for k,v in et_attr.items(): for k,v in et_attr.items():
r = mymisp.delete_attribute(v) r = mymisp.delete_attribute(v)
if r.get('errors'): if r.get('errors'):
print "Error deleting attribute {} ({}): {}\n".format(v,k,r['errors']) print("Error deleting attribute {} ({}): {}\n".format(v,k,r['errors']))
# Weed out ips already in the MISP event # Weed out ips already in the MISP event
for k,v in et_ips.items(): for k,v in et_ips.items():
@ -102,9 +102,9 @@ def update_et_event(name):
def echeck(r, eid=None): def echeck(r, eid=None):
if r.get('errors'): if r.get('errors'):
if eid: if eid:
print "Processing event {} failed: {}".format(eid, r['errors']) print("Processing event {} failed: {}".format(eid, r['errors']))
else: else:
print r['errors'] print(r['errors'])
sys.exit(1) sys.exit(1)
if __name__ == '__main__': if __name__ == '__main__':

8
examples/events/create_massive_dummy_events.py
View File

@ -4,9 +4,11 @@
from pymisp import ExpandedPyMISP from pymisp import ExpandedPyMISP
try: try:
from keys import url, key from keys import url, key
verifycert = False
except ImportError: except ImportError:
url = 'http://localhost:8080' url = 'https://localhost:8443'
key = '8h0gHbhS0fv6JUOlTED0AznLXFbf83TYtQrCycqb' key = 'd6OmdDFvU3Seau3UjwvHS1y3tFQbaRNhJhDX0tjh'
verifycert = False
import argparse import argparse
import tools import tools
@ -17,7 +19,7 @@ if __name__ == '__main__':
parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)") parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)")
args = parser.parse_args() args = parser.parse_args()
misp = ExpandedPyMISP(url, key, True) misp = ExpandedPyMISP(url, key, verifycert)
misp.toggle_global_pythonify() misp.toggle_global_pythonify()
if args.limit is None: if args.limit is None:

55
examples/events/tools.py
View File

@ -4,7 +4,7 @@
import random import random
from random import randint from random import randint
import string import string
from pymisp import MISPEvent from pymisp import MISPEvent, MISPAttribute
def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits): def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits):
@ -15,32 +15,34 @@ def randomIpGenerator():
return str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) return str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255))
def _attribute(category, type, value):
attribute = MISPAttribute()
attribute.category = category
attribute.type = type
attribute.value = value
return attribute
def floodtxt(misp, event, maxlength=255): def floodtxt(misp, event, maxlength=255):
text = randomStringGenerator(randint(1, maxlength)) text = randomStringGenerator(randint(1, maxlength))
textfunctions = [misp.add_internal_comment, misp.add_internal_text, misp.add_internal_other, misp.add_email_subject, misp.add_mutex, misp.add_filename] choose_from = [('Internal reference', 'comment', text), ('Internal reference', 'text', text),
textfunctions[randint(0, 5)](event, text) ('Internal reference', 'other', text), ('Network activity', 'email-subject', text),
('Artifacts dropped', 'mutex', text), ('Artifacts dropped', 'filename', text)]
misp.add_attribute(event, _attribute(*random.choice(choose_from)))
def floodip(misp, event): def floodip(misp, event):
ip = randomIpGenerator() ip = randomIpGenerator()
ipfunctions = [misp.add_ipsrc, misp.add_ipdst] choose_from = [('Network activity', 'ip-src', ip), ('Network activity', 'ip-dst', ip)]
ipfunctions[randint(0, 1)](event, ip) misp.add_attribute(event, _attribute(*random.choice(choose_from)))
def flooddomain(misp, event, maxlength=25): def flooddomain(misp, event, maxlength=25):
a = randomStringGenerator(randint(1, maxlength)) a = randomStringGenerator(randint(1, maxlength))
b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase)
domain = a + '.' + b domain = a + '.' + b
domainfunctions = [misp.add_hostname, misp.add_domain] choose_from = [('Network activity', 'domain', domain), ('Network activity', 'hostname', domain)]
domainfunctions[randint(0, 1)](event, domain) misp.add_attribute(event, _attribute(*random.choice(choose_from)))
def flooddomainip(misp, event, maxlength=25):
a = randomStringGenerator(randint(1, maxlength))
b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase)
domain = a + '.' + b
ip = randomIpGenerator()
misp.add_domain_ip(event, domain, ip)
def floodemail(misp, event, maxlength=25): def floodemail(misp, event, maxlength=25):
@ -48,19 +50,12 @@ def floodemail(misp, event, maxlength=25):
b = randomStringGenerator(randint(1, maxlength)) b = randomStringGenerator(randint(1, maxlength))
c = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) c = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase)
email = a + '@' + b + '.' + c email = a + '@' + b + '.' + c
emailfunctions = [misp.add_email_src, misp.add_email_dst] choose_from = [('Network activity', 'email-dst', email), ('Network activity', 'email-src', email)]
emailfunctions[randint(0, 1)](event, email) misp.add_attribute(event, _attribute(*random.choice(choose_from)))
def floodattachment(misp, eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id):
filename = randomStringGenerator(randint(1, 128))
misp.upload_sample(filename, 'dummy', eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id)
def create_dummy_event(misp): def create_dummy_event(misp):
event = misp.new_event(0, 4, 0, 'dummy event') return misp.new_event(0, 4, 0, 'dummy event')
flooddomainip(misp, event)
floodattachment(misp, event['Event']['id'], event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id'])
def create_massive_dummy_events(misp, nbattribute): def create_massive_dummy_events(misp, nbattribute):
@ -68,12 +63,6 @@ def create_massive_dummy_events(misp, nbattribute):
event.info = 'massive dummy event' event.info = 'massive dummy event'
event = misp.add_event(event) event = misp.add_event(event)
print(event) print(event)
eventid = event.id functions = [floodtxt, floodip, flooddomain, floodemail]
distribution = '0'
functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment]
for i in range(nbattribute): for i in range(nbattribute):
choice = randint(0, 5) functions[random.randint(0, len(functions) - 1)](misp, event)
if choice == 5:
floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event.info, event.analysis, event.threat_level_id)
else:
functions[choice](misp, event)

2
examples/feed-generator/README.md
View File

@ -7,7 +7,7 @@ This python script can be used to generate a MISP feed based on an existing MISP
```` ````
git clone https://github.com/MISP/PyMISP.git git clone https://github.com/MISP/PyMISP.git
cd examples/feed-generator cd examples/feed-generator
cp settings-default.py settings.py cp settings.default.py settings.py
vi settings.py #adjust your settings vi settings.py #adjust your settings
python3 generate.py python3 generate.py
```` ````

4
examples/feed-generator/generate.py
View File

@ -5,7 +5,7 @@ import sys
import json import json
import os import os
from pymisp import ExpandedPyMISP from pymisp import ExpandedPyMISP
from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels from settings import entries, url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
valid_attribute_distributions = [] valid_attribute_distributions = []
@ -52,7 +52,7 @@ def saveManifest(manifest):
if __name__ == '__main__': if __name__ == '__main__':
misp = init() misp = init()
try: try:
events = misp.search(metadata=True, limit=200, **filters, pythonify=True) events = misp.search(metadata=True, limit=entries, **filters, pythonify=True)
except Exception as e: except Exception as e:
print(e) print(e)
sys.exit("Invalid response received from MISP.") sys.exit("Invalid response received from MISP.")

3
examples/feed-generator/settings.default.py
View File

@ -12,6 +12,9 @@ ssl = False
# sure that you use a directory dedicated to the feed # sure that you use a directory dedicated to the feed
outputdir = 'output' outputdir = 'output'
# Determine the number of entries to output
entries = 200
# The filters to be used for by the feed. You can use any filter that # The filters to be used for by the feed. You can use any filter that
# you can use on the event index, such as organisation, tags, etc. # you can use on the event index, such as organisation, tags, etc.
# It uses the same joining and condition rules as the API parameters # It uses the same joining and condition rules as the API parameters

1
examples/keys.py.sample
View File

@ -5,3 +5,4 @@ misp_url = 'https://<your MISP URL>/'
misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section
misp_verifycert = True misp_verifycert = True
misp_client_cert = '' misp_client_cert = ''
proofpoint_key = 'Your Proofpoint TAP auth key'

201
examples/proofpoint_tap.py Normal file
View File

@ -0,0 +1,201 @@
import requests
import json
from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation
from keys import misp_url, misp_key, misp_verifycert, proofpoint_key
# initialize PyMISP and set url for Panorama
misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert)
urlSiem = "https://tap-api-v2.proofpoint.com/v2/siem/all"
alertType = ("messagesDelivered", "messagesBlocked", "clicksPermitted", "clicksBlocked")
# max query is 1h, and we want Proofpoint TAP api to return json
queryString = {
"sinceSeconds": "3600",
"format": "json"
}
# auth to api needs to be set as a header, not as part of the query string
headers = {
'Authorization': "Basic " + proofpoint_key
}
responseSiem = requests.request("GET", urlSiem, headers=headers, params=queryString)
jsonDataSiem = json.loads(responseSiem.text)
for alert in alertType:
for messages in jsonDataSiem[alert]:
orgc = MISPOrganisation()
orgc.name = 'Proofpoint'
orgc.id = '#{ORGC.ID}' # organisation id
orgc.uuid = '#{ORGC.UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
if alert == "messagesDelivered" or alert == "messagesBlocked":
if alert == "messagesDelivered":
event.info = alert
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # setting this to 0 breaks the integration
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
else:
event.info = alert
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # BLOCKED = LOW
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
recipient = event.add_attribute('email-dst', messages["recipient"][0])
recipient.comment = 'recipient address'
sender = event.add_attribute('email-src', messages["sender"])
sender.comment = 'sender address'
if messages["fromAddress"] is not None and messages["fromAddress"] != "" :
fromAddress = event.add_attribute('email-src-display-name', messages["fromAddress"])
headerFrom = event.add_attribute('email-header', messages["headerFrom"])
headerFrom.comment = 'email header from'
senderIP = event.add_attribute('ip-src', messages["senderIP"])
senderIP.comment = 'sender IP'
subject = event.add_attribute('email-subject', messages["subject"])
subject.comment = 'email subject'
if messages["quarantineFolder"] is not None and messages["quarantineFolder"] != "":
quarantineFolder = event.add_attribute('comment', messages["quarantineFolder"])
quarantineFolder.comment = 'quarantine folder'
if messages["quarantineRule"] is not None and messages["quarantineRule"] != "":
quarantineRule = event.add_attribute('comment', messages["quarantineRule"])
quarantineRule.comment = 'quarantine rule'
messageSize = event.add_attribute('size-in-bytes', messages["messageSize"])
messageSize.comment = 'size of email in bytes'
malwareScore = event.add_attribute('comment', messages["malwareScore"])
malwareScore.comment = 'malware score'
phishScore = event.add_attribute('comment', messages["phishScore"])
phishScore.comment = 'phish score'
spamScore = event.add_attribute('comment', messages["spamScore"])
spamScore.comment = 'spam score'
imposterScore = event.add_attribute('comment', messages["impostorScore"])
imposterScore.comment = 'impostor score'
completelyRewritten = event.add_attribute('comment', messages["completelyRewritten"])
completelyRewritten.comment = 'proofpoint url defense'
# grab the threat info for each message in TAP
for threatInfo in messages["threatsInfoMap"]:
threat_type = {
"url": "url",
"attachment": "email-attachment",
"message": "email-body"
}
threat = event.add_attribute(threat_type.get(threatInfo["threatType"]), threatInfo["threat"])
threat.comment = 'threat'
threatUrl = event.add_attribute('link', threatInfo["threatUrl"])
threatUrl.comment = 'link to threat in TAP'
threatStatus = event.add_attribute('comment', threatInfo["threatStatus"])
threatStatus.comment = "proofpoint's threat status"
event.add_tag(threatInfo["classification"])
# get campaignID from each TAP alert and query campaign API
if threatInfo["campaignID"] is not None and threatInfo["campaignID"] != "":
urlCampaign = "https://tap-api-v2.proofpoint.com/v2/campaign/" + threatInfo["campaignID"]
responseCampaign = requests.request("GET", urlCampaign, headers=headers)
jsonDataCampaign = json.loads(responseCampaign.text)
campaignType = ("actors", "families", "malware", "techniques")
# loop through campaignType and grab tags to add to MISP event
for tagType in campaignType:
for tag in jsonDataCampaign[tagType]:
event.add_tag(tag['name'])
# grab which policy route the message took
for policy in messages["policyRoutes"]:
policyRoute = event.add_attribute('comment', policy)
policyRoute.comment = 'email policy route'
# was the threat in the body of the email or is it an attachment?
for parts in messages["messageParts"]:
disposition = event.add_attribute('comment', parts["disposition"])
disposition.comment = 'email body or attachment'
# sha256 hash of threat
if parts["sha256"] is not None and parts["sha256"] != "":
sha256 = event.add_attribute('sha256', parts["sha256"])
sha256.comment = 'sha256 hash'
# md5 hash of threat
if parts["md5"] is not None and parts["md5"] != "":
md5 = event.add_attribute('md5', parts["md5"])
md5.comment = 'md5 hash'
# filename of threat
if parts["filename"] is not None and parts["filename"] != "":
filename = event.add_attribute('filename', parts["filename"])
filename.comment = 'filename'
misp.add_event(event.to_json())
if alert == "clicksPermitted" or alert == "clicksBlocked":
if alert == "clicksPermitted":
print(alert + " is a permitted click")
event.info = alert
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # setting this to 0 breaks the integration
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
else:
print(alert + " is a blocked click")
event.info = alert
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # BLOCKED = LOW
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
event.add_tag(messages["classification"])
campaignId = event.add_attribute('campaign-id', messages["campaignId"][0])
campaignId.comment = 'campaignId'
clickIP = event.add_attribute('ip-src', messages["clickIP"])
clickIP.comment = 'clickIP'
clickTime = event.add_attribute('datetime', messages["clickTime"])
clickTime.comment = 'clicked threat'
threatTime = event.add_attribute('datetime', messages["threatTime"])
threatTime.comment = 'identified threat'
GUID = event.add_attribute('comment', messages["GUID"])
GUID.comment = 'PPS message ID'
recipient = event.add_attribute('email-dst', messages["recipient"][0])
recipient.comment = 'recipient address'
sender = event.add_attribute('email-src', messages["sender"])
sender.comment = 'sender address'
senderIP = event.add_attribute('ip-src', messages["senderIP"])
senderIP.comment = 'sender IP'
threatURL = event.add_attribute('link', messages["threatURL"])
threatURL.comment = 'link to threat in TAP'
url = event.add_attribute('link', messages["url"])
url.comment = 'malicious url clicked'
userAgent = event.add_attribute('user-agent', messages["userAgent"])
misp.add_event(event.to_json())

65
examples/proofpoint_vap.py Normal file
View File

@ -0,0 +1,65 @@
import requests
import json
from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation
from keys import misp_url, misp_key, misp_verifycert, proofpoint_key
# initialize PyMISP and set url for Panorama
misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert)
urlVap = "https://tap-api-v2.proofpoint.com/v2/people/vap?window=30" # Window can be 14, 30, and 90 Days
headers = {
'Authorization': "Basic " + proofpoint_key
}
responseVap = requests.request("GET", urlVap, headers=headers)
jsonDataVap = json.loads(responseVap.text)
for alert in jsonDataVap["users"]:
orgc = MISPOrganisation()
orgc.name = 'Proofpoint'
orgc.id = '#{ORGC.ID}' # organisation id
orgc.uuid = '#{ORGC.UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = 'Very Attacked Person ' + jsonDataVap["interval"]
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # setting this to 0 breaks the integration
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
totalVapUsers = event.add_attribute('counter', jsonDataVap["totalVapUsers"], comment="Total VAP Users")
averageAttackIndex = event.add_attribute('counter', jsonDataVap["averageAttackIndex"], comment="Average Attack Count")
vapAttackIndexThreshold = event.add_attribute('counter', jsonDataVap["vapAttackIndexThreshold"], comment="Attack Threshold")
emails = event.add_attribute('email-dst', alert["identity"]["emails"], comment="Email Destination")
attack = event.add_attribute('counter', alert["threatStatistics"]["attackIndex"], comment="Attack Count")
vip = event.add_attribute('other', str(alert["identity"]["vip"]), comment="VIP")
guid = event.add_attribute('other', alert["identity"]["guid"], comment="GUID")
if alert["identity"]["customerUserId"] is not None:
customerUserId = event.add_attribute('other', alert["identity"]["customerUserId"], comment="Customer User Id")
if alert["identity"]["department"] is not None:
department = event.add_attribute(alert['other', "identity"]["department"], comment="Department")
if alert["identity"]["location"] is not None:
location = event.add_attribute('other', alert["identity"]["location"], comment="Location")
if alert["identity"]["name"] is not None:
name = event.add_attribute('target-user', alert["identity"]["name"], comment="Name")
if alert["identity"]["title"] is not None:
title = event.add_attribute('other', alert["identity"]["title"], comment="Title")
event.add_tag("VAP")
misp.add_event(event.to_json())

48
pymisp/__init__.py
View File

@ -1,4 +1,4 @@
__version__ = '2.4.119.1' __version__ = '2.4.120'
import logging import logging
import warnings import warnings
import sys import sys
@ -13,24 +13,18 @@ logger.addHandler(default_handler)
logger.setLevel(logging.WARNING) logger.setLevel(logging.WARNING)
def warning_2020(): everything_broken = '''Unknown error: the response is not in JSON.
Something is broken server-side, please send us everything that follows (careful with the auth key):
if sys.version_info < (3, 6): Request headers:
warnings.warn(""" {}
Python 2.7 is officially end of life the 2020-01-01. For this occasion, Request body:
we decided to review which versions of Python we support and our conclusion {}
is to only support python 3.6+ starting the 2020-01-01. Response (if any):
{}'''
Every version of pymisp released after the 2020-01-01 will fail if the
python interpreter is prior to python 3.6.
**Please update your codebase.**""", DeprecationWarning, stacklevel=3)
try: try:
warning_2020()
from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse # noqa
from .api import PyMISP # noqa
from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa
from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPUserSetting # noqa from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation, MISPUserSetting # noqa
from .tools import AbstractMISPObjectGenerator # noqa from .tools import AbstractMISPObjectGenerator # noqa
@ -39,18 +33,18 @@ try:
from .tools import openioc # noqa from .tools import openioc # noqa
from .tools import ext_lookups # noqa from .tools import ext_lookups # noqa
if sys.version_info >= (3, 6): from .api import PyMISP # noqa
from .aping import ExpandedPyMISP # noqa from .api import PyMISP as ExpandedPyMISP # noqa
from .tools import load_warninglists # noqa from .tools import load_warninglists # noqa
# Let's not bother with old python # Let's not bother with old python
try: try:
from .tools import reportlab_generator # noqa from .tools import reportlab_generator # noqa
except ImportError: except ImportError:
# FIXME: The import should not raise an exception if reportlab isn't installed # FIXME: The import should not raise an exception if reportlab isn't installed
pass pass
except NameError: except NameError:
# FIXME: The import should not raise an exception if reportlab isn't installed # FIXME: The import should not raise an exception if reportlab isn't installed
pass pass
logger.debug('pymisp loaded properly') logger.debug('pymisp loaded properly')
except ImportError as e: except ImportError as e:
logger.warning('Unable to load pymisp properly: {}'.format(e)) logger.warning('Unable to load pymisp properly: {}'.format(e))

232
pymisp/abstract.py
View File

@ -1,114 +1,54 @@
#!/usr/bin/env python #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import sys
import datetime import datetime
from deprecated import deprecated from deprecated import deprecated # type: ignore
from json import JSONEncoder from json import JSONEncoder
from uuid import UUID from uuid import UUID
from abc import ABCMeta
try: try:
from rapidjson import load from rapidjson import load # type: ignore
from rapidjson import loads from rapidjson import loads # type: ignore
from rapidjson import dumps from rapidjson import dumps # type: ignore
import rapidjson
HAS_RAPIDJSON = True HAS_RAPIDJSON = True
except ImportError: except ImportError:
from json import load from json import load
from json import loads from json import loads
from json import dumps from json import dumps
import json
HAS_RAPIDJSON = False HAS_RAPIDJSON = False
import logging import logging
from enum import Enum from enum import Enum
from typing import Union, Optional
from .exceptions import PyMISPInvalidFormat, PyMISPError from .exceptions import PyMISPInvalidFormat, PyMISPError
from collections.abc import MutableMapping
from functools import lru_cache
from pathlib import Path
logger = logging.getLogger('pymisp') logger = logging.getLogger('pymisp')
if sys.version_info < (3, 0): resources_path = Path(__file__).parent / 'data'
from collections import MutableMapping misp_objects_path = resources_path / 'misp-objects' / 'objects'
import os with (resources_path / 'describeTypes.json').open('r') as f:
from cachetools import cached, LRUCache describe_types = load(f)['result']
resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects')
with open(os.path.join(resources_path, 'describeTypes.json'), 'r') as f:
describe_types = load(f)['result']
# This is required because Python 2 is a pain. class MISPFileCache(object):
from datetime import tzinfo, timedelta # cache up to 150 JSON structures in class attribute
class UTC(tzinfo): @staticmethod
"""UTC""" @lru_cache(maxsize=150)
def _load_json(path: Path) -> Union[dict, None]:
def utcoffset(self, dt): if not path.exists():
return timedelta(0) return None
with path.open('r') as f:
def tzname(self, dt): data = load(f)
return "UTC" return data
def dst(self, dt):
return timedelta(0)
class MISPFileCache(object):
# cache up to 150 JSON structures in class attribute
@staticmethod
@cached(cache=LRUCache(maxsize=150))
def _load_json(path):
if not os.path.exists(path):
return None
with open(path, 'r') as f:
data = load(f)
return data
elif sys.version_info < (3, 4):
from collections.abc import MutableMapping
from functools import lru_cache
import os
resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects')
with open(os.path.join(resources_path, 'describeTypes.json'), 'r') as f:
describe_types = load(f)['result']
class MISPFileCache(object):
# cache up to 150 JSON structures in class attribute
@staticmethod
@lru_cache(maxsize=150)
def _load_json(path):
if not os.path.exists(path):
return None
with open(path, 'r') as f:
data = load(f)
return data
else:
from collections.abc import MutableMapping
from functools import lru_cache
from pathlib import Path
resources_path = Path(__file__).parent / 'data'
misp_objects_path = resources_path / 'misp-objects' / 'objects'
with (resources_path / 'describeTypes.json').open('r') as f:
describe_types = load(f)['result']
class MISPFileCache(object):
# cache up to 150 JSON structures in class attribute
@staticmethod
@lru_cache(maxsize=150)
def _load_json(path):
if not path.exists():
return None
with path.open('r') as f:
data = load(f)
return data
class Distribution(Enum): class Distribution(Enum):
@ -133,7 +73,7 @@ class Analysis(Enum):
completed = 2 completed = 2
def _int_to_str(d): def _int_to_str(d: dict) -> dict:
# transform all integer back to string # transform all integer back to string
for k, v in d.items(): for k, v in d.items():
if isinstance(v, (int, float)) and not isinstance(v, bool): if isinstance(v, (int, float)) and not isinstance(v, bool):
@ -155,31 +95,7 @@ class MISPEncode(JSONEncoder):
return JSONEncoder.default(self, obj) return JSONEncoder.default(self, obj)
if HAS_RAPIDJSON: class AbstractMISP(MutableMapping, MISPFileCache, metaclass=ABCMeta):
def pymisp_json_default(obj):
if isinstance(obj, AbstractMISP):
return obj.jsonable()
elif isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
elif isinstance(obj, Enum):
return obj.value
elif isinstance(obj, UUID):
return str(obj)
return rapidjson.default(obj)
else:
def pymisp_json_default(obj):
if isinstance(obj, AbstractMISP):
return obj.jsonable()
elif isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
elif isinstance(obj, Enum):
return obj.value
elif isinstance(obj, UUID):
return str(obj)
return json.default(obj)
class AbstractMISP(MutableMapping, MISPFileCache):
__resources_path = resources_path __resources_path = resources_path
__misp_objects_path = misp_objects_path __misp_objects_path = misp_objects_path
__describe_types = describe_types __describe_types = describe_types
@ -191,16 +107,17 @@ class AbstractMISP(MutableMapping, MISPFileCache):
To do so, you need to call the respective add_* or update_* To do so, you need to call the respective add_* or update_*
methods in ExpandedPyMISP/PyMISP. methods in ExpandedPyMISP/PyMISP.
""" """
super(AbstractMISP, self).__init__() super().__init__()
self.__edited = True # As we create a new object, we assume it is edited self.__edited: bool = True # As we create a new object, we assume it is edited
self.__not_jsonable = [] self.__not_jsonable: list = []
self.__self_defined_describe_types = None self._fields_for_feed: set = {}
self.__self_defined_describe_types: Union[dict, None] = None
if kwargs.get('force_timestamps') is not None: if kwargs.get('force_timestamps') is not None:
# Ignore the edited objects and keep the timestamps. # Ignore the edited objects and keep the timestamps.
self.__force_timestamps = True self.__force_timestamps: bool = True
else: else:
self.__force_timestamps = False self.__force_timestamps: bool = False
# List of classes having tags # List of classes having tags
from .mispevent import MISPAttribute, MISPEvent from .mispevent import MISPAttribute, MISPEvent
@ -211,30 +128,30 @@ class AbstractMISP(MutableMapping, MISPFileCache):
setattr(AbstractMISP, 'tags', property(AbstractMISP.__get_tags, AbstractMISP.__set_tags)) setattr(AbstractMISP, 'tags', property(AbstractMISP.__get_tags, AbstractMISP.__set_tags))
@property @property
def describe_types(self): def describe_types(self) -> dict:
if self.__self_defined_describe_types: if self.__self_defined_describe_types:
return self.__self_defined_describe_types return self.__self_defined_describe_types
return self.__describe_types return self.__describe_types
@describe_types.setter @describe_types.setter
def describe_types(self, describe_types): def describe_types(self, describe_types: dict):
self.__self_defined_describe_types = describe_types self.__self_defined_describe_types = describe_types
@property @property
def resources_path(self): def resources_path(self) -> Path:
return self.__resources_path return self.__resources_path
@property @property
def misp_objects_path(self): def misp_objects_path(self) -> Path:
return self.__misp_objects_path return self.__misp_objects_path
@misp_objects_path.setter @misp_objects_path.setter
def misp_objects_path(self, misp_objects_path): def misp_objects_path(self, misp_objects_path: Union[str, Path]):
if sys.version_info >= (3, 0) and isinstance(misp_objects_path, str): if isinstance(misp_objects_path, str):
misp_objects_path = Path(misp_objects_path) misp_objects_path = Path(misp_objects_path)
self.__misp_objects_path = misp_objects_path self.__misp_objects_path = misp_objects_path
def from_dict(self, **kwargs): def from_dict(self, **kwargs) -> None:
"""Loading all the parameters as class properties, if they aren't `None`. """Loading all the parameters as class properties, if they aren't `None`.
This method aims to be called when all the properties requiring a special This method aims to be called when all the properties requiring a special
treatment are processed. treatment are processed.
@ -247,19 +164,19 @@ class AbstractMISP(MutableMapping, MISPFileCache):
# We load an existing dictionary, marking it an not-edited # We load an existing dictionary, marking it an not-edited
self.__edited = False self.__edited = False
def update_not_jsonable(self, *args): def update_not_jsonable(self, *args) -> None:
"""Add entries to the __not_jsonable list""" """Add entries to the __not_jsonable list"""
self.__not_jsonable += args self.__not_jsonable += args
def set_not_jsonable(self, *args): def set_not_jsonable(self, args: list) -> None:
"""Set __not_jsonable to a new list""" """Set __not_jsonable to a new list"""
self.__not_jsonable = args self.__not_jsonable = args
def from_json(self, json_string): def from_json(self, json_string: str) -> None:
"""Load a JSON string""" """Load a JSON string"""
self.from_dict(**loads(json_string)) self.from_dict(**loads(json_string))
def to_dict(self): def to_dict(self) -> dict:
"""Dump the class to a dictionary. """Dump the class to a dictionary.
This method automatically removes the timestamp recursively in every object This method automatically removes the timestamp recursively in every object
that has been edited is order to let MISP update the event accordingly.""" that has been edited is order to let MISP update the event accordingly."""
@ -283,15 +200,15 @@ class AbstractMISP(MutableMapping, MISPFileCache):
to_return = _int_to_str(to_return) to_return = _int_to_str(to_return)
return to_return return to_return
def jsonable(self): def jsonable(self) -> dict:
"""This method is used by the JSON encoder""" """This method is used by the JSON encoder"""
return self.to_dict() return self.to_dict()
def _to_feed(self): def _to_feed(self) -> dict:
if not hasattr(self, '_fields_for_feed'): if not hasattr(self, '_fields_for_feed') or not self._fields_for_feed:
raise PyMISPError('Unable to export in the feed format, _fields_for_feed is missing.') raise PyMISPError('Unable to export in the feed format, _fields_for_feed is missing.')
if hasattr(self, '_set_default') and callable(self._set_default): if hasattr(self, '_set_default') and callable(self._set_default): # type: ignore
self._set_default() self._set_default() # type: ignore
to_return = {} to_return = {}
for field in self._fields_for_feed: for field in self._fields_for_feed:
if getattr(self, field, None) is not None: if getattr(self, field, None) is not None:
@ -308,7 +225,7 @@ class AbstractMISP(MutableMapping, MISPFileCache):
raise PyMISPError('The field {} is required in {} when generating a feed.'.format(field, self.__class__.__name__)) raise PyMISPError('The field {} is required in {} when generating a feed.'.format(field, self.__class__.__name__))
return to_return return to_return
def to_json(self, sort_keys=False, indent=None): def to_json(self, sort_keys: bool=False, indent: Optional[int]=None):
"""Dump recursively any class of type MISPAbstract to a json string""" """Dump recursively any class of type MISPAbstract to a json string"""
return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent) return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
@ -334,7 +251,7 @@ class AbstractMISP(MutableMapping, MISPFileCache):
return len([k for k in self.__dict__.keys() if not (k[0] == '_' or k in self.__not_jsonable)]) return len([k for k in self.__dict__.keys() if not (k[0] == '_' or k in self.__not_jsonable)])
@property @property
def edited(self): def edited(self) -> bool:
"""Recursively check if an object has been edited and update the flag accordingly """Recursively check if an object has been edited and update the flag accordingly
to the parent objects""" to the parent objects"""
if self.__edited: if self.__edited:
@ -362,17 +279,14 @@ class AbstractMISP(MutableMapping, MISPFileCache):
# The private members don't matter # The private members don't matter
# If we already have a key with that name, we're modifying it. # If we already have a key with that name, we're modifying it.
self.__edited = True self.__edited = True
super(AbstractMISP, self).__setattr__(name, value) super().__setattr__(name, value)
def _datetime_to_timestamp(self, d): def _datetime_to_timestamp(self, d: Union[int, float, str, datetime.datetime]) -> int:
"""Convert a datetime.datetime object to a timestamp (int)""" """Convert a datetime.datetime object to a timestamp (int)"""
if isinstance(d, (int, float, str)) or (sys.version_info < (3, 0) and isinstance(d, unicode)): if isinstance(d, (int, float, str)):
# Assume we already have a timestamp # Assume we already have a timestamp
return int(d) return int(d)
if sys.version_info >= (3, 3): return int(d.timestamp())
return int(d.timestamp())
else:
return int((d - datetime.datetime.fromtimestamp(0, UTC())).total_seconds())
def __add_tag(self, tag=None, **kwargs): def __add_tag(self, tag=None, **kwargs):
"""Add a tag to the attribute (by name or a MISPTag object)""" """Add a tag to the attribute (by name or a MISPTag object)"""
@ -388,10 +302,11 @@ class AbstractMISP(MutableMapping, MISPFileCache):
misp_tag = MISPTag() misp_tag = MISPTag()
misp_tag.from_dict(**kwargs) misp_tag.from_dict(**kwargs)
else: else:
raise PyMISPInvalidFormat("The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {}".format(tag)) raise PyMISPInvalidFormat(f"The tag is in an invalid format (can be either string, MISPTag, or an expanded dict): {tag}")
if misp_tag not in self.tags: if misp_tag not in self.tags:
self.Tag.append(misp_tag) self.Tag.append(misp_tag)
self.edited = True self.edited = True
return misp_tag
def __get_tags(self): def __get_tags(self):
"""Returns a lost of tags associated to this Attribute""" """Returns a lost of tags associated to this Attribute"""
@ -420,15 +335,12 @@ class AbstractMISP(MutableMapping, MISPFileCache):
class MISPTag(AbstractMISP): class MISPTag(AbstractMISP):
_fields_for_feed = {'name', 'colour'} _fields_for_feed: set = {'name', 'colour'}
def __init__(self):
super(MISPTag, self).__init__()
def from_dict(self, **kwargs): def from_dict(self, **kwargs):
if kwargs.get('Tag'): if kwargs.get('Tag'):
kwargs = kwargs.get('Tag') kwargs = kwargs.get('Tag')
super(MISPTag, self).from_dict(**kwargs) super().from_dict(**kwargs)
def _set_default(self): def _set_default(self):
if not hasattr(self, 'colour'): if not hasattr(self, 'colour'):
@ -437,4 +349,26 @@ class MISPTag(AbstractMISP):
def _to_feed(self): def _to_feed(self):
if hasattr(self, 'exportable') and not self.exportable: if hasattr(self, 'exportable') and not self.exportable:
return False return False
return super(MISPTag, self)._to_feed() return super()._to_feed()
if HAS_RAPIDJSON:
def pymisp_json_default(obj: Union[AbstractMISP, datetime.datetime, datetime.date, Enum, UUID]) -> Union[dict, str]:
if isinstance(obj, AbstractMISP):
return obj.jsonable()
elif isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
elif isinstance(obj, Enum):
return obj.value
elif isinstance(obj, UUID):
return str(obj)
else:
def pymisp_json_default(obj: Union[AbstractMISP, datetime.datetime, datetime.date, Enum, UUID]) -> Union[dict, str]:
if isinstance(obj, AbstractMISP):
return obj.jsonable()
elif isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
elif isinstance(obj, Enum):
return obj.value
elif isinstance(obj, UUID):
return str(obj)

4346
pymisp/api.py
View File

File diff suppressed because it is too large Load Diff

2243
pymisp/aping.py
View File

File diff suppressed because it is too large Load Diff

2
pymisp/data/misp-objects

@ -1 +1 @@
Subproject commit ce80fb6384d6a369d4327db045255bd35bc25dbb Subproject commit fa634803911d211f993049242d41eebaf342a9c4

1071
pymisp/mispevent.py
View File

File diff suppressed because it is too large Load Diff

23
pymisp/tools/__init__.py
View File

@ -1,5 +1,3 @@
import sys
from .vtreportobject import VTReportObject # noqa from .vtreportobject import VTReportObject # noqa
from .neo4j import Neo4j # noqa from .neo4j import Neo4j # noqa
from .fileobject import FileObject # noqa from .fileobject import FileObject # noqa
@ -16,14 +14,13 @@ from .domainipobject import DomainIPObject # noqa
from .asnobject import ASNObject # noqa from .asnobject import ASNObject # noqa
from .geolocationobject import GeolocationObject # noqa from .geolocationobject import GeolocationObject # noqa
if sys.version_info >= (3, 6): from .emailobject import EMailObject # noqa
from .emailobject import EMailObject # noqa from .vehicleobject import VehicleObject # noqa
from .vehicleobject import VehicleObject # noqa from .csvloader import CSVLoader # noqa
from .csvloader import CSVLoader # noqa from .sshauthkeyobject import SSHAuthorizedKeysObject # noqa
from .sshauthkeyobject import SSHAuthorizedKeysObject # noqa from .feed import feed_meta_generator # noqa
from .feed import feed_meta_generator # noqa try:
try: from .urlobject import URLObject # noqa
from .urlobject import URLObject # noqa except ImportError:
except ImportError: # Requires faup, which is a bit difficult to install
# Requires faup, which is a bit difficult to install pass
pass

9
setup.py
View File

@ -34,15 +34,12 @@ setup(
'Intended Audience :: Science/Research', 'Intended Audience :: Science/Research',
'Intended Audience :: Telecommunications Industry', 'Intended Audience :: Telecommunications Industry',
'Intended Audience :: Information Technology', 'Intended Audience :: Information Technology',
'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3',
'Topic :: Security', 'Topic :: Security',
'Topic :: Internet', 'Topic :: Internet',
], ],
install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'deprecated'],
'python-dateutil', 'enum34;python_version<"3.4"', extras_require={'fileobjects': ['lief>=0.10.1', 'python-magic', 'pydeep'],
'functools32;python_version<"3.0"', 'deprecated', 'cachetools;python_version<"3.0"'],
extras_require={'fileobjects': ['lief>=0.8,<0.10;python_version<"3.5"', 'lief>=0.10.1;python_version>"3.5"', 'python-magic', 'pydeep'],
'neo': ['py2neo'], 'neo': ['py2neo'],
'openioc': ['beautifulsoup4'], 'openioc': ['beautifulsoup4'],
'virustotal': ['validators'], 'virustotal': ['validators'],

2
tests/mispevent_testfiles/event_obj_attr_tag.json
View File

@ -30,7 +30,7 @@
"name": "file", "name": "file",
"sharing_group_id": "0", "sharing_group_id": "0",
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
"template_version": "18", "template_version": "19",
"uuid": "a" "uuid": "a"
}, },
{ {

4
tests/mispevent_testfiles/event_obj_def_param.json
View File

@ -30,7 +30,7 @@
"name": "file", "name": "file",
"sharing_group_id": "0", "sharing_group_id": "0",
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
"template_version": "18", "template_version": "19",
"uuid": "a" "uuid": "a"
}, },
{ {
@ -55,7 +55,7 @@
"name": "file", "name": "file",
"sharing_group_id": "0", "sharing_group_id": "0",
"template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215",
"template_version": "18", "template_version": "19",
"uuid": "b" "uuid": "b"
} }
] ]

314
tests/test.py
View File

@ -1,314 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import PyMISP, __version__
try:
from keys import url, key
except ImportError as e:
print(e)
url = 'https://localhost:8443'
key = 'd6OmdDFvU3Seau3UjwvHS1y3tFQbaRNhJhDX0tjh'
import time
import unittest
class TestBasic(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, False, 'json')
self.live_describe_types = self.misp.get_live_describe_types()
def _clean_event(self, event):
event['Event'].pop('orgc_id', None)
event['Event'].pop('uuid', None)
event['Event'].pop('sharing_group_id', None)
event['Event'].pop('timestamp', None)
event['Event'].pop('org_id', None)
event['Event'].pop('date', None)
event['Event'].pop('RelatedEvent', None)
event['Event'].pop('publish_timestamp', None)
if event['Event'].get('Attribute'):
for a in event['Event'].get('Attribute'):
a.pop('uuid', None)
a.pop('event_id', None)
a.pop('id', None)
a.pop('timestamp', None)
if event['Event'].get('Orgc'):
event['Event']['Orgc'].pop('uuid', None)
event['Event']['Orgc'].pop('id', None)
if event['Event'].get('Org'):
event['Event']['Org'].pop('uuid', None)
event['Event']['Org'].pop('id', None)
return event['Event'].pop('id', None)
def new_event(self):
event = self.misp.new_event(0, 1, 0, "This is a test")
event_id = self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'0', 'disable_correlation': False, u'analysis': u'0',
u'ShadowAttribute': [], u'published': False,
u'distribution': u'0', u'event_creator_email': u'admin@admin.test', u'Attribute': [], u'proposal_email_lock': False,
u'extends_uuid': '',
u'Object': [], u'Org': {'local': True, u'name': u'ORGNAME'},
u'Orgc': {'local': True, u'name': u'ORGNAME'},
u'Galaxy': [],
u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at creating a new Event')
return int(event_id)
def add_hashes(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.add_hashes(event,
category='Payload installation',
filename='dll_installer.dll',
md5='0a209ac0de4ac033f31d6ba9191a8f7a',
sha1='1f0ae54ac3f10d533013f74f48849de4e65817a7',
sha256='003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
ssdeep=None,
comment='Fanny modules',
to_ids=False,
distribution=2,
proposal=False)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'admin@admin.test',
u'Org': {'local': True, u'name': u'ORGNAME'},
u'Orgc': {'local': True, u'name': u'ORGNAME'},
u'Galaxy': [],
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at adding hashes')
def publish(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.publish(event)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'event_creator_email': u'admin@admin.test',
u'Org': {'local': True, u'name': u'ORGNAME'},
u'Orgc': {'local': True, u'name': u'ORGNAME'},
u'Galaxy': [],
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at publishing event')
def delete(self, eventid):
event = self.misp.delete_event(eventid)
print(event)
def delete_attr(self, attrid):
event = self.misp.delete_attribute(attrid)
print(event)
def get(self, eventid):
event = self.misp.get_event(eventid)
print(event)
def get_stix(self, **kwargs):
event = self.misp.get_stix(kwargs)
print(event)
def add(self):
event = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'admin@admin.test',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
event = self.misp.add_event(event)
print(event)
def add_user(self):
email = 'test@misp.local'
role_id = '5'
org_id = '1'
password = 'Password1234!'
external_auth_required = False
external_auth_key = ''
enable_password = False
nids_sid = '1238717'
server_id = '1'
gpgkey = ''
certif_public = ''
autoalert = False
contactalert = False
disabled = False
change_pw = '0'
termsaccepted = False
newsread = '0'
authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
to_check = {'User': {'email': email, 'org_id': org_id, 'role_id': role_id,
'password': password, 'external_auth_required': external_auth_required,
'external_auth_key': external_auth_key, 'enable_password': enable_password,
'nids_sid': nids_sid, 'server_id': server_id, 'gpgkey': gpgkey,
'certif_public': certif_public, 'autoalert': autoalert,
'contactalert': contactalert, 'disabled': disabled,
'change_pw': change_pw, 'termsaccepted': termsaccepted,
'newsread': newsread, 'authkey': authkey}}
user = self.misp.add_user(email=email,
role_id=role_id,
org_id=org_id,
password=password,
external_auth_required=external_auth_required,
external_auth_key=external_auth_key,
enable_password=enable_password,
nids_sid=nids_sid,
server_id=server_id,
gpgkey=gpgkey,
certif_public=certif_public,
autoalert=autoalert,
contactalert=contactalert,
disabled=disabled,
change_pw=change_pw,
termsaccepted=termsaccepted,
newsread=newsread,
authkey=authkey)
# delete user to allow reuse of test
uid = user.get('User').get('id')
self.misp.delete_user(uid)
# ----------------------------------
# test interesting keys only (some keys are modified(password) and some keys are added (lastlogin)
tested_keys = ['email', 'org_id', 'role_id', 'server_id', 'autoalert',
'authkey', 'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted',
'newsread', 'contactalert', 'disabled']
for k in tested_keys:
self.assertEqual(user.get('User').get(k), to_check.get('User').get(k), "Failed to match input with output on key: {}".format(k))
def add_organisation(self):
name = 'Organisation tests'
description = 'This is a test organisation'
orgtype = 'Type is a string'
nationality = 'French'
sector = 'Bank sector'
uuid = '16fd2706-8baf-433b-82eb-8c7fada847da'
contacts = 'Text field with no limitations'
local = False
to_check = {'Organisation': {'name': name, 'description': description,
'type': orgtype, 'nationality': nationality,
'sector': sector, 'uuid': uuid, 'contacts': contacts,
'local': local}}
org = self.misp.add_organisation(name=name,
description=description,
type=orgtype,
nationality=nationality,
sector=sector,
uuid=uuid,
contacts=contacts,
local=local,
)
# delete organisation to allow reuse of test
oid = org.get('Organisation').get('id')
self.misp.delete_organisation(oid)
# ----------------------------------
tested_keys = ['anonymise', 'contacts', 'description', 'local', 'name',
'nationality', 'sector', 'type', 'uuid']
for k in tested_keys:
self.assertEqual(org.get('Organisation').get(k), to_check.get('Organisation').get(k), "Failed to match input with output on key: {}".format(k))
def test_create_event(self):
eventid = self.new_event()
time.sleep(1)
self.delete(eventid)
def test_get_event(self):
eventid = self.new_event()
time.sleep(1)
self.get(eventid)
time.sleep(1)
self.delete(eventid)
def test_add_event(self):
self.add()
time.sleep(1)
self.delete(1)
def test_del_attr(self):
eventid = self.new_event()
time.sleep(1)
self.delete_attr(1)
time.sleep(1)
self.delete(eventid)
def test_one_or_more(self):
self.assertEqual(self.misp._one_or_more(1), (1,))
self.assertEqual(self.misp._one_or_more([1]), [1])
def test_create_user(self):
self.add_user()
def test_create_organisation(self):
self.add_organisation()
def test_describeTypes_sane_default(self):
sane_default = self.live_describe_types['sane_defaults']
self.assertEqual(sorted(sane_default.keys()), sorted(self.live_describe_types['types']))
def test_describeTypes_categories(self):
category_type_mappings = self.live_describe_types['category_type_mappings']
self.assertEqual(sorted(category_type_mappings.keys()), sorted(self.live_describe_types['categories']))
def test_describeTypes_types_in_categories(self):
category_type_mappings = self.live_describe_types['category_type_mappings']
for category, types in category_type_mappings.items():
existing_types = [t for t in types if t in self.live_describe_types['types']]
self.assertEqual(sorted(existing_types), sorted(types))
def test_describeTypes_types_have_category(self):
category_type_mappings = self.live_describe_types['category_type_mappings']
all_types = set()
for category, types in category_type_mappings.items():
all_types.update(types)
self.assertEqual(sorted(list(all_types)), sorted(self.live_describe_types['types']))
def test_describeTypes_sane_default_valid_category(self):
sane_default = self.live_describe_types['sane_defaults']
categories = self.live_describe_types['categories']
for t, sd in sane_default.items():
self.assertTrue(sd['to_ids'] in [0, 1])
self.assertTrue(sd['default_category'] in categories)
def test_live_acl(self):
query_acl = self.misp.get_live_query_acl()
self.assertEqual(query_acl['response'], [])
def test_recommended_pymisp_version(self):
response = self.misp.get_recommended_api_version()
recommended_version_tup = tuple(int(x) for x in response['version'].split('.'))
pymisp_version_tup = tuple(int(x) for x in __version__.split('.'))[:3]
self.assertEqual(recommended_version_tup, pymisp_version_tup)
if __name__ == '__main__':
unittest.main()

6
tests/testlive_comprehensive.py
View File

@ -9,7 +9,6 @@ import unittest
from pymisp.tools import make_binary_objects from pymisp.tools import make_binary_objects
from datetime import datetime, timedelta, date from datetime import datetime, timedelta, date
from io import BytesIO from io import BytesIO
import re
import json import json
from pathlib import Path from pathlib import Path
@ -993,10 +992,9 @@ class TestComprehensive(unittest.TestCase):
try: try:
first = self.user_misp_connector.add_event(first) first = self.user_misp_connector.add_event(first)
stix = self.user_misp_connector.search(return_format='stix', eventid=first.id) stix = self.user_misp_connector.search(return_format='stix', eventid=first.id)
found = re.findall('8.8.8.8', stix) self.assertTrue(stix['related_packages'][0]['package']['incidents'][0]['related_indicators']['indicators'][0]['indicator']['observable']['object']['properties']['address_value']['value'], '8.8.8.8')
self.assertTrue(found)
stix2 = self.user_misp_connector.search(return_format='stix2', eventid=first.id) stix2 = self.user_misp_connector.search(return_format='stix2', eventid=first.id)
json.dumps(stix2, indent=2) print(json.dumps(stix2, indent=2))
self.assertEqual(stix2['objects'][-1]['pattern'], "[network-traffic:src_ref.type = 'ipv4-addr' AND network-traffic:src_ref.value = '8.8.8.8']") self.assertEqual(stix2['objects'][-1]['pattern'], "[network-traffic:src_ref.type = 'ipv4-addr' AND network-traffic:src_ref.value = '8.8.8.8']")
finally: finally:
# Delete event # Delete event

11
travis/install_travis.sh
View File

@ -3,11 +3,6 @@
set -e set -e
set -x set -x
if [ ${LEGACY} == true ]; then # We're in python3, installing with pipenv.
pip install nose coveralls codecov requests-mock pydeep pip3 install pipenv
pip install .[fileobjects] pipenv update --dev
else
# We're in python3, installing with pipenv.
pip install pipenv
pipenv update --dev
fi

7
travis/test_travis.sh
View File

@ -3,9 +3,4 @@
set -e set -e
set -x set -x
if [ -z ${LEGACY} ]; then pipenv run nosetests-3.4 --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py
# We're in python3, test all and use pipenv.
pipenv run nosetests-3.4 --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py
else
nosetests --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_mispevent.py
fi