diff --git a/.travis.yml b/.travis.yml
index b574d4c..18c02c6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,7 @@ python:
- "3.7-dev"
install:
+ - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev
- pip install pipenv
- pipenv install --dev
@@ -19,14 +20,14 @@ script:
- pid=$!
- sleep 5
- pipenv run nosetests --with-coverage --cover-package=misp_modules
- - kill -s INT $pid
+ - kill -s KILL $pid
- pushd ~/
- pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
- pid=$!
- popd
- sleep 5
- pipenv run nosetests --with-coverage --cover-package=misp_modules
- - kill -s INT $pid
+ - kill -s KILL $pid
- pipenv run flake8 --ignore=E501,W503 misp_modules
after_success:
diff --git a/Pipfile b/Pipfile
index 45c05f5..36f059e 100644
--- a/Pipfile
+++ b/Pipfile
@@ -41,6 +41,20 @@ domaintools_api = "*"
misp-modules = {editable = true,path = "."}
pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"}
pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"}
+backscatter = "*"
+pyzbar = "*"
+opencv-python = "*"
+np = "*"
+ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git/"}
+python-pptx = "*"
+python-docx = "*"
+ezodf = "*"
+pandas = "*"
+pandas_ods_reader = "*"
+pdftotext = "*"
+lxml = "*"
+xlrd = "*"
+idna-ssl = {markers="python_version < '3.7'"}
[requires]
-python_version = "3.6"
+python_version = "3"
diff --git a/Pipfile.lock b/Pipfile.lock
index 85cd9db..fa07b4c 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,11 +1,11 @@
{
"_meta": {
"hash": {
- "sha256": "d0cd64bfe7702365d3ea66d1f51a1ec8592df2490899e7e163fe38f97172561e"
+ "sha256": "9aac0a9c45df16b9502c13f9468095cf5ffdb8bc407fe2b55faee3ff53d8eba3"
},
"pipfile-spec": 6,
"requires": {
- "python_version": "3.6"
+ "python_version": "3"
},
"sources": [
{
@@ -59,10 +59,18 @@
},
"attrs": {
"hashes": [
- "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
- "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
+ "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
+ "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
],
- "version": "==18.2.0"
+ "version": "==19.1.0"
+ },
+ "backscatter": {
+ "hashes": [
+ "sha256:7a0d1aa3661635de81e2a09b15d53e35cbe399a111cc58a70925f80e6874abd3",
+ "sha256:afb0efcf5d2551dac953ec4c38fb710b274b8e811775650e02c1ef42cafb14c8"
+ ],
+ "index": "pypi",
+ "version": "==0.2.4"
},
"beautifulsoup4": {
"hashes": [
@@ -82,10 +90,10 @@
},
"certifi": {
"hashes": [
- "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
- "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
+ "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
+ "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
],
- "version": "==2018.11.29"
+ "version": "==2019.3.9"
},
"chardet": {
"hashes": [
@@ -103,10 +111,10 @@
},
"click-plugins": {
"hashes": [
- "sha256:b1ee1ccc9421c73007fe290680d97984eb6eaf5f4512b7620c6aa46031d6cb6b",
- "sha256:dfed74b5063546a137de99baaaf742b4de4337ad2b3e1df5ec7c8a256adc0847"
+ "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b",
+ "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"
],
- "version": "==1.0.4"
+ "version": "==1.1.1"
},
"colorama": {
"hashes": [
@@ -142,6 +150,13 @@
],
"version": "==0.9"
},
+ "ezodf": {
+ "hashes": [
+ "sha256:000da534f689c6d55297a08f9e2ed7eada9810d194d31d164388162fb391122d"
+ ],
+ "index": "pypi",
+ "version": "==0.3.2"
+ },
"future": {
"hashes": [
"sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8"
@@ -150,9 +165,10 @@
},
"httplib2": {
"hashes": [
- "sha256:4ba6b8fd77d0038769bf3c33c9a96a6f752bc4cdf739701fdcaf210121f399d4"
+ "sha256:23914b5487dfe8ef09db6656d6d63afb0cf3054ad9ebc50868ddc8e166b5f8e8",
+ "sha256:a18121c7c72a56689efbf1aef990139ad940fee1e64c6f2458831736cd593600"
],
- "version": "==0.12.1"
+ "version": "==0.12.3"
},
"idna": {
"hashes": [
@@ -165,6 +181,7 @@
"hashes": [
"sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c"
],
+ "index": "pypi",
"markers": "python_version < '3.7'",
"version": "==1.1.0"
},
@@ -177,10 +194,42 @@
},
"jsonschema": {
"hashes": [
- "sha256:acc8a90c31d11060516cfd0b414b9f8bcf4bc691b21f0f786ea57dd5255c79db",
- "sha256:dd3f8ecb1b52d94d45eedb67cb86cac57b94ded562c5d98f63719e55ce58557b"
+ "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d",
+ "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a"
],
- "version": "==3.0.0"
+ "version": "==3.0.1"
+ },
+ "lxml": {
+ "hashes": [
+ "sha256:03984196d00670b2ab14ae0ea83d5cc0cfa4f5a42558afa9ab5fa745995328f5",
+ "sha256:0815b0c9f897468de6a386dc15917a0becf48cc92425613aa8bbfc7f0f82951f",
+ "sha256:175f3825f075cf02d15099eb52658457cf0ff103dcf11512b5d2583e1d40f58b",
+ "sha256:30e14c62d88d1e01a26936ecd1c6e784d4afc9aa002bba4321c5897937112616",
+ "sha256:3210da6f36cf4b835ff1be853962b22cc354d506f493b67a4303c88bbb40d57b",
+ "sha256:40f60819fbd5bad6e191ba1329bfafa09ab7f3f174b3d034d413ef5266963294",
+ "sha256:43b26a865a61549919f8a42e094dfdb62847113cf776d84bd6b60e4e3fc20ea3",
+ "sha256:4a03dd682f8e35a10234904e0b9508d705ff98cf962c5851ed052e9340df3d90",
+ "sha256:62f382cddf3d2e52cf266e161aa522d54fd624b8cc567bc18f573d9d50d40e8e",
+ "sha256:7b98f0325be8450da70aa4a796c4f06852949fe031878b4aa1d6c417a412f314",
+ "sha256:846a0739e595871041385d86d12af4b6999f921359b38affb99cdd6b54219a8f",
+ "sha256:a3080470559938a09a5d0ec558c005282e99ac77bf8211fb7b9a5c66390acd8d",
+ "sha256:ad841b78a476623955da270ab8d207c3c694aa5eba71f4792f65926dc46c6ee8",
+ "sha256:afdd75d9735e44c639ffd6258ce04a2de3b208f148072c02478162d0944d9da3",
+ "sha256:b4fbf9b552faff54742bcd0791ab1da5863363fb19047e68f6592be1ac2dab33",
+ "sha256:b90c4e32d6ec089d3fa3518436bdf5ce4d902a0787dbd9bb09f37afe8b994317",
+ "sha256:b91cfe4438c741aeff662d413fd2808ac901cc6229c838236840d11de4586d63",
+ "sha256:bdb0593a42070b0a5f138b79b872289ee73c8e25b3f0bea6564e795b55b6bcdd",
+ "sha256:c4e4bca2bb68ce22320297dfa1a7bf070a5b20bcbaec4ee023f83d2f6e76496f",
+ "sha256:cec4ab14af9eae8501be3266ff50c3c2aecc017ba1e86c160209bb4f0423df6a",
+ "sha256:e83b4b2bf029f5104bc1227dbb7bf5ace6fd8fabaebffcd4f8106fafc69fc45f",
+ "sha256:e995b3734a46d41ae60b6097f7c51ba9958648c6d1e0935b7e0ee446ee4abe22",
+ "sha256:f679d93dec7f7210575c85379a31322df4c46496f184ef650d3aba1484b38a2d",
+ "sha256:fd213bb5166e46974f113c8228daaef1732abc47cb561ce9c4c8eaed4bd3b09b",
+ "sha256:fdcb57b906dbc1f80666e6290e794ab8fb959a2e17aa5aee1758a85d1da4533f",
+ "sha256:ff424b01d090ffe1947ec7432b07f536912e0300458f9a7f48ea217dd8362b86"
+ ],
+ "index": "pypi",
+ "version": "==4.3.3"
},
"maclookup": {
"hashes": [
@@ -228,6 +277,41 @@
],
"version": "==4.5.2"
},
+ "np": {
+ "hashes": [
+ "sha256:781265283f3823663ad8fb48741aae62abcf4c78bc19f908f8aa7c1d3eb132f8"
+ ],
+ "index": "pypi",
+ "version": "==1.0.2"
+ },
+ "numpy": {
+ "hashes": [
+ "sha256:0e2eed77804b2a6a88741f8fcac02c5499bba3953ec9c71e8b217fad4912c56c",
+ "sha256:1c666f04553ef70fda54adf097dbae7080645435fc273e2397f26bbf1d127bbb",
+ "sha256:1f46532afa7b2903bfb1b79becca2954c0a04389d19e03dc73f06b039048ac40",
+ "sha256:315fa1b1dfc16ae0f03f8fd1c55f23fd15368710f641d570236f3d78af55e340",
+ "sha256:3d5fcea4f5ed40c3280791d54da3ad2ecf896f4c87c877b113576b8280c59441",
+ "sha256:48241759b99d60aba63b0e590332c600fc4b46ad597c9b0a53f350b871ef0634",
+ "sha256:4b4f2924b36d857cf302aec369caac61e43500c17eeef0d7baacad1084c0ee84",
+ "sha256:54fe3b7ed9e7eb928bbc4318f954d133851865f062fa4bbb02ef8940bc67b5d2",
+ "sha256:5a8f021c70e6206c317974c93eaaf9bc2b56295b6b1cacccf88846e44a1f33fc",
+ "sha256:754a6be26d938e6ca91942804eb209307b73f806a1721176278a6038869a1686",
+ "sha256:771147e654e8b95eea1293174a94f34e2e77d5729ad44aefb62fbf8a79747a15",
+ "sha256:78a6f89da87eeb48014ec652a65c4ffde370c036d780a995edaeb121d3625621",
+ "sha256:7fde5c2a3a682a9e101e61d97696687ebdba47637611378b4127fe7e47fdf2bf",
+ "sha256:80d99399c97f646e873dd8ce87c38cfdbb668956bbc39bc1e6cac4b515bba2a0",
+ "sha256:88a72c1e45a0ae24d1f249a529d9f71fe82e6fa6a3fd61414b829396ec585900",
+ "sha256:a4f4460877a16ac73302a9c077ca545498d9fe64e6a81398d8e1a67e4695e3df",
+ "sha256:a61255a765b3ac73ee4b110b28fccfbf758c985677f526c2b4b39c48cc4b509d",
+ "sha256:ab4896a8c910b9a04c0142871d8800c76c8a2e5ff44763513e1dd9d9631ce897",
+ "sha256:abbd6b1c2ef6199f4b7ca9f818eb6b31f17b73a6110aadc4e4298c3f00fab24e",
+ "sha256:b16d88da290334e33ea992c56492326ea3b06233a00a1855414360b77ca72f26",
+ "sha256:b78a1defedb0e8f6ae1eb55fa6ac74ab42acc4569c3a2eacc2a407ee5d42ebcb",
+ "sha256:cfef82c43b8b29ca436560d51b2251d5117818a8d1fb74a8384a83c096745dad",
+ "sha256:d160e57731fcdec2beda807ebcabf39823c47e9409485b5a3a1db3a8c6ce763e"
+ ],
+ "version": "==1.16.3"
+ },
"oauth2": {
"hashes": [
"sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6",
@@ -236,6 +320,78 @@
"index": "pypi",
"version": "==1.9.0.post1"
},
+ "odtreader": {
+ "editable": true,
+ "git": "https://github.com/cartertemm/ODTReader.git/",
+ "ref": "49d6938693f6faa3ff09998f86dba551ae3a996b"
+ },
+ "opencv-python": {
+ "hashes": [
+ "sha256:1703a296a96d3d46615e5053f224867977accb4240bcaa0fcabcb0768bf5ac13",
+ "sha256:1777ce7535ee7a1995cae168a107a1320e9df13648b930e72a1a2c2eccd64cda",
+ "sha256:1e5520482fb18fbd64d079e7f17ac0018f195fd75f6360a53bb82d7903106b50",
+ "sha256:25522dcf2529614750a71112a6659759080b4bdc2323f19d47f4d895960fd796",
+ "sha256:2af5f2842ad44c65ae2647377e0ff198719e1a1cfc9c6a19bc0c525c035d4bd8",
+ "sha256:31ec48d7eca13fc25c287dea7cecab453976e372cad8f50d55c054a247efda21",
+ "sha256:47cf48ff5dbd554e9f58cc9e98cf0b5de3f6a971172612bffa06bc5fb79ce872",
+ "sha256:494f98366bb5d6c2ac7e50e6617139f353704fd97a6d12ec9d392e72817d5cb0",
+ "sha256:4a9845870739e640e3350a8d98d511c92c087fe3d66090e83be7bf94e0ac64f7",
+ "sha256:4ac29cc0847d948a6636899014e84e165c30cc8779d6218394d44363462a01ce",
+ "sha256:5857ace03b7854221abf8072462d306c2c2ce4e366190b21d90ee8ee8aaf5bb4",
+ "sha256:5b4a23d99d5a2874767034466f5a8fd37b9f93ac14955a01b1a208983c76b9ad",
+ "sha256:734d87a5021c037064beb62133e135e66c7128e401a63b8b842b809ae2093749",
+ "sha256:78005c1c5d15ef4e32e0f485557bd15b5b6d87f49c19db7fe3e9246a61ebe7e4",
+ "sha256:81ae2283225c5c52fc3d72debd4241c30ccff2bb922578bf7867f9851cce3acb",
+ "sha256:88dbf900f297fdae0f62b899d6a784d8868ec2135854c5f8a9abbad00a6f0c5b",
+ "sha256:8c98ea7b8d327a31cd6028782a06147d0e0329ae8e829e881fb5d02f7ed8aec9",
+ "sha256:937d4686fef6967921145290f5b50c01c00c5b5d3542a6519e8a85cd88448723",
+ "sha256:a057958c0e362b3c4f03b9af1cbdb6d5af035fd22ecd7fd794eba8fdeb049eb8",
+ "sha256:c41eab31fa2c641226c6187caa391a688d064c99f078d604574f1912296b771f",
+ "sha256:cf4f7e62d1f80d1fa85a1693a3500def5cde54b2b75212b3609e552e4c25acfb",
+ "sha256:d90d60143e18334330c149f293071c9f2f3c79c896f33dc4ec65099e58baaaa7",
+ "sha256:db3106b7ca86999a7bd1f2fcc93e49314e5e6e451356774e421a69428df5020b",
+ "sha256:dbaf264db56f4771dfac6624f438bc4dc670aa94f61a6138848fcab7e9e77380",
+ "sha256:e65206c4cf651dc9cf0829962fae8bec986767c9f123d6a1ad17f9356bf7257e",
+ "sha256:eac94ddc78c58e891cff7180274317dad2938a4ddfc6ced1c04846c7f50e77e9",
+ "sha256:f2e828711f044a965509c862b3a59b3181e9c56c145a950cb53d43fec54e66d2"
+ ],
+ "index": "pypi",
+ "version": "==4.1.0.25"
+ },
+ "pandas": {
+ "hashes": [
+ "sha256:071e42b89b57baa17031af8c6b6bbd2e9a5c68c595bc6bf9adabd7a9ed125d3b",
+ "sha256:17450e25ae69e2e6b303817bdf26b2cd57f69595d8550a77c308be0cd0fd58fa",
+ "sha256:17916d818592c9ec891cbef2e90f98cc85e0f1e89ed0924c9b5220dc3209c846",
+ "sha256:2538f099ab0e9f9c9d09bbcd94b47fd889bad06dc7ae96b1ed583f1dc1a7a822",
+ "sha256:366f30710172cb45a6b4f43b66c220653b1ea50303fbbd94e50571637ffb9167",
+ "sha256:42e5ad741a0d09232efbc7fc648226ed93306551772fc8aecc6dce9f0e676794",
+ "sha256:4e718e7f395ba5bfe8b6f6aaf2ff1c65a09bb77a36af6394621434e7cc813204",
+ "sha256:4f919f409c433577a501e023943e582c57355d50a724c589e78bc1d551a535a2",
+ "sha256:4fe0d7e6438212e839fc5010c78b822664f1a824c0d263fd858f44131d9166e2",
+ "sha256:5149a6db3e74f23dc3f5a216c2c9ae2e12920aa2d4a5b77e44e5b804a5f93248",
+ "sha256:627594338d6dd995cfc0bacd8e654cd9e1252d2a7c959449228df6740d737eb8",
+ "sha256:83c702615052f2a0a7fb1dd289726e29ec87a27272d775cb77affe749cca28f8",
+ "sha256:8c872f7fdf3018b7891e1e3e86c55b190e6c5cee70cab771e8f246c855001296",
+ "sha256:90f116086063934afd51e61a802a943826d2aac572b2f7d55caaac51c13db5b5",
+ "sha256:a3352bacac12e1fc646213b998bce586f965c9d431773d9e91db27c7c48a1f7d",
+ "sha256:bcdd06007cca02d51350f96debe51331dec429ac8f93930a43eb8fb5639e3eb5",
+ "sha256:c1bd07ebc15285535f61ddd8c0c75d0d6293e80e1ee6d9a8d73f3f36954342d0",
+ "sha256:c9a4b7c55115eb278c19aa14b34fcf5920c8fe7797a09b7b053ddd6195ea89b3",
+ "sha256:cc8fc0c7a8d5951dc738f1c1447f71c43734244453616f32b8aa0ef6013a5dfb",
+ "sha256:d7b460bc316064540ce0c41c1438c416a40746fd8a4fb2999668bf18f3c4acf1"
+ ],
+ "index": "pypi",
+ "version": "==0.24.2"
+ },
+ "pandas-ods-reader": {
+ "hashes": [
+ "sha256:0f7d510639c8957a06aa1227b9f84d1be47a437dfd306464ce803b91cf5eeec4",
+ "sha256:d85ef58fc3aeac1616028d22954b6ef2e8983ab9bae015e1e90ce3979d138553"
+ ],
+ "index": "pypi",
+ "version": "==0.0.6"
+ },
"passivetotal": {
"hashes": [
"sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d"
@@ -243,60 +399,63 @@
"index": "pypi",
"version": "==1.0.30"
},
- "pillow": {
+ "pdftotext": {
"hashes": [
- "sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e",
- "sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7",
- "sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a",
- "sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3",
- "sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1",
- "sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1",
- "sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7",
- "sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1",
- "sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3",
- "sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055",
- "sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf",
- "sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f",
- "sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f",
- "sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239",
- "sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe",
- "sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c",
- "sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697",
- "sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494",
- "sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356",
- "sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6",
- "sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000",
- "sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f",
- "sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c",
- "sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca",
- "sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8",
- "sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3",
- "sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad",
- "sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9",
- "sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc",
- "sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e"
+ "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb"
],
"index": "pypi",
- "version": "==5.4.1"
+ "version": "==2.1.1"
+ },
+ "pillow": {
+ "hashes": [
+ "sha256:15c056bfa284c30a7f265a41ac4cbbc93bdbfc0dfe0613b9cb8a8581b51a9e55",
+ "sha256:1a4e06ba4f74494ea0c58c24de2bb752818e9d504474ec95b0aa94f6b0a7e479",
+ "sha256:1c3c707c76be43c9e99cb7e3d5f1bee1c8e5be8b8a2a5eeee665efbf8ddde91a",
+ "sha256:1fd0b290203e3b0882d9605d807b03c0f47e3440f97824586c173eca0aadd99d",
+ "sha256:24114e4a6e1870c5a24b1da8f60d0ba77a0b4027907860188ea82bd3508c80eb",
+ "sha256:258d886a49b6b058cd7abb0ab4b2b85ce78669a857398e83e8b8e28b317b5abb",
+ "sha256:33c79b6dd6bc7f65079ab9ca5bebffb5f5d1141c689c9c6a7855776d1b09b7e8",
+ "sha256:367385fc797b2c31564c427430c7a8630db1a00bd040555dfc1d5c52e39fcd72",
+ "sha256:3c1884ff078fb8bf5f63d7d86921838b82ed4a7d0c027add773c2f38b3168754",
+ "sha256:44e5240e8f4f8861d748f2a58b3f04daadab5e22bfec896bf5434745f788f33f",
+ "sha256:46aa988e15f3ea72dddd81afe3839437b755fffddb5e173886f11460be909dce",
+ "sha256:74d90d499c9c736d52dd6d9b7221af5665b9c04f1767e35f5dd8694324bd4601",
+ "sha256:809c0a2ce9032cbcd7b5313f71af4bdc5c8c771cb86eb7559afd954cab82ebb5",
+ "sha256:85d1ef2cdafd5507c4221d201aaf62fc9276f8b0f71bd3933363e62a33abc734",
+ "sha256:8c3889c7681af77ecfa4431cd42a2885d093ecb811e81fbe5e203abc07e0995b",
+ "sha256:9218d81b9fca98d2c47d35d688a0cea0c42fd473159dfd5612dcb0483c63e40b",
+ "sha256:9aa4f3827992288edd37c9df345783a69ef58bd20cc02e64b36e44bcd157bbf1",
+ "sha256:9d80f44137a70b6f84c750d11019a3419f409c944526a95219bea0ac31f4dd91",
+ "sha256:b7ebd36128a2fe93991293f997e44be9286503c7530ace6a55b938b20be288d8",
+ "sha256:c4c78e2c71c257c136cdd43869fd3d5e34fc2162dc22e4a5406b0ebe86958239",
+ "sha256:c6a842537f887be1fe115d8abb5daa9bc8cc124e455ff995830cc785624a97af",
+ "sha256:cf0a2e040fdf5a6d95f4c286c6ef1df6b36c218b528c8a9158ec2452a804b9b8",
+ "sha256:cfd28aad6fc61f7a5d4ee556a997dc6e5555d9381d1390c00ecaf984d57e4232",
+ "sha256:dca5660e25932771460d4688ccbb515677caaf8595f3f3240ec16c117deff89a",
+ "sha256:de7aedc85918c2f887886442e50f52c1b93545606317956d65f342bd81cb4fc3",
+ "sha256:e6c0bbf8e277b74196e3140c35f9a1ae3eafd818f7f2d3a15819c49135d6c062"
+ ],
+ "index": "pypi",
+ "version": "==6.0.0"
},
"psutil": {
"hashes": [
- "sha256:5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a",
- "sha256:72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810",
- "sha256:77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c",
- "sha256:8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d",
- "sha256:a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191",
- "sha256:ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f",
- "sha256:c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a",
- "sha256:ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59",
- "sha256:ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd"
+ "sha256:206eb909aa8878101d0eca07f4b31889c748f34ed6820a12eb3168c7aa17478e",
+ "sha256:649f7ffc02114dced8fbd08afcd021af75f5f5b2311bc0e69e53e8f100fe296f",
+ "sha256:6ebf2b9c996bb8c7198b385bade468ac8068ad8b78c54a58ff288cd5f61992c7",
+ "sha256:753c5988edc07da00dafd6d3d279d41f98c62cd4d3a548c4d05741a023b0c2e7",
+ "sha256:76fb0956d6d50e68e3f22e7cc983acf4e243dc0fcc32fd693d398cb21c928802",
+ "sha256:828e1c3ca6756c54ac00f1427fdac8b12e21b8a068c3bb9b631a1734cada25ed",
+ "sha256:a4c62319ec6bf2b3570487dd72d471307ae5495ce3802c1be81b8a22e438b4bc",
+ "sha256:acba1df9da3983ec3c9c963adaaf530fcb4be0cd400a8294f1ecc2db56499ddd",
+ "sha256:ef342cb7d9b60e6100364f50c57fa3a77d02ff8665d5b956746ac01901247ac4"
],
- "version": "==5.5.1"
+ "version": "==5.6.2"
},
"pybgpranking": {
"editable": true,
"git": "https://github.com/D4-project/BGP-Ranking.git/",
- "ref": "37c97ae252ec4bf1d67733a49d4895c8cb009cf9",
+ "ref": "4e0741056bcc0077de1120b8724a31330b26033e",
"subdirectory": "client"
},
"pydnstrails": {
@@ -327,13 +486,13 @@
"pyipasnhistory": {
"editable": true,
"git": "https://github.com/D4-project/IPASN-History.git/",
- "ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59",
+ "ref": "c0c2bbf8d70811982dad065ea463a7e01593a38d",
"subdirectory": "client"
},
"pymisp": {
"editable": true,
"git": "https://github.com/MISP/PyMISP.git",
- "ref": "62e047f3c1972e21aa36a8882bebf4488cdc1f84"
+ "ref": "582dda0ce2a8ca8e1dd2cf3842e0491caca51c62"
},
"pyonyphe": {
"editable": true,
@@ -342,10 +501,10 @@
},
"pyparsing": {
"hashes": [
- "sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a",
- "sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3"
+ "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a",
+ "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03"
],
- "version": "==2.3.1"
+ "version": "==2.4.0"
},
"pypdns": {
"hashes": [
@@ -363,9 +522,9 @@
},
"pyrsistent": {
"hashes": [
- "sha256:3ca82748918eb65e2d89f222b702277099aca77e34843c5eb9d52451173970e2"
+ "sha256:5403d37f4d55ff4572b5b5676890589f367a9569529c6f728c11046c4ea4272b"
],
- "version": "==0.14.11"
+ "version": "==0.15.1"
},
"pytesseract": {
"hashes": [
@@ -381,21 +540,51 @@
],
"version": "==2.8.0"
},
+ "python-docx": {
+ "hashes": [
+ "sha256:bc76ecac6b2d00ce6442a69d03a6f35c71cd72293cd8405a7472dfe317920024"
+ ],
+ "index": "pypi",
+ "version": "==0.8.10"
+ },
+ "python-pptx": {
+ "hashes": [
+ "sha256:1f2d5d1d923d91f50a1f0ed794935e7d670993fdcb6c12c81cc83977c1f23e14"
+ ],
+ "index": "pypi",
+ "version": "==0.6.17"
+ },
+ "pytz": {
+ "hashes": [
+ "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda",
+ "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141"
+ ],
+ "version": "==2019.1"
+ },
"pyyaml": {
"hashes": [
- "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b",
- "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf",
- "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a",
- "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3",
- "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1",
- "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1",
- "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613",
- "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04",
- "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f",
- "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537",
- "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531"
+ "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c",
+ "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95",
+ "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2",
+ "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4",
+ "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad",
+ "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba",
+ "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1",
+ "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e",
+ "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673",
+ "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13",
+ "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19"
],
- "version": "==3.13"
+ "version": "==5.1"
+ },
+ "pyzbar": {
+ "hashes": [
+ "sha256:0e204b904e093e5e75aa85e0203bb0e02888105732a509b51f31cff400f34265",
+ "sha256:496249b546be70ec98c0ff0ad9151e73daaffff129266df86150a15dcd8dac4c",
+ "sha256:7d6c01d2c0a352fa994aa91b5540d1caeaeaac466656eb41468ca5df33be9f2e"
+ ],
+ "index": "pypi",
+ "version": "==0.1.8"
},
"rdflib": {
"hashes": [
@@ -406,44 +595,44 @@
},
"redis": {
"hashes": [
- "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54",
- "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775"
+ "sha256:6946b5dca72e86103edc8033019cc3814c031232d339d5f4533b02ea85685175",
+ "sha256:8ca418d2ddca1b1a850afa1680a7d2fd1f3322739271de4b704e0d4668449273"
],
- "version": "==3.2.0"
+ "version": "==3.2.1"
},
"reportlab": {
"hashes": [
- "sha256:069f684cd0aaa518a27dc9124aed29cee8998e21ddf19604e53214ec8462bdd7",
- "sha256:09b68ec01d86b4b120456b3f3202570ec96f57624e3a4fc36f3829323391daa4",
- "sha256:0c32be9a406172c29ea20ff55a709ccac1e7fb09f15aba67cb7b455fd1d3dbe0",
- "sha256:233196cf25e97cfe7c452524ea29d9a4909f1cb66599299233be1efaaaa7a7a3",
- "sha256:2b5e4533f3e5b962835a5ce44467e66d1ecc822761d1b508077b5087a06be338",
- "sha256:2e860bcdace5a558356802a92ae8658d7e5fdaa00ded82e83a3f2987c562cb66",
- "sha256:3546029e63a9a9dc24ee38959eb417678c2425b96cd27b31e09e216dafc94666",
- "sha256:4452b93f9c73b6b70311e7d69082d64da81b38e91bfb4766397630092e6da6fd",
- "sha256:528c74a1c6527d1859c2c7a64a94a1cba485b00175162ea23699ae58a1e94939",
- "sha256:6116e750f98018febc08dfee6df20446cf954adbcfa378d2c703d56c8864aff3",
- "sha256:6b2b3580c647d75ef129172cb3da648cdb24566987b0b59c5ebb80ab770748d6",
- "sha256:727b5f2bed08552d143fc99649b1863c773729f580a416844f9d9967bb0a1ae8",
- "sha256:74c24a3ec0a3d4f8acb13a07192f45bdb54a1cc3c2286241677e7e8bcd5011fa",
- "sha256:98ccd2f8b4f8636db05f3f14db0b471ad6bb4b66ae0dc9052c4822b3bd5d6a7d",
- "sha256:a5905aa567946bc938b489a7249c7890c3fd3c9b7b5680dece5bc551c2ddbe0d",
- "sha256:acbb7f676b8586b770719e9683eda951fdb38eb7970d46fcbf3cdda88d912a64",
- "sha256:b5e30f865add48cf880f1c363eb505b97f2f7baaa88c155f87a335a76515a3e5",
- "sha256:be2a7c33a2c28bbd3f453ffe4f0e5200b88c803a097f4cf52d69c6b53fad7a8f",
- "sha256:c356bb600f59ac64955813d6497a08bfd5d0c451cb5829b61e3913d0ac084e26",
- "sha256:c7ec4ae2393beab584921b1287a04e94fd98c28315e348362d89b85f4b464546",
- "sha256:d476edc831bb3e9ebd04d1403abaf3ea57b3e4c2276c91a54fdfb6efbd3f9d97",
- "sha256:db059e1a0691c872784062421ec51848539eb4f5210142682e61059a5ca7cc55",
- "sha256:dd423a6753509ab14a0ac1b5be39d219c8f8d3781cce3deb4f45eda31969b5e8",
- "sha256:ed9b7c0d71ce6fe2b31c6cde530ad8238632b876a5d599218739bda142a77f7c",
- "sha256:f0a2465af4006f97b05e1f1546d67d3a3213d414894bf28be7f87f550a7f4a55",
- "sha256:f20bfe26e57e8e1f575a9e0325be04dd3562db9f247ffdd73b5d4df6dec53bc2",
- "sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c",
- "sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9"
+ "sha256:13714baa9753bfca94df67716cccb3eedcaaa30cf7bc40b282d338a718e0b610",
+ "sha256:16c1bb717a1a0e2ed065aa31eb5968dc03b34b728926216ef282cefeebf50c1b",
+ "sha256:2d9d66770880e8d112b6b925458593d34b84947c355847578cd974df0a3e3b8b",
+ "sha256:3334a30e477e1dfa0276eb41ed5bfd2a684c9917e55c6acb30d91abac46555f6",
+ "sha256:33796ea88d20c05958903c11ff34d896e462381f4a0f550854aabe6dd07cc189",
+ "sha256:5184f53c0babeedb4ebe297eb97794822cb122456ca03411c68256730c998d48",
+ "sha256:53589c5db35041920cd7a92a171506ff4eb5542ab8415af272fe4558927399a8",
+ "sha256:58ba0a9ca51d666d55ec7ecd83ab14763b79e7e5e0775b7717694e94c2fbbf18",
+ "sha256:6998652beba357da9687eba13b46ceccd0a7a2153d656cf8a03b7494c915e077",
+ "sha256:6c3b07c8a94ee9609c31a51e4131891c8330ffd379db23ab582fd225a06a4e59",
+ "sha256:7b248d2d9d4ab6d4cad91eb2b153b2c4c7b3fced89cb5a5b5bfbc7d09593871a",
+ "sha256:81d991c9994a576ea053b281b8c9afc28b12261197d478e72055d381f60fa26f",
+ "sha256:8a9a8be6841b88b13aa9c0f7d193c6d24b04b10c2e7cbf6657b1807bac5b1e9f",
+ "sha256:8de3107436e68014890adcec446207fd98d60c26e7feae6f550eea9eab3a622d",
+ "sha256:90f85afb68f7cd4fd8681af3123d23488274e4d1c3fea8d3831ef7257d9733c8",
+ "sha256:94857052c951ffa56de95cfce483fdf3de19587db4b1bc4f6a4043fb1c4af772",
+ "sha256:a47603d9b975e8870ed30ade22db3478b030dd7a6041c8272c3719d9bbeaef34",
+ "sha256:a5671b152d3f85963d8450e956ddecfb5d30af62dd1f73207fab9aa32a0240d2",
+ "sha256:a745cd1a4368fac093deff3b65614f052eced6afa9ed7fe223da2a52813f2e23",
+ "sha256:af454e8e844e3eeace5aead02701748b2a908f3e8cbc386cc5ddc185cef8c57f",
+ "sha256:c3c6c1234eed451111e969c776688e866554cb362250b88f782ab80ea62f9114",
+ "sha256:cc1cf8ba1b2f1669f5d873a7cfdb9e07a920243d74a66a78f8afa2cf78587864",
+ "sha256:cce3c9b0e115ea5553615a647b6644e5724bdc776e778593ffa5f383d907afb2",
+ "sha256:d137feacef83627d10adb869aa6998f29eb7af4cff3101c9fc94a1d73943b6cc",
+ "sha256:d7213814d050ca3b0bf7e018f94ed947e90477cd36aff298ff5932b849a0f36a",
+ "sha256:e381d08675807d2bb465717f69818040173351650af82730d721ecad429279a6",
+ "sha256:e39c6efdd64027be56ce991f7ffb86c7cee47da8c844c3544bbd68ef842831a0",
+ "sha256:f8526cfbbad599d22de6eb59b5a43610ba9b28f74ac2406125fe803f31a262a6"
],
"index": "pypi",
- "version": "==3.5.13"
+ "version": "==3.5.20"
},
"requests": {
"hashes": [
@@ -455,24 +644,24 @@
},
"requests-cache": {
"hashes": [
- "sha256:e9270030becc739b0a7f7f834234c73a878b2d794122bf76f40055a22419eb67",
- "sha256:fe561ca119879bbcfb51f03a35e35b425e18f338248e59fd5cf2166c77f457a2"
+ "sha256:6822f788c5ee248995c4bfbd725de2002ad710182ba26a666e85b64981866060",
+ "sha256:73a7211870f7d67af5fd81cad2f67cfe1cd3eb4ee6a85155e07613968cc72dfc"
],
- "version": "==0.4.13"
+ "version": "==0.5.0"
},
"shodan": {
"hashes": [
- "sha256:f93b7199e89eecf5c84647f66316c2c044c3aebfc1fe4d9caa43dfda07f74c4e"
+ "sha256:c30baebce853ad67677bf002dde96a1ca1a9729bdd300fbb3c5e5d889547a639"
],
"index": "pypi",
- "version": "==1.11.1"
+ "version": "==1.12.1"
},
"sigmatools": {
"hashes": [
- "sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b"
+ "sha256:ae980b6d6fd466294911efa493934d24e3c5df406da4a190b9fff0943a81cc5f"
],
"index": "pypi",
- "version": "==0.7.1"
+ "version": "==0.10"
},
"six": {
"hashes": [
@@ -483,19 +672,19 @@
},
"soupsieve": {
"hashes": [
- "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b",
- "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26"
+ "sha256:6898e82ecb03772a0d82bd0d0a10c0d6dcc342f77e0701d0ec4a8271be465ece",
+ "sha256:b20eff5e564529711544066d7dc0f7661df41232ae263619dede5059799cdfca"
],
- "version": "==1.8"
+ "version": "==1.9.1"
},
"sparqlwrapper": {
"hashes": [
- "sha256:2a95fdede2833be660b81092934c4a0054ff85f2693098556762a2759ea486f1",
- "sha256:7f4c8d38ea1bfcffbc358c9a05de35a3fd7152cc3e8ea57963ee7a0a242f7a5e",
- "sha256:acf6d60f0a3684cb673653b07871acb0c350a974b891f20f8ac94926ff9eb2ff"
+ "sha256:14ec551f0d60b4a496ffcc31f15337e844c085b8ead8cbe9a7178748a6de3794",
+ "sha256:21928e7a97f565e772cdeeb0abad428960f4307e3a13dbdd8f6d3da8a6a506c9",
+ "sha256:abc3e7eadcad32fa69a85c003853e2f6f73bda6cc999853838f401a5a1ea1109"
],
"index": "pypi",
- "version": "==1.8.2"
+ "version": "==1.8.4"
},
"stix2-patterns": {
"hashes": [
@@ -504,17 +693,23 @@
"index": "pypi",
"version": "==1.1.0"
},
+ "tabulate": {
+ "hashes": [
+ "sha256:8af07a39377cee1103a5c8b3330a421c2d99b9141e9cc5ddd2e3263fea416943"
+ ],
+ "version": "==0.8.3"
+ },
"tornado": {
"hashes": [
- "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d",
- "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409",
- "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f",
- "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f",
- "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5",
- "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb",
- "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444"
+ "sha256:1174dcb84d08887b55defb2cda1986faeeea715fff189ef3dc44cce99f5fca6b",
+ "sha256:2613fab506bd2aedb3722c8c64c17f8f74f4070afed6eea17f20b2115e445aec",
+ "sha256:44b82bc1146a24e5b9853d04c142576b4e8fa7a92f2e30bc364a85d1f75c4de2",
+ "sha256:457fcbee4df737d2defc181b9073758d73f54a6cfc1f280533ff48831b39f4a8",
+ "sha256:49603e1a6e24104961497ad0c07c799aec1caac7400a6762b687e74c8206677d",
+ "sha256:8c2f40b99a8153893793559919a355d7b74649a11e59f411b0b0a1793e160bc0",
+ "sha256:e1d897889c3b5a829426b7d52828fb37b28bc181cd598624e65c8be40ee3f7fa"
],
- "version": "==5.1.1"
+ "version": "==6.0.2"
},
"url-normalize": {
"hashes": [
@@ -532,10 +727,10 @@
},
"urllib3": {
"hashes": [
- "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
- "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
+ "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0",
+ "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3"
],
- "version": "==1.24.1"
+ "version": "==1.24.2"
},
"uwhois": {
"editable": true,
@@ -545,27 +740,36 @@
},
"vulners": {
"hashes": [
- "sha256:40041bcf893fa1bfaf29c650369d9a249991911f28b4d8795f7bc06508013e14",
- "sha256:6d00709300dcc7e2727499d8a60f51eaced1dc6b63cc19cb8a4b065b658c51aa",
- "sha256:de8cef247c9852c39bd54434e63026b46bdb2bd4ca22813bf66626b7d359b0f3"
+ "sha256:146ef130f215b50cdff790b06b4886c7edb325c075e9fce4bf1d3ab8d64a10d0",
+ "sha256:53406a86126159eaee9575fa667c99459bfdf9dd8c06bd0ce73fbe536b305e30",
+ "sha256:a258ccdbaee586207bc80d3590f0315ff151cfe16ea54f2e1629a6018fd9f2a3"
],
"index": "pypi",
- "version": "==1.4.4"
+ "version": "==1.5.0"
},
"wand": {
"hashes": [
- "sha256:7d6b8dc9d4eaccc430b9c86e6b749013220c994970a3f39e902b397e2fa732c3",
- "sha256:cc0b5c9cd50fecd10dc8888b739dd5984c6f8085d2954f34903b83ca39a91236"
+ "sha256:63ab24dee0264a44f5f045d4ecc0d392bc1cc195e5a2f80ce537b2c205c3033b",
+ "sha256:a2c318993791fab4fcfd460045415176f81d42f8c6fd8a88fb8d74d2f0f34b97",
+ "sha256:f68f32f2e4eca663a361d36148f06372de560442dcf8c785a53a64ee282572c9"
],
"index": "pypi",
- "version": "==0.5.1"
+ "version": "==0.5.3"
+ },
+ "xlrd": {
+ "hashes": [
+ "sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2",
+ "sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde"
+ ],
+ "index": "pypi",
+ "version": "==1.2.0"
},
"xlsxwriter": {
"hashes": [
- "sha256:de9ef46088489915eaaee00c7088cff93cf613e9990b46b933c98eb46f21b47f",
- "sha256:df96eafc3136d9e790e35d6725b473e46ada6f585c1f6519da69b27f5c8873f7"
+ "sha256:2a40b427dac0f640031e5b33abe97e761de6e0f12d4d346e7b2e2b67cf6ee927",
+ "sha256:431edc9ba1132eec1996939aa83fffe41885d3042ab09d47c3086f41a156c430"
],
- "version": "==1.1.5"
+ "version": "==1.1.7"
},
"yara-python": {
"hashes": [
@@ -611,17 +815,17 @@
},
"attrs": {
"hashes": [
- "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
- "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
+ "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
+ "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
],
- "version": "==18.2.0"
+ "version": "==19.1.0"
},
"certifi": {
"hashes": [
- "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
- "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
+ "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
+ "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
],
- "version": "==2018.11.29"
+ "version": "==2019.3.9"
},
"chardet": {
"hashes": [
@@ -640,39 +844,39 @@
},
"coverage": {
"hashes": [
- "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f",
- "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe",
- "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d",
- "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0",
- "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607",
- "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d",
- "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b",
- "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3",
- "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e",
- "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815",
- "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36",
- "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1",
- "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14",
- "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c",
- "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794",
- "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b",
- "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840",
- "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd",
- "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82",
- "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952",
- "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389",
- "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f",
- "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4",
- "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da",
- "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647",
- "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d",
- "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42",
- "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478",
- "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b",
- "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb",
- "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9"
+ "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9",
+ "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74",
+ "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390",
+ "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8",
+ "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe",
+ "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf",
+ "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e",
+ "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741",
+ "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09",
+ "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd",
+ "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034",
+ "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420",
+ "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c",
+ "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab",
+ "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba",
+ "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e",
+ "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609",
+ "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2",
+ "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49",
+ "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b",
+ "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d",
+ "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce",
+ "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9",
+ "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4",
+ "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773",
+ "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723",
+ "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c",
+ "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f",
+ "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1",
+ "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260",
+ "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a"
],
- "version": "==4.5.2"
+ "version": "==4.5.3"
},
"entrypoints": {
"hashes": [
@@ -705,11 +909,11 @@
},
"more-itertools": {
"hashes": [
- "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40",
- "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1"
+ "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7",
+ "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a"
],
"markers": "python_version > '2.7'",
- "version": "==6.0.0"
+ "version": "==7.0.0"
},
"nose": {
"hashes": [
@@ -743,18 +947,18 @@
},
"pyflakes": {
"hashes": [
- "sha256:5e8c00e30c464c99e0b501dc160b13a14af7f27d4dffb529c556e30a159e231d",
- "sha256:f277f9ca3e55de669fba45b7393a1449009cff5a37d1af10ebb76c52765269cd"
+ "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
+ "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
],
- "version": "==2.1.0"
+ "version": "==2.1.1"
},
"pytest": {
"hashes": [
- "sha256:067a1d4bf827ffdd56ad21bd46674703fce77c5957f6c1eef731f6146bfcef1c",
- "sha256:9687049d53695ad45cf5fdc7bbd51f0c49f1ea3ecfc4b7f3fde7501b541f17f4"
+ "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d",
+ "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5"
],
"index": "pypi",
- "version": "==4.3.0"
+ "version": "==4.4.1"
},
"requests": {
"hashes": [
@@ -773,10 +977,10 @@
},
"urllib3": {
"hashes": [
- "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
- "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
+ "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0",
+ "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3"
],
- "version": "==1.24.1"
+ "version": "==1.24.2"
}
}
}
diff --git a/README.md b/README.md
index 951de64..6ec5bc6 100644
--- a/README.md
+++ b/README.md
@@ -26,21 +26,32 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
* [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API.
* [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE).
+* [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox.
* [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name.
* [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes.
+* [docx-enrich](misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser).
* [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois.
* [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en).
* [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
* [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind.
+* [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise.
* [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset.
+* [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned?
* [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com).
* [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address.
* [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net.
* [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module).
+* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information.
+* [ocr-enrich](misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP.
+* [ods-enrich](misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser).
+* [odt-enrich](misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser).
* [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe.
* [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe.
* [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/).
* [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets.
+* [pdf-enrich](misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser).
+* [pptx-enrich](misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser).
+* [qrcode](misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values.
* [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
* [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
* [securitytrails](misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/).
@@ -59,12 +70,14 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
* [whois](misp_modules/modules/expansion) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd).
* [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module.
* [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module.
+* [xlsx-enrich](misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser).
* [YARA query](misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes.
* [YARA syntax validator](misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator.
### Export modules
* [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF).
+* [Cisco FireSight Manager ACL rule](misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL.
* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html).
* [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event.
* [PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF.
@@ -84,47 +97,41 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
* [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports.
* [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export.
-## How to install and start MISP modules in a Python virtualenv?
+## How to install and start MISP modules in a Python virtualenv? (recommended)
~~~~bash
-sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv
+sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick virtualenv libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev -y
sudo -u www-data virtualenv -p python3 /var/www/MISP/venv
cd /usr/local/src/
sudo git clone https://github.com/MISP/misp-modules.git
cd misp-modules
sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
sudo -u www-data /var/www/MISP/venv/bin/pip install .
-sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
-/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
-~~~~
-
-## How to install and start MISP modules on Debian-based distributions ?
-
-~~~~bash
-sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
-cd /usr/local/src/
-sudo git clone https://github.com/MISP/misp-modules.git
-cd misp-modules
-sudo pip3 install -I -r REQUIREMENTS
-sudo pip3 install -I .
-sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
+# Start misp-modules as a service
+sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now misp-modules
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
~~~~
## How to install and start MISP modules on RHEL-based distributions ?
As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository.
+
~~~~bash
-yum install rh-ruby22
+sudo yum install rh-ruby22
+sudo yum install openjpeg-devel
+sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel
cd /var/www/MISP
git clone https://github.com/MISP/misp-modules.git
cd misp-modules
-scl enable rh-python36 ‘python3 –m pip install cryptography’
-scl enable rh-python36 ‘python3 –m pip install -I -r REQUIREMENTS’
-scl enable rh-python36 ‘python3 –m pip install –I .’
+sudo -u apache /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
+sudo -u apache /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
+sudo -u apache /var/www/MISP/venv/bin/pip install -U .
~~~~
-Create the service file /etc/systemd/system/misp-workers.service :
+
+Create the service file /etc/systemd/system/misp-modules.service :
~~~~
-[Unit]
+echo "[Unit]
Description=MISP's modules
After=misp-workers.service
@@ -132,15 +139,16 @@ After=misp-workers.service
Type=simple
User=apache
Group=apache
-ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 ‘/opt/rh/rh-python36/root/bin/misp-modules –l 127.0.0.1 –s’
+ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/opt/rh/rh-python36/root/bin/misp-modules –l 127.0.0.1 –s'
Restart=always
RestartSec=10
[Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
~~~~
+
The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service.
-Then, enable the misp-modules service and start it ;
+Then, enable the misp-modules service and start it:
~~~~bash
systemctl daemon-reload
systemctl enable --now misp-modules
@@ -487,7 +495,7 @@ Download a pre-built virtual image from the [MISP training materials](https://ww
- Create a Host-Only adapter in VirtualBox
- Set your Misp OVA to that Host-Only adapter
- Start the virtual machine
-- Get the IP address of the virutal machine
+- Get the IP address of the virtual machine
- SSH into the machine (Login info on training page)
- Go into the misp-modules directory
@@ -505,16 +513,18 @@ sudo git checkout MyModBranch
Remove the contents of the build directory and re-install misp-modules.
-~~~python
+~~~bash
sudo rm -fr build/*
-sudo pip3 install --upgrade .
+sudo -u www-data /var/www/MISP/venv/bin/pip install --upgrade .
~~~
SSH in with a different terminal and run `misp-modules` with debugging enabled.
-~~~python
-sudo killall misp-modules
-misp-modules -d
+~~~bash
+# In case misp-modules is not a service do:
+# sudo killall misp-modules
+sudo systemctl disable --now misp-modules
+sudo -u www-data /var/www/MISP/venv/bin/misp-modules -d
~~~
diff --git a/REQUIREMENTS b/REQUIREMENTS
index 4709747..9447c47 100644
--- a/REQUIREMENTS
+++ b/REQUIREMENTS
@@ -1,65 +1,81 @@
-i https://pypi.org/simple
-e .
--e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client
--e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client
+-e git+https://github.com/D4-project/BGP-Ranking.git/@4e0741056bcc0077de1120b8724a31330b26033e#egg=pybgpranking&subdirectory=client
+-e git+https://github.com/D4-project/IPASN-History.git/@c0c2bbf8d70811982dad065ea463a7e01593a38d#egg=pyipasnhistory&subdirectory=client
-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
--e git+https://github.com/MISP/PyMISP.git@62e047f3c1972e21aa36a8882bebf4488cdc1f84#egg=pymisp
+-e git+https://github.com/MISP/PyMISP.git@582dda0ce2a8ca8e1dd2cf3842e0491caca51c62#egg=pymisp
-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client
+-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
-e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe
aiohttp==3.4.4
antlr4-python3-runtime==4.7.2 ; python_version >= '3'
async-timeout==3.0.1
-attrs==18.2.0
+attrs==19.1.0
+backscatter==0.2.4
beautifulsoup4==4.7.1
blockchain==1.4.4
-certifi==2018.11.29
+certifi==2019.3.9
chardet==3.0.4
-click-plugins==1.0.4
+click-plugins==1.1.1
click==7.0
colorama==0.4.1
dnspython==1.16.0
domaintools-api==0.3.3
enum-compat==0.0.2
ez-setup==0.9
+ezodf==0.3.2
future==0.17.1
-httplib2==0.12.1
+httplib2==0.12.3
idna-ssl==1.1.0 ; python_version < '3.7'
idna==2.8
isodate==0.6.0
-jsonschema==3.0.0
+jsonschema==3.0.1
+lxml==4.3.3
maclookup==1.0.3
multidict==4.5.2
+np==1.0.2
+numpy==1.16.3
oauth2==1.9.0.post1
+opencv-python==4.1.0.25
+pandas-ods-reader==0.0.6
+pandas==0.24.2
passivetotal==1.0.30
-pillow==5.4.1
-psutil==5.5.1
+pdftotext==2.1.1
+pillow==6.0.0
+psutil==5.6.2
pyeupi==1.0
pygeoip==0.3.2
-pyparsing==2.3.1
+pyparsing==2.4.0
pypdns==1.3
pypssl==2.1
-pyrsistent==0.14.11
+pyrsistent==0.15.1
pytesseract==0.2.6
python-dateutil==2.8.0
-pyyaml==3.13
+python-docx==0.8.10
+python-pptx==0.6.17
+pytz==2019.1
+pyyaml==5.1
+pyzbar==0.1.8
rdflib==4.2.2
-redis==3.2.0
-reportlab==3.5.13
-requests-cache==0.4.13
+redis==3.2.1
+reportlab==3.5.20
+requests-cache==0.5.0
requests==2.21.0
-shodan==1.11.1
-sigmatools==0.7.1
+shodan==1.12.1
+sigmatools==0.10
six==1.12.0
-soupsieve==1.8
-sparqlwrapper==1.8.2
+soupsieve==1.9.1
+sparqlwrapper==1.8.4
stix2-patterns==1.1.0
-tornado==5.1.1
+tabulate==0.8.3
+tornado==6.0.2
url-normalize==1.4.1
urlarchiver==0.2
-urllib3==1.24.1
-vulners==1.4.4
-wand==0.5.1
-xlsxwriter==1.1.5
+urllib3==1.24.2
+vulners==1.5.0
+wand==0.5.3
+xlrd==1.2.0
+xlsxwriter==1.1.7
yara-python==3.8.1
yarl==1.3.0
diff --git a/doc/README.md b/doc/README.md
index c32a8c4..02506ab 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -178,6 +178,25 @@ Module to query Crowdstrike Falcon.
-----
+#### [cuckoo_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py)
+
+
+
+An expansion module to submit files and URLs to Cuckoo Sandbox.
+- **features**:
+>The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox.
+> The returned task id can be used to retrieve results when the analysis completed.
+- **input**:
+>A malware-sample or attachment for files. A url or domain for URLs.
+- **output**:
+>A text field containing 'Cuckoo task id: '
+- **references**:
+>https://cuckoosandbox.org/, https://cuckoo.sh/docs/
+- **requirements**:
+>Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)
+
+-----
+
#### [cve](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py)
@@ -1081,7 +1100,13 @@ OSQuery export of a MISP event.
Simple export of a MISP event to PDF.
- **features**:
->The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event.
+>The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.
+> 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata
+> 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event
+> 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.
+> 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !
+> 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.
+> 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option
- **input**:
>MISP Event
- **output**:
@@ -1089,7 +1114,7 @@ Simple export of a MISP event to PDF.
- **references**:
>https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html
- **requirements**:
->PyMISP, asciidoctor
+>PyMISP, reportlab
-----
diff --git a/doc/expansion/cuckoo_submit.json b/doc/expansion/cuckoo_submit.json
new file mode 100644
index 0000000..7fe8067
--- /dev/null
+++ b/doc/expansion/cuckoo_submit.json
@@ -0,0 +1,9 @@
+{
+ "description": "An expansion module to submit files and URLs to Cuckoo Sandbox.",
+ "logo": "logos/cuckoo.png",
+ "requirements": ["Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)"],
+ "input": "A malware-sample or attachment for files. A url or domain for URLs.",
+ "output": "A text field containing 'Cuckoo task id: '",
+ "references": ["https://cuckoosandbox.org/", "https://cuckoo.sh/docs/"],
+ "features": "The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox.\n The returned task id can be used to retrieve results when the analysis completed."
+}
diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py
index d933dc9..440ad3f 100644
--- a/misp_modules/__init__.py
+++ b/misp_modules/__init__.py
@@ -55,7 +55,7 @@ log = logging.getLogger('misp-modules')
def handle_signal(sig, frame):
- IOLoop.instance().add_callback(IOLoop.instance().stop)
+ IOLoop.instance().add_callback_from_signal(IOLoop.instance().stop)
def init_logger(level=False):
@@ -210,37 +210,59 @@ class QueryModule(tornado.web.RequestHandler):
self.finish()
+def _launch_from_current_dir():
+ log.info('Launch MISP modules server from current directory.')
+ os.chdir(os.path.dirname(__file__))
+ modulesdir = 'modules'
+ helpersdir = 'helpers'
+ load_helpers(helpersdir=helpersdir)
+ return load_modules(modulesdir)
+
+
def main():
global mhandlers
global loaded_modules
signal.signal(signal.SIGINT, handle_signal)
signal.signal(signal.SIGTERM, handle_signal)
- argParser = argparse.ArgumentParser(description='misp-modules server')
+ argParser = argparse.ArgumentParser(description='misp-modules server', formatter_class=argparse.RawTextHelpFormatter)
argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
argParser.add_argument('-d', default=False, action='store_true', help='Enable debugging')
argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
argParser.add_argument('-m', default=[], action='append', help='Register a custom module')
+ argParser.add_argument('--devel', default=False, action='store_true', help='''Start in development mode, enable debug, start only the module(s) listed in -m.\nExample: -m misp_modules.modules.expansion.bgpranking''')
args = argParser.parse_args()
port = args.p
listen = args.l
- log = init_logger(level=args.d)
- if args.s:
- log.info('Launch MISP modules server from package.')
- load_package_helpers()
- mhandlers, loaded_modules = load_package_modules()
+ if args.devel:
+ log = init_logger(level=True)
+ log.info('Launch MISP modules server in developement mode. Enable debug, load a list of modules is -m is used.')
+ if args.m:
+ mhandlers = {}
+ modules = []
+ for module in args.m:
+ splitted = module.split(".")
+ modulename = splitted[-1]
+ moduletype = splitted[2]
+ mhandlers[modulename] = importlib.import_module(module)
+ mhandlers['type:' + modulename] = moduletype
+ modules.append(modulename)
+ log.info('MISP modules {0} imported'.format(modulename))
+ else:
+ mhandlers, loaded_modules = _launch_from_current_dir()
else:
- log.info('Launch MISP modules server from current directory.')
- os.chdir(os.path.dirname(__file__))
- modulesdir = 'modules'
- helpersdir = 'helpers'
- load_helpers(helpersdir=helpersdir)
- mhandlers, loaded_modules = load_modules(modulesdir)
+ log = init_logger(level=args.d)
+ if args.s:
+ log.info('Launch MISP modules server from package.')
+ load_package_helpers()
+ mhandlers, loaded_modules = load_package_modules()
+ else:
+ mhandlers, loaded_modules = _launch_from_current_dir()
- for module in args.m:
- mispmod = importlib.import_module(module)
- mispmod.register(mhandlers, loaded_modules)
+ for module in args.m:
+ mispmod = importlib.import_module(module)
+ mispmod.register(mhandlers, loaded_modules)
service = [(r'/modules', ListModules), (r'/query', QueryModule)]
@@ -266,8 +288,11 @@ def main():
if args.t:
log.info('MISP modules started in test-mode, quitting immediately.')
sys.exit()
- IOLoop.instance().start()
- IOLoop.instance().stop()
+ try:
+ IOLoop.instance().start()
+ finally:
+ IOLoop.instance().stop()
+
return 0
diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py
index d8fb153..70aca68 100644
--- a/misp_modules/modules/expansion/__init__.py
+++ b/misp_modules/modules/expansion/__init__.py
@@ -1,6 +1,6 @@
from . import _vmray # noqa
-__all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
+__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',
'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal',
'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep',
@@ -8,4 +8,6 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl',
'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator',
'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io',
- 'intel471', 'backscatter_io', 'btc_scam_check']
+ 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors',
+ 'qrcode', 'ocr-enrich', 'pdf-enrich', 'docx-enrich', 'xlsx-enrich', 'pptx-enrich',
+ 'ods-enrich', 'odt-enrich']
diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py
index 430c67d..04b7138 100755
--- a/misp_modules/modules/expansion/btc_steroids.py
+++ b/misp_modules/modules/expansion/btc_steroids.py
@@ -1,4 +1,3 @@
-import sys
import json
import requests
import time
@@ -91,6 +90,7 @@ def mprint(input):
def handler(q=False):
global result_text
global conversion_rates
+ result_text = ""
# start_time = time.time()
# now = time.time()
if q is False:
@@ -105,7 +105,6 @@ def handler(q=False):
btc = request['btc']
else:
return False
-
mprint("\nAddress:\t" + btc)
try:
req = requests.get(blockchain_all.format(btc, "&limit=50"))
@@ -113,8 +112,18 @@ def handler(q=False):
except Exception:
# print(e)
print(req.text)
- result_text = ""
- sys.exit(1)
+ result_text = "Not a valid BTC address"
+ r = {
+ 'results': [
+ {
+ 'types': ['text'],
+ 'values':[
+ str(result_text)
+ ]
+ }
+ ]
+ }
+ return r
n_tx = jreq['n_tx']
balance = float(jreq['final_balance'] / 100000000)
diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py
new file mode 100644
index 0000000..c1ded90
--- /dev/null
+++ b/misp_modules/modules/expansion/cuckoo_submit.py
@@ -0,0 +1,153 @@
+import base64
+import io
+import json
+import logging
+import requests
+import sys
+import urllib.parse
+import zipfile
+
+from requests.exceptions import RequestException
+
+log = logging.getLogger("cuckoo_submit")
+log.setLevel(logging.DEBUG)
+sh = logging.StreamHandler(sys.stdout)
+sh.setLevel(logging.DEBUG)
+fmt = logging.Formatter(
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+sh.setFormatter(fmt)
+log.addHandler(sh)
+
+moduleinfo = {
+ "version": "0.1", "author": "Evert Kors",
+ "description": "Submit files and URLs to Cuckoo Sandbox",
+ "module-type": ["expansion", "hover"]
+}
+misperrors = {"error": "Error"}
+moduleconfig = ["api_url", "api_key"]
+mispattributes = {
+ "input": ["attachment", "malware-sample", "url", "domain"],
+ "output": ["text"]
+}
+
+
+class APIKeyError(RequestException):
+ """Raised if the Cuckoo API returns a 401. This means no or an invalid
+ bearer token was supplied."""
+ pass
+
+
+class CuckooAPI(object):
+
+ def __init__(self, api_url, api_key=""):
+ self.api_key = api_key
+ if not api_url.startswith("http"):
+ api_url = "https://{}".format(api_url)
+
+ self.api_url = api_url
+
+ def _post_api(self, endpoint, files=None, data={}):
+ data.update({
+ "owner": "MISP"
+ })
+
+ try:
+ response = requests.post(
+ urllib.parse.urljoin(self.api_url, endpoint),
+ files=files, data=data,
+ headers={"Authorization": "Bearer {}".format(self.api_key)}
+ )
+ except RequestException as e:
+ log.error("Failed to submit sample to Cuckoo Sandbox. %s", e)
+ return None
+
+ if response.status_code == 401:
+ raise APIKeyError("Invalid or no Cuckoo Sandbox API key provided")
+
+ if response.status_code != 200:
+ log.error("Invalid Cuckoo API response")
+ return None
+
+ return response.json()
+
+ def create_task(self, filename, fp):
+ response = self._post_api(
+ "/tasks/create/file", files={"file": (filename, fp)}
+ )
+ if not response:
+ return False
+
+ return response["task_id"]
+
+ def create_url(self, url):
+ response = self._post_api(
+ "/tasks/create/url", data={"url": url}
+ )
+ if not response:
+ return False
+
+ return response["task_id"]
+
+
+def handler(q=False):
+ if q is False:
+ return False
+
+ request = json.loads(q)
+
+ # See if the API URL was provided. The API key is optional, as it can
+ # be disabled in the Cuckoo API settings.
+ api_url = request["config"].get("api_url")
+ api_key = request["config"].get("api_key", "")
+ if not api_url:
+ misperrors["error"] = "No Cuckoo API URL provided"
+ return misperrors
+
+ url = request.get("url") or request.get("domain")
+ data = request.get("data")
+ filename = None
+ if data:
+ data = base64.b64decode(data)
+
+ if "malware-sample" in request:
+ filename = request.get("malware-sample").split("|", 1)[0]
+ with zipfile.ZipFile(io.BytesIO(data)) as zipf:
+ data = zipf.read(zipf.namelist()[0], pwd=b"infected")
+
+ elif "attachment" in request:
+ filename = request.get("attachment")
+
+ cuckoo_api = CuckooAPI(api_url=api_url, api_key=api_key)
+ task_id = None
+ try:
+ if url:
+ log.debug("Submitting URL to Cuckoo Sandbox %s", api_url)
+ task_id = cuckoo_api.create_url(url)
+ elif data and filename:
+ log.debug("Submitting file to Cuckoo Sandbox %s", api_url)
+ task_id = cuckoo_api.create_task(
+ filename=filename, fp=io.BytesIO(data)
+ )
+ except APIKeyError as e:
+ misperrors["error"] = "Failed to submit to Cuckoo: {}".format(e)
+ return misperrors
+
+ if not task_id:
+ misperrors["error"] = "File or URL submission failed"
+ return misperrors
+
+ return {
+ "results": [
+ {"types": "text", "values": "Cuckoo task id: {}".format(task_id)}
+ ]
+ }
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo["config"] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/docx-enrich.py b/misp_modules/modules/expansion/docx-enrich.py
new file mode 100644
index 0000000..d5da3f8
--- /dev/null
+++ b/misp_modules/modules/expansion/docx-enrich.py
@@ -0,0 +1,61 @@
+import json
+import binascii
+import np
+import docx
+import io
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'],
+ 'output': ['freetext', 'text']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+ 'description': '.docx to freetext-import IOC extractor',
+ 'module-type': ['expansion']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ q = json.loads(q)
+ filename = q['attachment']
+ try:
+ docx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8)
+ except Exception as e:
+ print(e)
+ err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?"
+ misperrors['error'] = err
+ print(err)
+ return misperrors
+
+ doc_content = ""
+ doc_file = io.BytesIO(docx_array)
+ try:
+ doc = docx.Document(doc_file)
+ for para in doc.paragraphs:
+ print(para.text)
+ doc_content = doc_content + "\n" + para.text
+ tables = doc.tables
+ for table in tables:
+ for row in table.rows:
+ for cell in row.cells:
+ for para in cell.paragraphs:
+ print(para.text)
+ doc_content = doc_content + "\n" + para.text
+ print(doc_content)
+ return {'results': [{'types': ['freetext'], 'values': doc_content, 'comment': ".docx-to-text from file " + filename},
+ {'types': ['text'], 'values': doc_content, 'comment': ".docx-to-text from file " + filename}]}
+ except Exception as e:
+ print(e)
+ err = "Couldn't analyze file as .docx. Error was: " + str(e)
+ misperrors['error'] = err
+ return misperrors
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py
new file mode 100644
index 0000000..d26736d
--- /dev/null
+++ b/misp_modules/modules/expansion/greynoise.py
@@ -0,0 +1,43 @@
+import requests
+import json
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']}
+moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab ', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']}
+moduleconfig = ['user-agent'] # TODO take this into account in the code
+
+greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip'
+default_user_agent = 'MISP-Module'
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ request = json.loads(q)
+ for input_type in mispattributes['input']:
+ if input_type in request:
+ ip = request[input_type]
+ break
+ else:
+ misperrors['error'] = "Unsupported attributes type"
+ return misperrors
+ data = {'ip': ip}
+ r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent}) # Real request
+ if r.status_code == 200: # OK (record found)
+ response = json.loads(r.text)
+ if response:
+ return {'results': [{'types': mispattributes['output'], 'values': response}]}
+ elif r.status_code == 404: # Not found (not an error)
+ return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]}
+ else: # Real error
+ misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')'
+ return misperrors['error']
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py
new file mode 100644
index 0000000..8db3fa7
--- /dev/null
+++ b/misp_modules/modules/expansion/hibp.py
@@ -0,0 +1,43 @@
+import requests
+import json
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']} # All mails as input
+moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API.', 'module-type': ['hover']}
+moduleconfig = ['user-agent'] # TODO take this into account in the code
+
+haveibeenpwned_api_url = 'https://api.haveibeenpwned.com/api/v2/breachedaccount/'
+default_user_agent = 'MISP-Module' # User agent (must be set, requiered by API))
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ request = json.loads(q)
+ for input_type in mispattributes['input']:
+ if input_type in request:
+ email = request[input_type]
+ break
+ else:
+ misperrors['error'] = "Unsupported attributes type"
+ return misperrors
+
+ r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent}) # Real request
+ if r.status_code == 200: # OK (record found)
+ breaches = json.loads(r.text)
+ if breaches:
+ return {'results': [{'types': mispattributes['output'], 'values': breaches}]}
+ elif r.status_code == 404: # Not found (not an error)
+ return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]}
+ else: # Real error
+ misperrors['error'] = 'haveibeenpwned.com API not accessible (HTTP ' + str(r.status_code) + ')'
+ return misperrors['error']
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/macvendors.py b/misp_modules/modules/expansion/macvendors.py
new file mode 100644
index 0000000..55d0ef3
--- /dev/null
+++ b/misp_modules/modules/expansion/macvendors.py
@@ -0,0 +1,42 @@
+import requests
+import json
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['mac-address'], 'output': ['text']}
+moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access Macvendors API.', 'module-type': ['hover']}
+moduleconfig = ['user-agent'] # TODO take this into account in the code
+
+macvendors_api_url = 'https://api.macvendors.com/'
+default_user_agent = 'MISP-Module'
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ request = json.loads(q)
+ for input_type in mispattributes['input']:
+ if input_type in request:
+ mac = request[input_type]
+ break
+ else:
+ misperrors['error'] = "Unsupported attributes type"
+ return misperrors
+ r = requests.get(macvendors_api_url + mac, headers={'user-agent': default_user_agent}) # Real request
+ if r.status_code == 200: # OK (record found)
+ response = r.text
+ if response:
+ return {'results': [{'types': mispattributes['output'], 'values': response}]}
+ elif r.status_code == 404: # Not found (not an error)
+ return {'results': [{'types': mispattributes['output'], 'values': 'Not found'}]}
+ else: # Real error
+ misperrors['error'] = 'MacVendors API not accessible (HTTP ' + str(r.status_code) + ')'
+ return misperrors['error']
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/ocr-enrich.py b/misp_modules/modules/expansion/ocr-enrich.py
new file mode 100644
index 0000000..cd6baca
--- /dev/null
+++ b/misp_modules/modules/expansion/ocr-enrich.py
@@ -0,0 +1,50 @@
+import json
+import binascii
+import cv2
+import np
+import pytesseract
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'],
+ 'output': ['freetext', 'text']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+ 'description': 'OCR decoder',
+ 'module-type': ['expansion']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ q = json.loads(q)
+ filename = q['attachment']
+ try:
+ img_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8)
+ except Exception as e:
+ print(e)
+ err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?"
+ misperrors['error'] = err
+ print(err)
+ return misperrors
+
+ image = img_array
+ image = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
+ try:
+ decoded = pytesseract.image_to_string(image)
+ return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename},
+ {'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]}
+ except Exception as e:
+ print(e)
+ err = "Couldn't analyze file type. Only images are supported right now."
+ misperrors['error'] = err
+ return misperrors
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/ods-enrich.py b/misp_modules/modules/expansion/ods-enrich.py
new file mode 100644
index 0000000..b247c44
--- /dev/null
+++ b/misp_modules/modules/expansion/ods-enrich.py
@@ -0,0 +1,56 @@
+import json
+import binascii
+import np
+import ezodf
+import pandas_ods_reader
+import io
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'],
+ 'output': ['freetext', 'text']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+ 'description': '.ods to freetext-import IOC extractor',
+ 'module-type': ['expansion']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ q = json.loads(q)
+ filename = q['attachment']
+ try:
+ ods_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8)
+ except Exception as e:
+ print(e)
+ err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?"
+ misperrors['error'] = err
+ print(err)
+ return misperrors
+
+ ods_content = ""
+ ods_file = io.BytesIO(ods_array)
+ doc = ezodf.opendoc(ods_file)
+ num_sheets = len(doc.sheets)
+ try:
+ for i in range(0, num_sheets):
+ ods = pandas_ods_reader.read_ods(ods_file, i, headers=False)
+ ods_content = ods_content + "\n" + ods.to_string(max_rows=None)
+ print(ods_content)
+ return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename},
+ {'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]}
+ except Exception as e:
+ print(e)
+ err = "Couldn't analyze file as .ods. Error was: " + str(e)
+ misperrors['error'] = err
+ return misperrors
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/odt-enrich.py b/misp_modules/modules/expansion/odt-enrich.py
new file mode 100644
index 0000000..c4513ae
--- /dev/null
+++ b/misp_modules/modules/expansion/odt-enrich.py
@@ -0,0 +1,51 @@
+import json
+import binascii
+import np
+from ODTReader.odtreader import odtToText
+import io
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'],
+ 'output': ['freetext', 'text']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+ 'description': '.odt to freetext-import IOC extractor',
+ 'module-type': ['expansion']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ q = json.loads(q)
+ filename = q['attachment']
+ try:
+ odt_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8)
+ except Exception as e:
+ print(e)
+ err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?"
+ misperrors['error'] = err
+ print(err)
+ return misperrors
+
+ odt_content = ""
+ odt_file = io.BytesIO(odt_array)
+ try:
+ odt_content = odtToText(odt_file)
+ print(odt_content)
+ return {'results': [{'types': ['freetext'], 'values': odt_content, 'comment': ".odt-to-text from file " + filename},
+ {'types': ['text'], 'values': odt_content, 'comment': ".odt-to-text from file " + filename}]}
+ except Exception as e:
+ print(e)
+ err = "Couldn't analyze file as .odt. Error was: " + str(e)
+ misperrors['error'] = err
+ return misperrors
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/pdf-enrich.py b/misp_modules/modules/expansion/pdf-enrich.py
new file mode 100644
index 0000000..ef85fde
--- /dev/null
+++ b/misp_modules/modules/expansion/pdf-enrich.py
@@ -0,0 +1,48 @@
+import json
+import binascii
+import np
+import pdftotext
+import io
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'],
+ 'output': ['freetext', 'text']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+ 'description': 'PDF to freetext-import IOC extractor',
+ 'module-type': ['expansion']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ q = json.loads(q)
+ filename = q['attachment']
+ try:
+ pdf_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8)
+ except Exception as e:
+ print(e)
+ err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?"
+ misperrors['error'] = err
+ print(err)
+ return misperrors
+
+ pdf_file = io.BytesIO(pdf_array)
+ try:
+ pdf_content = "\n\n".join(pdftotext.PDF(pdf_file))
+ return {'results': [{'types': ['freetext'], 'values': pdf_content, 'comment': "PDF-to-text from file " + filename}]}
+ except Exception as e:
+ print(e)
+ err = "Couldn't analyze file as PDF. Error was: " + str(e)
+ misperrors['error'] = err
+ return misperrors
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/pptx-enrich.py b/misp_modules/modules/expansion/pptx-enrich.py
new file mode 100644
index 0000000..816e439
--- /dev/null
+++ b/misp_modules/modules/expansion/pptx-enrich.py
@@ -0,0 +1,55 @@
+import json
+import binascii
+import np
+from pptx import Presentation
+import io
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'],
+ 'output': ['freetext', 'text']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+ 'description': '.pptx to freetext-import IOC extractor',
+ 'module-type': ['expansion']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ q = json.loads(q)
+ filename = q['attachment']
+ try:
+ pptx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8)
+ except Exception as e:
+ print(e)
+ err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?"
+ misperrors['error'] = err
+ print(err)
+ return misperrors
+
+ ppt_content = ""
+ ppt_file = io.BytesIO(pptx_array)
+ try:
+ ppt = Presentation(ppt_file)
+ for slide in ppt.slides:
+ for shape in slide.shapes:
+ if hasattr(shape, "text"):
+ print(shape.text)
+ ppt_content = ppt_content + "\n" + shape.text
+ return {'results': [{'types': ['freetext'], 'values': ppt_content, 'comment': ".pptx-to-text from file " + filename},
+ {'types': ['text'], 'values': ppt_content, 'comment': ".pptx-to-text from file " + filename}]}
+ except Exception as e:
+ print(e)
+ err = "Couldn't analyze file as .pptx. Error was: " + str(e)
+ misperrors['error'] = err
+ return misperrors
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py
new file mode 100644
index 0000000..9a62827
--- /dev/null
+++ b/misp_modules/modules/expansion/qrcode.py
@@ -0,0 +1,89 @@
+import json
+from pyzbar import pyzbar
+import cv2
+import re
+import binascii
+import np
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'],
+ 'output': ['url', 'btc']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+ 'description': 'QR code decoder',
+ 'module-type': ['expansion', 'hover']}
+
+debug = True
+debug_prefix = "[DEBUG] QR Code module: "
+# format example: bitcoin:1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh?amount=0.15424
+# format example: http://example.com
+cryptocurrencies = ['bitcoin']
+schemas = ['http://', 'https://', 'ftp://']
+moduleconfig = []
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ q = json.loads(q)
+ filename = q['attachment']
+ try:
+ img_array = np.fromstring(binascii.a2b_base64(q['data']), np.uint8)
+ except Exception as e:
+ err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?"
+ misperrors['error'] = err
+ print(err)
+ print(e)
+ return misperrors
+ image = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
+ if q:
+ barcodes = pyzbar.decode(image)
+ for item in barcodes:
+ try:
+ result = item.data.decode()
+ except Exception as e:
+ print(e)
+ return
+ if debug:
+ print(debug_prefix + result)
+ for item in cryptocurrencies:
+ if item in result:
+ try:
+ currency, address, extra = re.split(r'\:|\?', result)
+ except Exception as e:
+ print(e)
+ if currency in cryptocurrencies:
+ try:
+ amount = re.split('=', extra)[1]
+ if debug:
+ print(debug_prefix + address)
+ print(debug_prefix + amount)
+ return {'results': [{'types': ['btc'], 'values': address, 'comment': "BTC: " + amount + " from file " + filename}]}
+ except Exception as e:
+ print(e)
+ else:
+ print(address)
+ for item in schemas:
+ if item in result:
+ try:
+ url = result
+ if debug:
+ print(debug_prefix + url)
+ return {'results': [{'types': ['url'], 'values': url, 'comment': "from QR code of file " + filename}]}
+ except Exception as e:
+ print(e)
+ else:
+ try:
+ return {'results': [{'types': ['text'], 'values': result, 'comment': "from QR code of file " + filename}]}
+ except Exception as e:
+ print(e)
+ misperrors['error'] = "Couldn't decode QR code in attachment."
+ return misperrors
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/expansion/xlsx-enrich.py b/misp_modules/modules/expansion/xlsx-enrich.py
new file mode 100644
index 0000000..6e0ee73
--- /dev/null
+++ b/misp_modules/modules/expansion/xlsx-enrich.py
@@ -0,0 +1,53 @@
+import json
+import binascii
+import np
+import pandas
+import io
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['attachment'],
+ 'output': ['freetext', 'text']}
+moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen',
+ 'description': '.xlsx to freetext-import IOC extractor',
+ 'module-type': ['expansion']}
+
+moduleconfig = []
+
+
+def handler(q=False):
+ if q is False:
+ return False
+ q = json.loads(q)
+ filename = q['attachment']
+ try:
+ xlsx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8)
+ except Exception as e:
+ print(e)
+ err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?"
+ misperrors['error'] = err
+ print(err)
+ return misperrors
+
+ xls_content = ""
+ xls_file = io.BytesIO(xlsx_array)
+ pandas.set_option('display.max_colwidth', -1)
+ try:
+ xls = pandas.read_excel(xls_file)
+ xls_content = xls.to_string(max_rows=None)
+ print(xls_content)
+ return {'results': [{'types': ['freetext'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename},
+ {'types': ['text'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}]}
+ except Exception as e:
+ print(e)
+ err = "Couldn't analyze file as .xlsx. Error was: " + str(e)
+ misperrors['error'] = err
+ return misperrors
+
+
+def introspection():
+ return mispattributes
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py
new file mode 100644
index 0000000..ab79692
--- /dev/null
+++ b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py
@@ -0,0 +1,140 @@
+######################################################
+# #
+# Author: Stanislav Klevtsov, Ukraine; Feb 2019. #
+# #
+# #
+# Script was tested on the following configuration: #
+# MISP v2.4.90 #
+# Cisco Firesight Manager Console v6.2.3 (bld 84) #
+# #
+######################################################
+
+import json
+import base64
+from urllib.parse import quote
+
+misperrors = {'error': 'Error'}
+
+moduleinfo = {'version': '1', 'author': 'Stanislav Klevtsov',
+ 'description': 'Export malicious network activity attributes of the MISP event to Cisco firesight manager block rules',
+ 'module-type': ['export']}
+
+
+moduleconfig = ['fmc_ip_addr', 'fmc_login', 'fmc_pass', 'domain_id', 'acpolicy_id']
+
+fsmapping = {"ip-dst": "dst", "url": "request"}
+
+mispattributes = {'input': list(fsmapping.keys())}
+
+# options: event, attribute, event-collection, attribute-collection
+inputSource = ['event']
+
+outputFileExtension = 'sh'
+responseType = 'application/txt'
+
+# .sh file templates
+SH_FILE_HEADER = """#!/bin/sh\n\n"""
+
+BLOCK_JSON_TMPL = """
+BLOCK_RULE='{{ "action": "BLOCK", "enabled": true, "type": "AccessRule", "name": "{rule_name}", "destinationNetworks": {{ "literals": [ {dst_networks} ] }}, "urls": {{ "literals": [ {urls} ] }}, "newComments": [ "{event_info_comment}" ] }}'\n
+"""
+
+BLOCK_DST_JSON_TMPL = """{{ "type": "Host", "value": "{ipdst}" }} """
+BLOCK_URL_JSON_TMPL = """{{ "type": "Url", "url": "{url}" }} """
+
+CURL_ADD_RULE_TMPL = """
+curl -X POST -v -k -H 'Content-Type: application/json' -H \"Authorization: Basic $LOGINPASS_BASE64\" -H \"X-auth-access-token: $ACC_TOKEN\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_config/v1/domain/$DOMAIN_ID/policy/accesspolicies/$ACPOLICY_ID/accessrules\" --data \"$BLOCK_RULE\" """
+
+
+def handler(q=False):
+ if q is False:
+ return False
+
+ r = {'results': []}
+ request = json.loads(q)
+
+ if "config" in request:
+ config = request["config"]
+
+ # check if config is empty
+ if not config['fmc_ip_addr']:
+ config['fmc_ip_addr'] = "0.0.0.0"
+ if not config['fmc_login']:
+ config['fmc_login'] = "login"
+ if not config['fmc_pass']:
+ config['fmc_pass'] = "password"
+ if not config['domain_id']:
+ config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID"
+ if not config['acpolicy_id']:
+ config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID"
+
+ data = request["data"]
+ output = ""
+ ipdst = []
+ urls = []
+
+ # populate the ACL rule with attributes
+ for ev in data:
+
+ event = ev["Attribute"]
+ event_id = ev["Event"]["id"]
+ event_info = ev["Event"]["info"]
+
+ for index, attr in enumerate(event):
+ if attr["to_ids"] is True:
+ if attr["type"] in fsmapping:
+ if attr["type"] == "ip-dst":
+ ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"]))
+ else:
+ urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*')))
+
+ # building the .sh file
+ output += SH_FILE_HEADER
+ output += "FIRESIGHT_IP_ADDR='{}'\n".format(config['fmc_ip_addr'])
+
+ output += "LOGINPASS_BASE64=`echo -n '{}:{}' | base64`\n".format(config['fmc_login'], config['fmc_pass'])
+ output += "DOMAIN_ID='{}'\n".format(config['domain_id'])
+ output += "ACPOLICY_ID='{}'\n\n".format(config['acpolicy_id'])
+
+ output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`\n"
+
+ output += BLOCK_JSON_TMPL.format(rule_name="misp_event_{}".format(event_id),
+ dst_networks=', '.join(ipdst),
+ urls=', '.join(urls),
+ event_info_comment=event_info) + "\n"
+
+ output += CURL_ADD_RULE_TMPL
+ # END building the .sh file
+
+ r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')}
+ return r
+
+
+def introspection():
+ modulesetup = {}
+ try:
+ responseType
+ modulesetup['responseType'] = responseType
+ except NameError:
+ pass
+ try:
+ userConfig
+ modulesetup['userConfig'] = userConfig
+ except NameError:
+ pass
+ try:
+ outputFileExtension
+ modulesetup['outputFileExtension'] = outputFileExtension
+ except NameError:
+ pass
+ try:
+ inputSource
+ modulesetup['inputSource'] = inputSource
+ except NameError:
+ pass
+ return modulesetup
+
+
+def version():
+ moduleinfo['config'] = moduleconfig
+ return moduleinfo
diff --git a/tests/test_expansions.py b/tests/test_expansions.py
new file mode 100644
index 0000000..d581a31
--- /dev/null
+++ b/tests/test_expansions.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import unittest
+import requests
+from urllib.parse import urljoin
+
+
+class TestExpansions(unittest.TestCase):
+
+ def setUp(self):
+ self.maxDiff = None
+ self.headers = {'Content-Type': 'application/json'}
+ self.url = "http://127.0.0.1:6666/"
+
+ def misp_modules_post(self, query):
+ return requests.post(urljoin(self.url, "query"), json=query)
+
+ def get_values(self, response):
+ return response.json()['results'][0]['values']
+
+ def test_cve(self):
+ query = {"module": "cve", "vulnerability": "CVE-2010-3333"}
+ response = self.misp_modules_post(query)
+ self.assertTrue(self.get_values(response).startswith("Stack-based buffer overflow in Microsoft Office XP SP3, Office 2003 SP3"))
+
+ def test_dns(self):
+ query = {"module": "dns", "hostname": "www.circl.lu", "config": {"nameserver": "8.8.8.8"}}
+ response = self.misp_modules_post(query)
+ self.assertEqual(self.get_values(response), ['149.13.33.14'])
+
+ def test_macvendors(self):
+ query = {"module": "macvendors", "mac-address": "FC-A1-3E-2A-1C-33"}
+ response = self.misp_modules_post(query)
+ self.assertEqual(self.get_values(response), 'Samsung Electronics Co.,Ltd')
+
+ def test_haveibeenpwned(self):
+ query = {"module": "hibp", "email-src": "info@circl.lu"}
+ response = self.misp_modules_post(query)
+ self.assertEqual(self.get_values(response), 'OK (Not Found)')
+
+ def test_greynoise(self):
+ query = {"module": "greynoise", "ip-dst": "1.1.1.1"}
+ response = self.misp_modules_post(query)
+ self.assertEqual(self.get_values(response)['status'], 'ok')
+
+ def test_ipasn(self):
+ query = {"module": "ipasn", "ip-dst": "1.1.1.1"}
+ response = self.misp_modules_post(query)
+ key = list(self.get_values(response)['response'].keys())[0]
+ entry = self.get_values(response)['response'][key]['asn']
+ self.assertEqual(entry, '13335')
+
+ def test_bgpranking(self):
+ query = {"module": "bgpranking", "AS": "13335"}
+ response = self.misp_modules_post(query)
+ self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET - Cloudflare, Inc., US')