pull/285/head
Sascha Rommelfangen 2019-03-14 14:41:24 +01:00
commit 5af667edff
28 changed files with 1931 additions and 1694 deletions

View File

@ -19,14 +19,14 @@ script:
- pid=$!
- sleep 5
- pipenv run nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid
- kill -s KILL $pid
- pushd ~/
- pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
- pid=$!
- popd
- sleep 5
- pipenv run nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid
- kill -s KILL $pid
- pipenv run flake8 --ignore=E501,W503 misp_modules
after_success:

View File

@ -25,12 +25,13 @@ pytesseract = "*"
pygeoip = "*"
beautifulsoup4 = "*"
oauth2 = "*"
yara-python = ">=3.8.0"
yara-python = "==3.8.1"
sigmatools = "*"
stix2-patterns = "*"
maclookup = "*"
vulners = "*"
blockchain = "*"
reportlab = "*"
pyintel471 = {editable = true,git = "https://github.com/MISP/PyIntel471.git"}
shodan = "*"
Pillow = "*"
@ -40,6 +41,7 @@ domaintools_api = "*"
misp-modules = {editable = true,path = "."}
pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"}
pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"}
backscatter = "*"
[requires]
python_version = "3.6"

301
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "f501a84bdd41ca21a2af020278ce030985cccd5f2f5683cd075797be4523587d"
"sha256": "23dec0fa6400c828e294ea9981b433903c17358ca61d7abdaec8df5a1c89f08c"
},
"pipfile-spec": 6,
"requires": {
@ -59,10 +59,18 @@
},
"attrs": {
"hashes": [
"sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
"sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
"sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
"sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
],
"version": "==18.2.0"
"version": "==19.1.0"
},
"backscatter": {
"hashes": [
"sha256:7a0d1aa3661635de81e2a09b15d53e35cbe399a111cc58a70925f80e6874abd3",
"sha256:afb0efcf5d2551dac953ec4c38fb710b274b8e811775650e02c1ef42cafb14c8"
],
"index": "pypi",
"version": "==0.2.4"
},
"beautifulsoup4": {
"hashes": [
@ -82,10 +90,10 @@
},
"certifi": {
"hashes": [
"sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
"sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
"sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
"sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
],
"version": "==2018.11.29"
"version": "==2019.3.9"
},
"chardet": {
"hashes": [
@ -150,9 +158,9 @@
},
"httplib2": {
"hashes": [
"sha256:f61fb838a94ce3b349aa32c92fd8430f7e3511afdb18bf9640d647e30c90a6d6"
"sha256:4ba6b8fd77d0038769bf3c33c9a96a6f752bc4cdf739701fdcaf210121f399d4"
],
"version": "==0.12.0"
"version": "==0.12.1"
},
"idna": {
"hashes": [
@ -177,10 +185,10 @@
},
"jsonschema": {
"hashes": [
"sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08",
"sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02"
"sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d",
"sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a"
],
"version": "==2.6.0"
"version": "==3.0.1"
},
"maclookup": {
"hashes": [
@ -281,22 +289,22 @@
},
"psutil": {
"hashes": [
"sha256:1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736",
"sha256:1c71b9716790e202a00ab0931a6d1e25db1aa1198bcacaea2f5329f75d257fff",
"sha256:3b7a4daf4223dae171a67a89314ac5ca0738e94064a78d99cfd751c55d05f315",
"sha256:3e19be3441134445347af3767fa7770137d472a484070840eee6653b94ac5576",
"sha256:6e265c8f3da00b015d24b842bfeb111f856b13d24f2c57036582568dc650d6c3",
"sha256:809c9cef0402e3e48b5a1dddc390a8a6ff58b15362ea5714494073fa46c3d293",
"sha256:b4d1b735bf5b120813f4c89db8ac22d89162c558cbd7fdd298866125fe906219",
"sha256:bbffac64cfd01c6bcf90eb1bedc6c80501c4dae8aef4ad6d6dd49f8f05f6fc5a",
"sha256:bfcea4f189177b2d2ce4a34b03c4ac32c5b4c22e21f5b093d9d315e6e253cd81"
"sha256:1020a37214c4138e34962881372b40f390582b5c8245680c04349c2afb785a25",
"sha256:151c9858c268a1523e16fab33e3bc3bae8a0e57b57cf7fcad85fb409cbac6baf",
"sha256:1c8e6444ca1cee9a60a1a35913b8409722f7474616e0e21004e4ffadba59964b",
"sha256:722dc0dcce5272f3c5c41609fdc2c8f0ee3f976550c2d2f2057e26ba760be9c0",
"sha256:86f61a1438c026c980a4c3e2dd88a5774a3a0f00d6d0954d6c5cf8d1921b804e",
"sha256:c4a2f42abee709ed97b4498c21aa608ac31fc1f7cc8aa60ebdcd3c80757a038d",
"sha256:d9cdc2e82aeb82200fff3640f375fac39d88b1bed27ce08377cd7fb0e3621cb7",
"sha256:da6676a484adec2fdd3e1ce1b70799881ffcb958e40208dd4c5beba0011f3589",
"sha256:dca71c08335fbfc6929438fe3a502f169ba96dd20e50b3544053d6be5cb19d82"
],
"version": "==5.4.8"
"version": "==5.6.0"
},
"pybgpranking": {
"editable": true,
"git": "https://github.com/D4-project/BGP-Ranking.git/",
"ref": "7e698f87366e6f99b4d0d11852737db28e3ddc62",
"ref": "37c97ae252ec4bf1d67733a49d4895c8cb009cf9",
"subdirectory": "client"
},
"pydnstrails": {
@ -333,12 +341,12 @@
"pymisp": {
"editable": true,
"git": "https://github.com/MISP/PyMISP.git",
"ref": "d4934cdf5f537c9f42ae37be7878de1848961de0"
"ref": "b8759673b91e733c307698abdc0d5ed82fd7e0de"
},
"pyonyphe": {
"editable": true,
"git": "https://github.com/sebdraven/pyonyphe",
"ref": "66329baeee7cab844f2203c047c2551828eaf14d"
"ref": "cbb0168d5cb28a9f71f7ab3773164a7039ccdb12"
},
"pyparsing": {
"hashes": [
@ -361,6 +369,12 @@
"index": "pypi",
"version": "==2.1"
},
"pyrsistent": {
"hashes": [
"sha256:3ca82748918eb65e2d89f222b702277099aca77e34843c5eb9d52451173970e2"
],
"version": "==0.14.11"
},
"pytesseract": {
"hashes": [
"sha256:11c20321595b6e2e904b594633edf1a717212b13bac7512986a2d807b8849770"
@ -370,10 +384,10 @@
},
"python-dateutil": {
"hashes": [
"sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
"sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
],
"version": "==2.7.5"
"version": "==2.8.0"
},
"pyyaml": {
"hashes": [
@ -400,10 +414,44 @@
},
"redis": {
"hashes": [
"sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf",
"sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a"
"sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54",
"sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775"
],
"version": "==3.0.1"
"version": "==3.2.0"
},
"reportlab": {
"hashes": [
"sha256:069f684cd0aaa518a27dc9124aed29cee8998e21ddf19604e53214ec8462bdd7",
"sha256:09b68ec01d86b4b120456b3f3202570ec96f57624e3a4fc36f3829323391daa4",
"sha256:0c32be9a406172c29ea20ff55a709ccac1e7fb09f15aba67cb7b455fd1d3dbe0",
"sha256:233196cf25e97cfe7c452524ea29d9a4909f1cb66599299233be1efaaaa7a7a3",
"sha256:2b5e4533f3e5b962835a5ce44467e66d1ecc822761d1b508077b5087a06be338",
"sha256:2e860bcdace5a558356802a92ae8658d7e5fdaa00ded82e83a3f2987c562cb66",
"sha256:3546029e63a9a9dc24ee38959eb417678c2425b96cd27b31e09e216dafc94666",
"sha256:4452b93f9c73b6b70311e7d69082d64da81b38e91bfb4766397630092e6da6fd",
"sha256:528c74a1c6527d1859c2c7a64a94a1cba485b00175162ea23699ae58a1e94939",
"sha256:6116e750f98018febc08dfee6df20446cf954adbcfa378d2c703d56c8864aff3",
"sha256:6b2b3580c647d75ef129172cb3da648cdb24566987b0b59c5ebb80ab770748d6",
"sha256:727b5f2bed08552d143fc99649b1863c773729f580a416844f9d9967bb0a1ae8",
"sha256:74c24a3ec0a3d4f8acb13a07192f45bdb54a1cc3c2286241677e7e8bcd5011fa",
"sha256:98ccd2f8b4f8636db05f3f14db0b471ad6bb4b66ae0dc9052c4822b3bd5d6a7d",
"sha256:a5905aa567946bc938b489a7249c7890c3fd3c9b7b5680dece5bc551c2ddbe0d",
"sha256:acbb7f676b8586b770719e9683eda951fdb38eb7970d46fcbf3cdda88d912a64",
"sha256:b5e30f865add48cf880f1c363eb505b97f2f7baaa88c155f87a335a76515a3e5",
"sha256:be2a7c33a2c28bbd3f453ffe4f0e5200b88c803a097f4cf52d69c6b53fad7a8f",
"sha256:c356bb600f59ac64955813d6497a08bfd5d0c451cb5829b61e3913d0ac084e26",
"sha256:c7ec4ae2393beab584921b1287a04e94fd98c28315e348362d89b85f4b464546",
"sha256:d476edc831bb3e9ebd04d1403abaf3ea57b3e4c2276c91a54fdfb6efbd3f9d97",
"sha256:db059e1a0691c872784062421ec51848539eb4f5210142682e61059a5ca7cc55",
"sha256:dd423a6753509ab14a0ac1b5be39d219c8f8d3781cce3deb4f45eda31969b5e8",
"sha256:ed9b7c0d71ce6fe2b31c6cde530ad8238632b876a5d599218739bda142a77f7c",
"sha256:f0a2465af4006f97b05e1f1546d67d3a3213d414894bf28be7f87f550a7f4a55",
"sha256:f20bfe26e57e8e1f575a9e0325be04dd3562db9f247ffdd73b5d4df6dec53bc2",
"sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c",
"sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9"
],
"index": "pypi",
"version": "==3.5.13"
},
"requests": {
"hashes": [
@ -422,17 +470,17 @@
},
"shodan": {
"hashes": [
"sha256:c40abb6ff2fd66bdee9f773746fb961eefdfaa8e720a07cb12fb70def136268d"
"sha256:f93b7199e89eecf5c84647f66316c2c044c3aebfc1fe4d9caa43dfda07f74c4e"
],
"index": "pypi",
"version": "==1.10.4"
"version": "==1.11.1"
},
"sigmatools": {
"hashes": [
"sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b"
"sha256:3bdbd2ee99c32f245e948d6b882219729ab379685dd7366e4d6149c390e08170"
],
"index": "pypi",
"version": "==0.7.1"
"version": "==0.9"
},
"six": {
"hashes": [
@ -443,10 +491,10 @@
},
"soupsieve": {
"hashes": [
"sha256:10687fc53eeb3518e01a0ac84d3d711da623d3298a3039459d3f649927c4a270",
"sha256:b23a0d7da0247200fe83c67c34de9d7599ad404106367313d8e65e04174d0b4b"
"sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b",
"sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26"
],
"version": "==1.7.2"
"version": "==1.8"
},
"sparqlwrapper": {
"hashes": [
@ -464,17 +512,23 @@
"index": "pypi",
"version": "==1.1.0"
},
"tabulate": {
"hashes": [
"sha256:8af07a39377cee1103a5c8b3330a421c2d99b9141e9cc5ddd2e3263fea416943"
],
"version": "==0.8.3"
},
"tornado": {
"hashes": [
"sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d",
"sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409",
"sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f",
"sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f",
"sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5",
"sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb",
"sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444"
"sha256:1a58f2d603476d5e462f7c28ca1dbb5ac7e51348b27a9cac849cdec3471101f8",
"sha256:33f93243cd46dd398e5d2bbdd75539564d1f13f25d704cfc7541db74066d6695",
"sha256:34e59401afcecf0381a28228daad8ed3275bcb726810654612d5e9c001f421b7",
"sha256:35817031611d2c296c69e5023ea1f9b5720be803e3bb119464bb2a0405d5cd70",
"sha256:666b335cef5cc2759c21b7394cff881f71559aaf7cb8c4458af5bb6cb7275b47",
"sha256:81203efb26debaaef7158187af45bc440796de9fb1df12a75b65fae11600a255",
"sha256:de274c65f45f6656c375cdf1759dbf0bc52902a1e999d12a35eb13020a641a53"
],
"version": "==5.1.1"
"version": "==6.0.1"
},
"url-normalize": {
"hashes": [
@ -500,32 +554,32 @@
"uwhois": {
"editable": true,
"git": "https://github.com/Rafiot/uwhoisd.git",
"ref": "f6f035e52213c8abc20f2084d28cfffb399457cb",
"ref": "411572840eba4c72dc321c549b36a54ed5cea9de",
"subdirectory": "client"
},
"vulners": {
"hashes": [
"sha256:8b468db8f8b0bad39ae51ebd4247f6ead90b6f53699e03b91ff9d63da70554d7",
"sha256:ad72378c842096cad9ebf83aa53d330117ece5d208ed7c419a21c70a8d5e2236",
"sha256:ffc92a099eeddea840fd199665992c0eb6d7ad69ac3a6730a286d00600bc5f2c"
"sha256:08a7ccb2b210d45143354c6161c73fe209dc14fae8692e8b793b36b79330ad11",
"sha256:bfe2478cc11c69ba7e436d7a5df925e227565782c0bd603929fb3d612c73d78d",
"sha256:d035f6a883625878a1dc377830d17d9702ef138ca31569ac01cb8686874f89cd"
],
"index": "pypi",
"version": "==1.3.6"
"version": "==1.4.5"
},
"wand": {
"hashes": [
"sha256:3e59e4bda9ef9d643d90e881cc950c8eee1508ec2cde1c150a1cbd5a12c1c007",
"sha256:52763dbf65d00cf98d7bc910b49329eea15896249c5555d47e169f2b6efbe166"
"sha256:7d6b8dc9d4eaccc430b9c86e6b749013220c994970a3f39e902b397e2fa732c3",
"sha256:cc0b5c9cd50fecd10dc8888b739dd5984c6f8085d2954f34903b83ca39a91236"
],
"index": "pypi",
"version": "==0.5.0"
"version": "==0.5.1"
},
"xlsxwriter": {
"hashes": [
"sha256:7cc07619760641b67112dbe0df938399d4d915d9b9924bb58eb5c17384d29cc6",
"sha256:ae22658a0fc5b9e875fa97c213d1ffd617d86dc49bf08be99ebdac814db7bf36"
"sha256:de9ef46088489915eaaee00c7088cff93cf613e9990b46b933c98eb46f21b47f",
"sha256:df96eafc3136d9e790e35d6725b473e46ada6f585c1f6519da69b27f5c8873f7"
],
"version": "==1.1.2"
"version": "==1.1.5"
},
"yara-python": {
"hashes": [
@ -564,24 +618,24 @@
"develop": {
"atomicwrites": {
"hashes": [
"sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
"sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
"sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
"sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
],
"version": "==1.2.1"
"version": "==1.3.0"
},
"attrs": {
"hashes": [
"sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
"sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
"sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
"sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
],
"version": "==18.2.0"
"version": "==19.1.0"
},
"certifi": {
"hashes": [
"sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
"sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
"sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
"sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
],
"version": "==2018.11.29"
"version": "==2019.3.9"
},
"chardet": {
"hashes": [
@ -600,47 +654,54 @@
},
"coverage": {
"hashes": [
"sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f",
"sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe",
"sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d",
"sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0",
"sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607",
"sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d",
"sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b",
"sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3",
"sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e",
"sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815",
"sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36",
"sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1",
"sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14",
"sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c",
"sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794",
"sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b",
"sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840",
"sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd",
"sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82",
"sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952",
"sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389",
"sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f",
"sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4",
"sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da",
"sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647",
"sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d",
"sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42",
"sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478",
"sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b",
"sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb",
"sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9"
"sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9",
"sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74",
"sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390",
"sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8",
"sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe",
"sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf",
"sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e",
"sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741",
"sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09",
"sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd",
"sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034",
"sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420",
"sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c",
"sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab",
"sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba",
"sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e",
"sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609",
"sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2",
"sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49",
"sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b",
"sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d",
"sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce",
"sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9",
"sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4",
"sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773",
"sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723",
"sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c",
"sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f",
"sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1",
"sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260",
"sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a"
],
"version": "==4.5.2"
"version": "==4.5.3"
},
"entrypoints": {
"hashes": [
"sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19",
"sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"
],
"version": "==0.3"
},
"flake8": {
"hashes": [
"sha256:6a35f5b8761f45c5513e3405f110a86bea57982c3b75b766ce7b65217abe1670",
"sha256:c01f8a3963b3571a8e6bd7a4063359aff90749e160778e03817cd9b71c9e07d2"
"sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661",
"sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8"
],
"index": "pypi",
"version": "==3.6.0"
"version": "==3.7.7"
},
"idna": {
"hashes": [
@ -658,11 +719,11 @@
},
"more-itertools": {
"hashes": [
"sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4",
"sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc",
"sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"
"sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40",
"sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1"
],
"version": "==5.0.0"
"markers": "python_version > '2.7'",
"version": "==6.0.0"
},
"nose": {
"hashes": [
@ -675,39 +736,39 @@
},
"pluggy": {
"hashes": [
"sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616",
"sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a"
"sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f",
"sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746"
],
"version": "==0.8.1"
"version": "==0.9.0"
},
"py": {
"hashes": [
"sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
"sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
"sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
"sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
],
"version": "==1.7.0"
"version": "==1.8.0"
},
"pycodestyle": {
"hashes": [
"sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
"sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
],
"version": "==2.4.0"
"version": "==2.5.0"
},
"pyflakes": {
"hashes": [
"sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49",
"sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae"
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
],
"version": "==2.0.0"
"version": "==2.1.1"
},
"pytest": {
"hashes": [
"sha256:41568ea7ecb4a68d7f63837cf65b92ce8d0105e43196ff2b26622995bb3dc4b2",
"sha256:c3c573a29d7c9547fb90217ece8a8843aa0c1328a797e200290dc3d0b4b823be"
"sha256:067a1d4bf827ffdd56ad21bd46674703fce77c5957f6c1eef731f6146bfcef1c",
"sha256:9687049d53695ad45cf5fdc7bbd51f0c49f1ea3ecfc4b7f3fde7501b541f17f4"
],
"index": "pypi",
"version": "==4.1.1"
"version": "==4.3.0"
},
"requests": {
"hashes": [

View File

@ -17,7 +17,9 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
### Expansion modules
* [Backscatter.io](misp_modules/modules/expansion/backscatter_io) - a hover and expansion module to expand an IP address with mass-scanning observations.
* [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking.
* [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.
@ -65,7 +67,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
* [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF).
* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html).
* [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event.
* [Simple PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export in PDF (required: asciidoctor-pdf).
* [PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF.
* [Nexthink query format](misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format.
* [osquery](misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format.
* [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format.
@ -85,20 +87,18 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
## How to install and start MISP modules in a Python virtualenv?
~~~~bash
sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv
sudo -u www-data virtualenv -p python3 /var/www/MISP/venv
cd /usr/local/src/
sudo git clone https://github.com/MISP/misp-modules.git
cd misp-modules
sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
sudo -u www-data /var/www/MISP/venv/bin/pip install .
sudo apt install ruby-pygments.rb -y
sudo gem install asciidoctor-pdf --pre
sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
~~~~
## How to install and start MISP modules?
## How to install and start MISP modules on Debian-based distributions ?
~~~~bash
sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
@ -107,12 +107,45 @@ sudo git clone https://github.com/MISP/misp-modules.git
cd misp-modules
sudo pip3 install -I -r REQUIREMENTS
sudo pip3 install -I .
sudo apt install ruby-pygments.rb -y
sudo gem install asciidoctor-pdf --pre
sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
~~~~
## How to install and start MISP modules on RHEL-based distributions ?
As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository.
~~~~bash
yum install rh-ruby22
cd /var/www/MISP
git clone https://github.com/MISP/misp-modules.git
cd misp-modules
scl enable rh-python36 python3 m pip install cryptography
scl enable rh-python36 python3 m pip install -I -r REQUIREMENTS
scl enable rh-python36 python3 m pip install I .
~~~~
Create the service file /etc/systemd/system/misp-workers.service :
~~~~
[Unit]
Description=MISP's modules
After=misp-workers.service
[Service]
Type=simple
User=apache
Group=apache
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 /opt/rh/rh-python36/root/bin/misp-modules l 127.0.0.1 s
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
~~~~
The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service.
Then, enable the misp-modules service and start it ;
~~~~bash
systemctl daemon-reload
systemctl enable --now misp-modules
~~~~
## How to add your own MISP modules?
Create your module in [misp_modules/modules/expansion/](misp_modules/modules/expansion/), [misp_modules/modules/export_mod/](misp_modules/modules/export_mod/), or [misp_modules/modules/import_mod/](misp_modules/modules/import_mod/). The module should have at minimum three functions:

View File

@ -1,19 +1,20 @@
-i https://pypi.org/simple
-e .
-e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client
-e git+https://github.com/D4-project/BGP-Ranking.git/@37c97ae252ec4bf1d67733a49d4895c8cb009cf9#egg=pybgpranking&subdirectory=client
-e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client
-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
-e git+https://github.com/MISP/PyMISP.git@d4934cdf5f537c9f42ae37be7878de1848961de0#egg=pymisp
-e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client
-e git+https://github.com/MISP/PyMISP.git@b8759673b91e733c307698abdc0d5ed82fd7e0de#egg=pymisp
-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
-e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe
-e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe
aiohttp==3.4.4
antlr4-python3-runtime==4.7.2 ; python_version >= '3'
async-timeout==3.0.1
attrs==18.2.0
attrs==19.1.0
backscatter==0.2.4
beautifulsoup4==4.7.1
blockchain==1.4.4
certifi==2018.11.29
certifi==2019.3.9
chardet==3.0.4
click-plugins==1.0.4
click==7.0
@ -23,41 +24,44 @@ domaintools-api==0.3.3
enum-compat==0.0.2
ez-setup==0.9
future==0.17.1
httplib2==0.12.0
httplib2==0.12.1
idna-ssl==1.1.0 ; python_version < '3.7'
idna==2.8
isodate==0.6.0
jsonschema==2.6.0
jsonschema==3.0.1
maclookup==1.0.3
multidict==4.5.2
oauth2==1.9.0.post1
passivetotal==1.0.30
pillow==5.4.1
psutil==5.4.8
psutil==5.6.0
pyeupi==1.0
pygeoip==0.3.2
pyparsing==2.3.1
pypdns==1.3
pypssl==2.1
pyrsistent==0.14.11
pytesseract==0.2.6
python-dateutil==2.7.5
python-dateutil==2.8.0
pyyaml==3.13
rdflib==4.2.2
redis==3.0.1
redis==3.2.0
reportlab==3.5.13
requests-cache==0.4.13
requests==2.21.0
shodan==1.10.4
sigmatools==0.7.1
shodan==1.11.1
sigmatools==0.9
six==1.12.0
soupsieve==1.7.2
soupsieve==1.8
sparqlwrapper==1.8.2
stix2-patterns==1.1.0
tornado==5.1.1
tabulate==0.8.3
tornado==6.0.1
url-normalize==1.4.1
urlarchiver==0.2
urllib3==1.24.1
vulners==1.3.6
wand==0.5.0
xlsxwriter==1.1.2
vulners==1.4.5
wand==0.5.1
xlsxwriter==1.1.5
yara-python==3.8.1
yarl==1.3.0

View File

@ -1 +0,0 @@
documentation.md

1281
doc/README.md Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,9 @@
{
"description": "Query backscatter.io (https://backscatter.io/).",
"requirements": ["backscatter python library"],
"features": "The module takes a source or destination IP address as input and displays the information known by backscatter.io.\n\n",
"logo": "logos/backscatter_io.png",
"references": ["https://pypi.org/project/backscatter/"],
"input": "IP addresses.",
"output": "Text containing a history of the IP addresses especially on scanning based on backscatter.io information ."
}

View File

@ -0,0 +1,9 @@
{
"description": "An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.",
"requirements": ["dnspython3: dns python library"],
"features": "The module queries a dns blacklist directly with the bitcoin address and get a response if the address has been abused.",
"logo": "logos/bitcoin.png",
"input": "btc address attribute.",
"output" : "Text to indicate if the BTC address has been abused.",
"references": ["https://btcblack.it/"]
}

View File

@ -1,7 +1,7 @@
{
"description": "Simple export of a MISP event to PDF.",
"requirements": ["PyMISP", "asciidoctor"],
"features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of asciidoctor, used to create the file, there is no special feature concerning the Event.",
"requirements": ["PyMISP", "reportlab"],
"features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option ",
"references": ["https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html"],
"input": "MISP Event",
"output": "MISP Event in a PDF file."

View File

@ -30,7 +30,7 @@ def generate_doc(root_path):
value = ', '.join(value) if isinstance(value, list) else '{}'.format(value.replace('\n', '\n>'))
markdown.append('- **{}**:\n>{}\n'.format(field, value))
markdown.append('\n-----\n')
with open('documentation.md', 'w') as w:
with open('README.md', 'w') as w:
w.write(''.join(markdown))

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

View File

@ -0,0 +1,14 @@
[Unit]
Description=System-wide instance of the MISP Modules
After=network.target
[Service]
User=www-data
Group=www-data
WorkingDirectory=/usr/local/src/misp-modules
Environment="PATH=/var/www/MISP/venv/bin"
ExecStart=/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s
[Install]
WantedBy=multi-user.target

View File

@ -8,4 +8,4 @@ __all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl',
'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator',
'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io',
'intel471']
'intel471', 'backscatter_io', 'btc_scam_check']

View File

@ -0,0 +1,74 @@
# -*- coding: utf-8 -*-
"""Backscatter.io Module."""
import json
try:
from backscatter import Backscatter
except ImportError:
print("Backscatter.io library not installed.")
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']}
moduleinfo = {'version': '1', 'author': 'brandon@backscatter.io',
'description': 'Backscatter.io module to bring mass-scanning observations into MISP.',
'module-type': ['expansion', 'hover']}
moduleconfig = ['api_key']
query_playbook = [
{'inputs': ['ip-src', 'ip-dst'],
'services': ['observations', 'enrichment'],
'name': 'generic'}
]
def check_query(request):
"""Check the incoming request for a valid configuration."""
output = {'success': False}
config = request.get('config', None)
if not config:
misperrors['error'] = "Configuration is missing from the request."
return output
for item in moduleconfig:
if config.get(item, None):
continue
misperrors['error'] = "Backscatter.io authentication is missing."
return output
if not request.get('ip-src') and request.get('ip-dst'):
misperrors['error'] = "Unsupported attributes type."
return output
profile = {'success': True, 'config': config, 'playbook': 'generic'}
if 'ip-src' in request:
profile.update({'value': request.get('ip-src')})
else:
profile.update({'value': request.get('ip-dst')})
return profile
def handler(q=False):
"""Handle gathering data."""
if not q:
return q
request = json.loads(q)
checks = check_query(request)
if not checks['success']:
return misperrors
try:
bs = Backscatter(checks['config']['api_key'])
response = bs.get_observations(query=checks['value'], query_type='ip')
if not response['success']:
misperrors['error'] = '%s: %s' % (response['error'], response['message'])
return misperrors
output = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]}
except Exception as e:
misperrors['error'] = str(e)
return misperrors
return output
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -0,0 +1,44 @@
import json
import sys
try:
from dns.resolver import Resolver, NXDOMAIN
from dns.name import LabelTooLong
resolver = Resolver()
resolver.timeout = 1
resolver.lifetime = 1
except ImportError:
sys.exit("dnspython3 in missing. use 'pip install dnspython3' to install it.")
misperrors = {'error': 'Error'}
mispattributes = {'input': ['btc'], 'output': ['text']}
moduleinfo = {'version': '0.1', 'author': 'Christian Studer',
'description': 'Checks if a BTC address has been abused.',
'module-type': ['hover']}
moduleconfig = []
url = 'bl.btcblack.it'
def handler(q=False):
if q is False:
return False
request = json.loads(q)
btc = request['btc']
query = f"{btc}.{url}"
try:
result = ' - '.join([str(r) for r in resolver.query(query, 'TXT')])[1:-1]
except NXDOMAIN:
result = f"{btc} is not known as a scam address."
except LabelTooLong:
result = f"{btc} is probably not a valid BTC address."
return {'results': [{'types': mispattributes['output'], 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -201,7 +201,7 @@ def handler(q=False):
value = float(tx['value'] / 100000000)
u, e = convert(value, transactions['time'])
mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0'))
#i += 1
# i += 1
i += 1
r = {

View File

@ -32,7 +32,7 @@ def handler(q=False):
res = x.query(toquery)
out = ''
for v in res:
out = out + "{} ".format(v['rdata'])
out = out + "{} ".format(v['rdata'])
r = {'results': [{'types': mispattributes['output'], 'values': out}]}
return r

View File

@ -1,103 +1,104 @@
import requests
import json
import sys
BASEurl = "https://api.xforce.ibmcloud.com/"
extensions = {"ip1": "ipr/%s",
"ip2": "ipr/malware/%s",
"url": "url/%s",
"hash": "malware/%s",
"vuln": "/vulnerabilities/search/%s",
"dns": "resolve/%s"}
sys.path.append('./')
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'vulnerability', 'md5', 'sha1', 'sha256'],
'output': ['ip-src', 'ip-dst', 'text', 'domain']}
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)',
'description': 'IBM X-Force Exchange expansion module',
'module-type': ['expansion', 'hover']}
# config fields that your code expects from the site admin
moduleconfig = ["apikey", "event_limit"]
limit = 5000 # Default
def MyHeader(key=False):
global limit
if key is False:
return None
return {"Authorization": "Basic %s " % key,
"Accept": "application/json",
'User-Agent': 'Mozilla 5.0'}
def handler(q=False):
global limit
if q is False:
return False
q = json.loads(q)
key = q["config"]["apikey"]
limit = int(q["config"].get("event_limit", 5))
r = {"results": []}
if "ip-src" in q:
r["results"] += apicall("dns", q["ip-src"], key)
if "ip-dst" in q:
r["results"] += apicall("dns", q["ip-dst"], key)
if "md5" in q:
r["results"] += apicall("hash", q["md5"], key)
if "sha1" in q:
r["results"] += apicall("hash", q["sha1"], key)
if "sha256" in q:
r["results"] += apicall("hash", q["sha256"], key)
if 'vulnerability' in q:
r["results"] += apicall("vuln", q["vulnerability"], key)
if "domain" in q:
r["results"] += apicall("dns", q["domain"], key)
uniq = []
for res in r["results"]:
if res not in uniq:
uniq.append(res)
r["results"] = uniq
return r
def apicall(indicator_type, indicator, key=False):
try:
myURL = BASEurl + (extensions[str(indicator_type)]) % indicator
jsondata = requests.get(myURL, headers=MyHeader(key)).json()
except Exception:
jsondata = None
redata = []
# print(jsondata)
if jsondata is not None:
if indicator_type is "hash":
if "malware" in jsondata:
lopointer = jsondata["malware"]
redata.append({"type": "text", "values": lopointer["risk"]})
if indicator_type is "dns":
if "records" in str(jsondata):
lopointer = jsondata["Passive"]["records"]
for dataset in lopointer:
redata.append({"type": "domain", "values": dataset["value"]})
return redata
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
import requests
import json
import sys
BASEurl = "https://api.xforce.ibmcloud.com/"
extensions = {"ip1": "ipr/%s",
"ip2": "ipr/malware/%s",
"url": "url/%s",
"hash": "malware/%s",
"vuln": "/vulnerabilities/search/%s",
"dns": "resolve/%s"}
sys.path.append('./')
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst', 'vulnerability', 'md5', 'sha1', 'sha256'],
'output': ['ip-src', 'ip-dst', 'text', 'domain']}
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)',
'description': 'IBM X-Force Exchange expansion module',
'module-type': ['expansion', 'hover']}
# config fields that your code expects from the site admin
moduleconfig = ["apikey", "event_limit"]
limit = 5000 # Default
def MyHeader(key=False):
global limit
if key is False:
return None
return {"Authorization": "Basic %s " % key,
"Accept": "application/json",
'User-Agent': 'Mozilla 5.0'}
def handler(q=False):
global limit
if q is False:
return False
q = json.loads(q)
key = q["config"]["apikey"]
limit = int(q["config"].get("event_limit", 5))
r = {"results": []}
if "ip-src" in q:
r["results"] += apicall("dns", q["ip-src"], key)
if "ip-dst" in q:
r["results"] += apicall("dns", q["ip-dst"], key)
if "md5" in q:
r["results"] += apicall("hash", q["md5"], key)
if "sha1" in q:
r["results"] += apicall("hash", q["sha1"], key)
if "sha256" in q:
r["results"] += apicall("hash", q["sha256"], key)
if 'vulnerability' in q:
r["results"] += apicall("vuln", q["vulnerability"], key)
if "domain" in q:
r["results"] += apicall("dns", q["domain"], key)
uniq = []
for res in r["results"]:
if res not in uniq:
uniq.append(res)
r["results"] = uniq
return r
def apicall(indicator_type, indicator, key=False):
try:
myURL = BASEurl + (extensions[str(indicator_type)]) % indicator
jsondata = requests.get(myURL, headers=MyHeader(key)).json()
except Exception:
jsondata = None
redata = []
# print(jsondata)
if jsondata is not None:
if indicator_type == "hash":
if "malware" in jsondata:
lopointer = jsondata["malware"]
redata.append({"type": "text", "values": lopointer["risk"]})
if indicator_type == "dns":
if "records" in str(jsondata):
lopointer = jsondata["Passive"]["records"]
for dataset in lopointer:
redata.append(
{"type": "domain", "values": dataset["value"]})
return redata
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -16,73 +16,73 @@ responseType = "application/json"
def handler(q=False):
if q is False:
return False
if q is False:
return False
request = json.loads(q)
request = json.loads(q)
config = {}
if "config" in request:
config = request["config"]
else:
config = {"indent_json_export": None}
config = {}
if "config" in request:
config = request["config"]
else:
config = {"indent_json_export": None}
if config['indent_json_export'] is not None:
try:
config['indent_json_export'] = int(config['indent_json_export'])
except Exception:
config['indent_json_export'] = None
if config['indent_json_export'] is not None:
try:
config['indent_json_export'] = int(config['indent_json_export'])
except Exception:
config['indent_json_export'] = None
if 'data' not in request:
return False
if 'data' not in request:
return False
# ~ Misp json structur
liteEvent = {'Event': {}}
# ~ Misp json structur
liteEvent = {'Event': {}}
for evt in request['data']:
rawEvent = evt['Event']
liteEvent['Event']['info'] = rawEvent['info']
liteEvent['Event']['Attribute'] = []
for evt in request['data']:
rawEvent = evt['Event']
liteEvent['Event']['info'] = rawEvent['info']
liteEvent['Event']['Attribute'] = []
attrs = evt['Attribute']
for attr in attrs:
if 'Internal reference' not in attr['category']:
liteAttr = {}
liteAttr['category'] = attr['category']
liteAttr['type'] = attr['type']
liteAttr['value'] = attr['value']
liteEvent['Event']['Attribute'].append(liteAttr)
attrs = evt['Attribute']
for attr in attrs:
if 'Internal reference' not in attr['category']:
liteAttr = {}
liteAttr['category'] = attr['category']
liteAttr['type'] = attr['type']
liteAttr['value'] = attr['value']
liteEvent['Event']['Attribute'].append(liteAttr)
return {'response': [],
'data': str(base64.b64encode(bytes(
json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), 'utf-8')}
return {'response': [],
'data': str(base64.b64encode(bytes(
json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), 'utf-8')}
def introspection():
modulesetup = {}
try:
responseType
modulesetup['responseType'] = responseType
except NameError:
pass
try:
userConfig
modulesetup['userConfig'] = userConfig
except NameError:
pass
try:
outputFileExtension
modulesetup['outputFileExtension'] = outputFileExtension
except NameError:
pass
try:
inputSource
modulesetup['inputSource'] = inputSource
except NameError:
pass
return modulesetup
modulesetup = {}
try:
responseType
modulesetup['responseType'] = responseType
except NameError:
pass
try:
userConfig
modulesetup['userConfig'] = userConfig
except NameError:
pass
try:
outputFileExtension
modulesetup['outputFileExtension'] = outputFileExtension
except NameError:
pass
try:
inputSource
modulesetup['inputSource'] = inputSource
except NameError:
pass
return modulesetup
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -86,7 +86,7 @@ def handler(q=False):
for event in request["data"]:
for attribute in event["Attribute"]:
if attribute['type'] in types_to_use:
output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n'
output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n'
r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')}
return r

View File

@ -80,7 +80,7 @@ def handler(q=False):
for event in request["data"]:
for attribute in event["Attribute"]:
if attribute['type'] in types_to_use:
output = output + handlers[attribute['type']](attribute['value']) + '\n'
output = output + handlers[attribute['type']](attribute['value']) + '\n'
r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')}
return r

View File

@ -1,67 +1,29 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from datetime import date
import json
import shlex
import subprocess
import base64
from pymisp import MISPEvent
from pymisp.tools import reportlab_generator
misperrors = {'error': 'Error'}
moduleinfo = {'version': '1',
'author': 'Raphaël Vinot',
moduleinfo = {'version': '2',
'author': 'Vincent Falconieri (prev. Raphaël Vinot)',
'description': 'Simple export to PDF',
'module-type': ['export'],
'require_standard_format': True}
moduleconfig = []
# config fields that your code expects from the site admin
moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"]
mispattributes = {}
outputFileExtension = "pdf"
responseType = "application/pdf"
types_to_attach = ['ip-dst', 'url', 'domain']
objects_to_attach = ['domain-ip']
headers = """
:toc: right
:toclevels: 1
:toc-title: Daily Report
:icons: font
:sectanchors:
:sectlinks:
= Daily report by {org_name}
{date}
:icons: font
"""
event_level_tags = """
IMPORTANT: This event is classified TLP:{value}.
{expanded}
"""
attributes = """
=== Indicator(s) of compromise
{list_attributes}
"""
title = """
== ({internal_id}) {title}
{summary}
"""
class ReportGenerator():
def __init__(self):
@ -79,60 +41,6 @@ class ReportGenerator():
self.misp_event = MISPEvent()
self.misp_event.load(event)
def attributes(self):
if not self.misp_event.attributes:
return ''
list_attributes = []
for attribute in self.misp_event.attributes:
if attribute.type in types_to_attach:
list_attributes.append("* {}".format(attribute.value))
for obj in self.misp_event.Object:
if obj.name in objects_to_attach:
for attribute in obj.Attribute:
if attribute.type in types_to_attach:
list_attributes.append("* {}".format(attribute.value))
return attributes.format(list_attributes="\n".join(list_attributes))
def _get_tag_info(self, machinetag):
return self.taxonomies.revert_machinetag(machinetag)
def report_headers(self):
content = {'org_name': 'name',
'date': date.today().isoformat()}
self.report += headers.format(**content)
def event_level_tags(self):
if not self.misp_event.Tag:
return ''
for tag in self.misp_event.Tag:
# Only look for TLP for now
if tag['name'].startswith('tlp'):
tax, predicate = self._get_tag_info(tag['name'])
return self.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded)
def title(self):
internal_id = ''
summary = ''
# Get internal refs for report
if not hasattr(self.misp_event, 'Object'):
return ''
for obj in self.misp_event.Object:
if obj.name != 'report':
continue
for a in obj.Attribute:
if a.object_relation == 'case-number':
internal_id = a.value
if a.object_relation == 'summary':
summary = a.value
return title.format(internal_id=internal_id, title=self.misp_event.info,
summary=summary)
def asciidoc(self, lang='en'):
self.report += self.title()
self.report += self.event_level_tags()
self.report += self.attributes()
def handler(q=False):
if q is False:
@ -143,45 +51,48 @@ def handler(q=False):
if 'data' not in request:
return False
for evt in request['data']:
report = ReportGenerator()
report.report_headers()
report.from_event(evt)
report.asciidoc()
config = {}
command_line = 'asciidoctor-pdf -'
args = shlex.split(command_line)
with subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) as process:
cmd_out, cmd_err = process.communicate(input=report.report.encode('utf-8'))
return {'response': [], 'data': str(base64.b64encode(cmd_out), 'utf-8')}
# Construct config object for reportlab_generator
for config_item in moduleconfig:
if (request.get('config')) and (request['config'].get(config_item) is not None):
config[config_item] = request['config'].get(config_item)
for evt in request['data']:
misp_event = MISPEvent()
misp_event.load(evt)
pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event, config))
return {'response': [], 'data': str(pdf, 'utf-8')}
def introspection():
modulesetup = {}
try:
responseType
modulesetup['responseType'] = responseType
except NameError:
pass
modulesetup = {}
try:
responseType
modulesetup['responseType'] = responseType
except NameError:
pass
try:
userConfig
modulesetup['userConfig'] = userConfig
except NameError:
pass
try:
outputFileExtension
modulesetup['outputFileExtension'] = outputFileExtension
except NameError:
pass
try:
inputSource
modulesetup['inputSource'] = inputSource
except NameError:
pass
return modulesetup
try:
userConfig
modulesetup['userConfig'] = userConfig
except NameError:
pass
try:
outputFileExtension
modulesetup['outputFileExtension'] = outputFileExtension
except NameError:
pass
try:
inputSource
modulesetup['inputSource'] = inputSource
except NameError:
pass
return modulesetup
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -63,7 +63,7 @@ def handler(q=False):
"comment": getattr(attrib, 'comment', '')}
# add tag
if q.get('config') and q['config'].get('default tag') is not None:
toAppend["tags"] = q['config']['default tag'].split(",")
toAppend["tags"] = q['config']['default tag'].split(",")
r["results"].append(toAppend)
return r

View File

@ -325,7 +325,7 @@ def process_analysis_json(analysis_json):
for stored_created_file in process['stored_files']['stored_created_file']:
stored_created_file['@filename'] = cleanup_filepath(stored_created_file['@filename'])
if stored_created_file['@filename']:
if stored_created_file['@filesize'] is not '0':
if stored_created_file['@filesize'] != '0':
val = '{}|{}'.format(stored_created_file['@filename'], stored_created_file['@md5'])
# print("stored_created_file filename|md5: {}|{} IDS:yes".format(
# stored_created_file['@filename'], # filename
@ -346,7 +346,7 @@ def process_analysis_json(analysis_json):
for stored_modified_file in process['stored_files']['stored_modified_file']:
stored_modified_file['@filename'] = cleanup_filepath(stored_modified_file['@filename'])
if stored_modified_file['@filename']:
if stored_modified_file['@filesize'] is not '0':
if stored_modified_file['@filesize'] != '0':
val = '{}|{}'.format(stored_modified_file['@filename'], stored_modified_file['@md5'])
# print("stored_modified_file MODIFY FILE: {}\t{}".format(
# stored_modified_file['@filename'], # filename

View File

@ -12,6 +12,7 @@ setup(
description='MISP modules are autonomous modules that can be used for expansion and other services in MISP',
packages=find_packages(),
entry_points={'console_scripts': ['misp-modules = misp_modules:main']},
scripts=['tools/update_misp_modules.sh'],
test_suite="tests",
classifiers=[
'License :: OSI Approved :: GNU Affero General Public License v3',

37
tools/update_misp_modules.sh Executable file
View File

@ -0,0 +1,37 @@
#!/usr/bin/env bash
set -e
set -x
# Updates the MISP Modules while respecting the current permissions
# It aims to support the two following installation methods:
# * Everything is runinng on the same machine following the MISP installation guide.
# * The modules are installed using pipenv on a different machine from the one where MISP is running.
if [ -d "/var/www/MISP" ] && [ -d "/usr/local/src/misp-modules" ]
then
echo "MISP is installed on the same machine, following the recommanded install script. Using MISP virtualenv."
PATH_TO_MISP="/var/www/MISP"
PATH_TO_MISP_MODULES="/usr/local/src/misp-modules"
pushd ${PATH_TO_MISP_MODULES}
USER=`stat -c "%U" .`
sudo -H -u ${USER} git pull
sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS
sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e .
service misp-modules restart
popd
else
if ! [ -x "$(command -v pipenv)" ]; then
echo 'Error: pipenv not available, unable to automatically update.' >&2
exit 1
fi
echo "Standalone mode, use pipenv from the current directory."
git pull
pipenv install
fi