diff --git a/Pipfile b/Pipfile index 1a99c42..9e651de 100644 --- a/Pipfile +++ b/Pipfile @@ -59,6 +59,7 @@ jbxapi = "*" geoip2 = "*" apiosintDS = "*" assemblyline_client = "*" +vt-graph-api = "*" [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index dab4860..b977ce7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "30e84f4986146c248e706f52f425649660225889bfcdf5075c99854442ae5f42" + "sha256": "b62db6df8a7b42f4c6915d6fbb1d4c38ccbb7209e559708433d28cdddebd3df9" }, "pipfile-spec": 6, "requires": { @@ -96,12 +96,12 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", - "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931", - "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" + "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a", + "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887", + "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae" ], "index": "pypi", - "version": "==4.8.1" + "version": "==4.8.2" }, "blockchain": { "hashes": [ @@ -264,20 +264,28 @@ ], "version": "==0.18.2" }, + "futures": { + "hashes": [ + "sha256:3a44f286998ae64f0cc083682fcfec16c406134a81a589a5de445d7bb7c2751b", + "sha256:51ecb45f0add83c806c68e4b06106f90db260585b25ef2abfcda0bd95c0132fd", + "sha256:c4884a65654a7c45435063e14ae85280eb1f111d94e542396717ba9828c4337f" + ], + "version": "==3.1.1" + }, "geoip2": { "hashes": [ - "sha256:a37ddac2d200ffb97c736da8b8ba9d5d8dc47da6ec0f162a461b681ecac53a14", - "sha256:f7ffe9d258e71a42cf622ce6350d976de1d0312b9f2fbce3975c7d838b57ecf0" + "sha256:5869e987bc54c0d707264fec4710661332cc38d2dca5a7f9bb5362d0308e2ce0", + "sha256:99ec12d2f1271a73a0a4a2b663fe6ce25fd02289c0a6bef05c0a1c3b30ee95a4" ], "index": "pypi", - "version": "==2.9.0" + "version": "==3.0.0" }, "httplib2": { "hashes": [ - "sha256:34537dcdd5e0f2386d29e0e2c6d4a1703a3b982d34c198a5102e6e5d6194b107", - "sha256:409fa5509298f739b34d5a652df762cb0042507dc93f6633e306b11289d6249d" + "sha256:1d1f4ad7a6e55d325830ab274190f98894e069850a871fac19921caf4363259d", + "sha256:a5f914f18f99cb9541660454a159e3b3c63241fc3ab60005bb88d97cc7a4fb58" ], - "version": "==0.14.0" + "version": "==0.15.0" }, "idna": { "hashes": [ @@ -384,9 +392,9 @@ }, "maxminddb": { "hashes": [ - "sha256:449a1713d37320d777d0db286286ab22890f0a176492ecf3ad8d9319108f2f79" + "sha256:d0ce131d901eb11669996b49a59f410efd3da2c6dbe2c0094fe2fef8d85b6336" ], - "version": "==1.5.1" + "version": "==1.5.2" }, "misp-modules": { "editable": true, @@ -401,25 +409,25 @@ }, "multidict": { "hashes": [ - "sha256:09c19f642e055550c9319d5123221b7e07fc79bda58122aa93910e52f2ab2f29", - "sha256:0c1a5d5f7aa7189f7b83c4411c2af8f1d38d69c4360d5de3eea129c65d8d7ce2", - "sha256:12f22980e7ed0972a969520fb1e55682c9fca89a68b21b49ec43132e680be812", - "sha256:258660e9d6b52de1a75097944e12718d3aa59adc611b703361e3577d69167aaf", - "sha256:3374a23e707848f27b3438500db0c69eca82929337656fce556bd70031fbda74", - "sha256:503b7fce0054c73aa631cc910a470052df33d599f3401f3b77e54d31182525d5", - "sha256:6ce55f2c45ffc90239aab625bb1b4864eef33f73ea88487ef968291fbf09fb3f", - "sha256:725496dde5730f4ad0a627e1a58e2620c1bde0ad1c8080aae15d583eb23344ce", - "sha256:a3721078beff247d0cd4fb19d915c2c25f90907cf8d6cd49d0413a24915577c6", - "sha256:ba566518550f81daca649eded8b5c7dd09210a854637c82351410aa15c49324a", - "sha256:c42362750a51a15dc905cb891658f822ee5021bfbea898c03aa1ed833e2248a5", - "sha256:cf14aaf2ab067ca10bca0b14d5cbd751dd249e65d371734bc0e47ddd8fafc175", - "sha256:cf24e15986762f0e75a622eb19cfe39a042e952b8afba3e7408835b9af2be4fb", - "sha256:d7b6da08538302c5245cd3103f333655ba7f274915f1f5121c4f4b5fbdb3febe", - "sha256:e27e13b9ff0a914a6b8fb7e4947d4ac6be8e4f61ede17edffabd088817df9e26", - "sha256:e53b205f8afd76fc6c942ef39e8ee7c519c775d336291d32874082a87802c67c", - "sha256:ec804fc5f68695d91c24d716020278fcffd50890492690a7e1fef2e741f7172c" + "sha256:0f04bf4c15d8417401a10a650c349ccc0285943681bfd87d3690587d7714a9b4", + "sha256:15a61c0df2d32487e06f6084eabb48fd9e8b848315e397781a70caf9670c9d78", + "sha256:3c5e2dcbe6b04cbb4303e47a896757a77b676c5e5db5528be7ff92f97ba7ab95", + "sha256:5d2b32b890d9e933d3ced417924261802a857abdee9507b68c75014482145c03", + "sha256:5e5fb8bfebf87f2e210306bf9dd8de2f1af6782b8b78e814060ae9254ab1f297", + "sha256:63ba2be08d82ea2aa8b0f7942a74af4908664d26cb4ff60c58eadb1e33e7da00", + "sha256:73740fcdb38f0adcec85e97db7557615b50ec4e5a3e73e35878720bcee963382", + "sha256:78bed18e7f1eb21f3d10ff3acde900b4d630098648fe1d65bb4abfb3e22c4900", + "sha256:a02fade7b5476c4f88efe9593ff2f3286698d8c6d715ba4f426954f73f382026", + "sha256:aacbde3a8875352a640efa2d1b96e5244a29b0f8df79cbf1ec6470e86fd84697", + "sha256:be813fb9e5ce41a5a99a29cdb857144a1bd6670883586f995b940a4878dc5238", + "sha256:bfcad6da0b8839f01a819602aaa5c5a5b4c85ecbfae9b261a31df3d9262fb31e", + "sha256:c2bfc0db3166e68515bc4a2b9164f4f75ae9c793e9635f8651f2c9ffc65c8dad", + "sha256:c66d11870ae066499a3541963e6ce18512ca827c2aaeaa2f4e37501cee39ac5d", + "sha256:cc7f2202b753f880c2e4123f9aacfdb94560ba893e692d24af271dac41f8b8d9", + "sha256:d1f45e5bb126662ba66ee579831ce8837b1fd978115c9657e32eb3c75b92973d", + "sha256:ed5f3378c102257df9e2dc9ce6468dabf68bee9ec34969cfdc472631aba00316" ], - "version": "==4.7.1" + "version": "==4.7.3" }, "np": { "hashes": [ @@ -430,29 +438,29 @@ }, "numpy": { "hashes": [ - "sha256:0a7a1dd123aecc9f0076934288ceed7fd9a81ba3919f11a855a7887cbe82a02f", - "sha256:0c0763787133dfeec19904c22c7e358b231c87ba3206b211652f8cbe1241deb6", - "sha256:3d52298d0be333583739f1aec9026f3b09fdfe3ddf7c7028cb16d9d2af1cca7e", - "sha256:43bb4b70585f1c2d153e45323a886839f98af8bfa810f7014b20be714c37c447", - "sha256:475963c5b9e116c38ad7347e154e5651d05a2286d86455671f5b1eebba5feb76", - "sha256:64874913367f18eb3013b16123c9fed113962e75d809fca5b78ebfbb73ed93ba", - "sha256:683828e50c339fc9e68720396f2de14253992c495fdddef77a1e17de55f1decc", - "sha256:6ca4000c4a6f95a78c33c7dadbb9495c10880be9c89316aa536eac359ab820ae", - "sha256:75fd817b7061f6378e4659dd792c84c0b60533e867f83e0d1e52d5d8e53df88c", - "sha256:7d81d784bdbed30137aca242ab307f3e65c8d93f4c7b7d8f322110b2e90177f9", - "sha256:8d0af8d3664f142414fd5b15cabfd3b6cc3ef242a3c7a7493257025be5a6955f", - "sha256:9679831005fb16c6df3dd35d17aa31dc0d4d7573d84f0b44cc481490a65c7725", - "sha256:a8f67ebfae9f575d85fa859b54d3bdecaeece74e3274b0b5c5f804d7ca789fe1", - "sha256:acbf5c52db4adb366c064d0b7c7899e3e778d89db585feadd23b06b587d64761", - "sha256:ada4805ed51f5bcaa3a06d3dd94939351869c095e30a2b54264f5a5004b52170", - "sha256:c7354e8f0eca5c110b7e978034cd86ed98a7a5ffcf69ca97535445a595e07b8e", - "sha256:e2e9d8c87120ba2c591f60e32736b82b67f72c37ba88a4c23c81b5b8fa49c018", - "sha256:e467c57121fe1b78a8f68dd9255fbb3bb3f4f7547c6b9e109f31d14569f490c3", - "sha256:ede47b98de79565fcd7f2decb475e2dcc85ee4097743e551fe26cfc7eb3ff143", - "sha256:f58913e9227400f1395c7b800503ebfdb0772f1c33ff8cb4d6451c06cabdf316", - "sha256:fe39f5fd4103ec4ca3cb8600b19216cd1ff316b4990f4c0b6057ad982c0a34d5" + "sha256:1786a08236f2c92ae0e70423c45e1e62788ed33028f94ca99c4df03f5be6b3c6", + "sha256:17aa7a81fe7599a10f2b7d95856dc5cf84a4eefa45bc96123cbbc3ebc568994e", + "sha256:20b26aaa5b3da029942cdcce719b363dbe58696ad182aff0e5dcb1687ec946dc", + "sha256:2d75908ab3ced4223ccba595b48e538afa5ecc37405923d1fea6906d7c3a50bc", + "sha256:39d2c685af15d3ce682c99ce5925cc66efc824652e10990d2462dfe9b8918c6a", + "sha256:56bc8ded6fcd9adea90f65377438f9fea8c05fcf7c5ba766bef258d0da1554aa", + "sha256:590355aeade1a2eaba17617c19edccb7db8d78760175256e3cf94590a1a964f3", + "sha256:70a840a26f4e61defa7bdf811d7498a284ced303dfbc35acb7be12a39b2aa121", + "sha256:77c3bfe65d8560487052ad55c6998a04b654c2fbc36d546aef2b2e511e760971", + "sha256:9537eecf179f566fd1c160a2e912ca0b8e02d773af0a7a1120ad4f7507cd0d26", + "sha256:9acdf933c1fd263c513a2df3dceecea6f3ff4419d80bf238510976bf9bcb26cd", + "sha256:ae0975f42ab1f28364dcda3dde3cf6c1ddab3e1d4b2909da0cb0191fa9ca0480", + "sha256:b3af02ecc999c8003e538e60c89a2b37646b39b688d4e44d7373e11c2debabec", + "sha256:b6ff59cee96b454516e47e7721098e6ceebef435e3e21ac2d6c3b8b02628eb77", + "sha256:b765ed3930b92812aa698a455847141869ef755a87e099fddd4ccf9d81fffb57", + "sha256:c98c5ffd7d41611407a1103ae11c8b634ad6a43606eca3e2a5a269e5d6e8eb07", + "sha256:cf7eb6b1025d3e169989416b1adcd676624c2dbed9e3bcb7137f51bfc8cc2572", + "sha256:d92350c22b150c1cae7ebb0ee8b5670cc84848f6359cf6b5d8f86617098a9b73", + "sha256:e422c3152921cece8b6a2fb6b0b4d73b6579bd20ae075e7d15143e711f3ca2ca", + "sha256:e840f552a509e3380b0f0ec977e8124d0dc34dc0e68289ca28f4d7c1d0d79474", + "sha256:f3d0a94ad151870978fb93538e95411c83899c9dc63e6fb65542f769568ecfa5" ], - "version": "==1.17.4" + "version": "==1.18.1" }, "oauth2": { "hashes": [ @@ -543,46 +551,38 @@ }, "pdftotext": { "hashes": [ - "sha256:c8bdc47b08baa17b8e03ba1f960fc6335b183d2644eaf7300e088516758a6090" + "sha256:b56f6ff1a564803ab8d849b3bb350b27087c15f5fe4e542a6370645543b0adf9" ], "index": "pypi", - "version": "==2.1.2" + "version": "==2.1.3" }, "pillow": { "hashes": [ - "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", - "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", - "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", - "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", - "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", - "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", - "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", - "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", - "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", - "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", - "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", - "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", - "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", - "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", - "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", - "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", - "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", - "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", - "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", - "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", - "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", - "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", - "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9", - "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1", - "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a", - "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96", - "sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132", - "sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a", - "sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5", - "sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0" + "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be", + "sha256:4d9ed9a64095e031435af120d3c910148067087541131e82b3e8db302f4c8946", + "sha256:54ebae163e8412aff0b9df1e88adab65788f5f5b58e625dc5c7f51eaf14a6837", + "sha256:5bfef0b1cdde9f33881c913af14e43db69815c7e8df429ceda4c70a5e529210f", + "sha256:5f3546ceb08089cedb9e8ff7e3f6a7042bb5b37c2a95d392fb027c3e53a2da00", + "sha256:5f7ae9126d16194f114435ebb79cc536b5682002a4fa57fa7bb2cbcde65f2f4d", + "sha256:62a889aeb0a79e50ecf5af272e9e3c164148f4bd9636cc6bcfa182a52c8b0533", + "sha256:7406f5a9b2fd966e79e6abdaf700585a4522e98d6559ce37fc52e5c955fade0a", + "sha256:8453f914f4e5a3d828281a6628cf517832abfa13ff50679a4848926dac7c0358", + "sha256:87269cc6ce1e3dee11f23fa515e4249ae678dbbe2704598a51cee76c52e19cda", + "sha256:875358310ed7abd5320f21dd97351d62de4929b0426cdb1eaa904b64ac36b435", + "sha256:8ac6ce7ff3892e5deaab7abaec763538ffd011f74dc1801d93d3c5fc541feee2", + "sha256:91b710e3353aea6fc758cdb7136d9bbdcb26b53cefe43e2cba953ac3ee1d3313", + "sha256:9d2ba4ed13af381233e2d810ff3bab84ef9f18430a9b336ab69eaf3cd24299ff", + "sha256:a62ec5e13e227399be73303ff301f2865bf68657d15ea50b038d25fc41097317", + "sha256:ab76e5580b0ed647a8d8d2d2daee170e8e9f8aad225ede314f684e297e3643c2", + "sha256:bf4003aa538af3f4205c5fac56eacaa67a6dd81e454ffd9e9f055fff9f1bc614", + "sha256:bf598d2e37cf8edb1a2f26ed3fb255191f5232badea4003c16301cb94ac5bdd0", + "sha256:c18f70dc27cc5d236f10e7834236aff60aadc71346a5bc1f4f83a4b3abee6386", + "sha256:c5ed816632204a2fc9486d784d8e0d0ae754347aba99c811458d69fcdfd2a2f9", + "sha256:dc058b7833184970d1248135b8b0ab702e6daa833be14035179f2acb78ff5636", + "sha256:ff3797f2f16bf9d17d53257612da84dd0758db33935777149b3334c01ff68865" ], "index": "pypi", - "version": "==6.2.1" + "version": "==7.0.0" }, "progressbar2": { "hashes": [ @@ -736,7 +736,7 @@ "fileobjects,openioc,virustotal,pdfexport" ], "git": "https://github.com/MISP/PyMISP.git", - "ref": "a26a8e450b14d48bb0c8ef46b32bff2f1eadc514" + "ref": "3ee7d8c67601bee658f1c0f488635796e5d7eb04" }, "pyonyphe": { "editable": true, @@ -752,10 +752,10 @@ }, "pyparsing": { "hashes": [ - "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", - "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.5" + "version": "==2.4.6" }, "pypdns": { "hashes": [ @@ -774,16 +774,16 @@ }, "pyrsistent": { "hashes": [ - "sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b" + "sha256:cdc7b5e3ed77bed61270a47d35434a30617b9becdf2478af76ad2c6ade307280" ], - "version": "==0.15.6" + "version": "==0.15.7" }, "pytesseract": { "hashes": [ - "sha256:ae1dce01413d1f8eb0614fd65d831e26e649dc1a31699b7275455c57aa563b59" + "sha256:03735b242439f8dbedc0f33ac9d0e980d755d19ed5e51dda1dcd866d9422edc8" ], "index": "pypi", - "version": "==0.3.0" + "version": "==0.3.1" }, "python-dateutil": { "hashes": [ @@ -829,19 +829,19 @@ }, "pyyaml": { "hashes": [ - "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", - "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", - "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", - "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", - "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", - "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", - "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", - "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", - "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", - "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", - "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" + "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", + "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", + "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", + "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", + "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", + "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", + "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", + "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", + "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", + "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", + "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" ], - "version": "==5.2" + "version": "==5.3" }, "pyzbar": { "hashes": [ @@ -928,10 +928,10 @@ }, "shodan": { "hashes": [ - "sha256:eab999bca9d3b30e6fc549e609194ff2d6fac3caea252414e1d8d735efab8342" + "sha256:ed3c38c749a5d77c935b226b6a7761e972269bd0d55c5c08526af73896aa6edd" ], "index": "pypi", - "version": "==1.21.0" + "version": "==1.21.2" }, "sigmatools": { "hashes": [ @@ -963,12 +963,12 @@ }, "sparqlwrapper": { "hashes": [ - "sha256:14ec551f0d60b4a496ffcc31f15337e844c085b8ead8cbe9a7178748a6de3794", - "sha256:21928e7a97f565e772cdeeb0abad428960f4307e3a13dbdd8f6d3da8a6a506c9", - "sha256:abc3e7eadcad32fa69a85c003853e2f6f73bda6cc999853838f401a5a1ea1109" + "sha256:357ee8a27bc910ea13d77836dbddd0b914991495b8cc1bf70676578155e962a8", + "sha256:c7f9c9d8ebb13428771bc3b6dee54197422507dcc3dea34e30d5dcfc53478dec", + "sha256:d6a66b5b8cda141660e07aeb00472db077a98d22cb588c973209c7336850fb3c" ], "index": "pypi", - "version": "==1.8.4" + "version": "==1.8.5" }, "stix2-patterns": { "hashes": [ @@ -1028,14 +1028,22 @@ ], "version": "==0.14.0" }, - "vulners": { + "vt-graph-api": { "hashes": [ - "sha256:245c07e49e55a604efde43cba723ac7b9345247e5ac8c4f998dcd36c05e4b1b9", - "sha256:82d47d7de208289a746bdb2dd9daf0fadf9fd290618015126091c7d9e2f8a96c", - "sha256:ef0c8e8c4e7d75fbd4d5bb1195109bd7a5b142f60dddc6cea77b3e20a3de1fa8" + "sha256:200c4f5a7c0a518502e890c4f4508a5ea042af9407d2889ef16a17ef11b7d25c", + "sha256:223c1cf32d69e10b5d3e178ec315589c7dfa7d43ccff6630a11ed5c5f498715c" ], "index": "pypi", - "version": "==1.5.4" + "version": "==1.0.1" + }, + "vulners": { + "hashes": [ + "sha256:00ff8744d07f398880afc1efcab6dac4abb614c84553fa31b2d439f986b8e0db", + "sha256:90a855915b4fb4dbd0325643d9e643602975fcb931162e5dc2e7778d1daa2fd8", + "sha256:f230bfcd42663326b7c9b8fa117752e26cad4ccca528caaab531c5b592af8cb5" + ], + "index": "pypi", + "version": "==1.5.5" }, "wand": { "hashes": [ @@ -1047,10 +1055,10 @@ }, "websocket-client": { "hashes": [ - "sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9", - "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" + "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549", + "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010" ], - "version": "==0.56.0" + "version": "==0.57.0" }, "wrapt": { "hashes": [ @@ -1068,10 +1076,10 @@ }, "xlsxwriter": { "hashes": [ - "sha256:027fa3d22ccfb5da5d77c29ed740aece286a9a6cc101b564f2f7ca11eb1d490b", - "sha256:5d480cee5babf3865227d5c81269d96be8e87914fc96403ca6fa1b1e4f64c080" + "sha256:18fe8f891a4adf7556c05d56059e136f9fbce5b19f9335f6d7b42c389c4592bc", + "sha256:5d3630ff9b2a277c939bd5053d0e7466499593abebbab9ce1dc9b1481a8ebbb6" ], - "version": "==1.2.6" + "version": "==1.2.7" }, "yara-python": { "hashes": [ @@ -1152,39 +1160,39 @@ }, "coverage": { "hashes": [ - "sha256:0cd13a6e98c37b510a2d34c8281d5e1a226aaf9b65b7d770ef03c63169965351", - "sha256:1a4b6b6a2a3a6612e6361130c2cc3dc4378d8c221752b96167ccbad94b47f3cd", - "sha256:2ee55e6dba516ddf6f484aa83ccabbb0adf45a18892204c23486938d12258cde", - "sha256:3be5338a2eb4ef03c57f20917e1d12a1fd10e3853fed060b6d6b677cb3745898", - "sha256:44b783b02db03c4777d8cf71bae19eadc171a6f2a96777d916b2c30a1eb3d070", - "sha256:475bf7c4252af0a56e1abba9606f1e54127cdf122063095c75ab04f6f99cf45e", - "sha256:47c81ee687eafc2f1db7f03fbe99aab81330565ebc62fb3b61edfc2216a550c8", - "sha256:4a7f8e72b18f2aca288ff02255ce32cc830bc04d993efbc87abf6beddc9e56c0", - "sha256:50197163a22fd17f79086e087a787883b3ec9280a509807daf158dfc2a7ded02", - "sha256:56b13000acf891f700f5067512b804d1ec8c301d627486c678b903859d07f798", - "sha256:79388ae29c896299b3567965dbcd93255f175c17c6c7bca38614d12718c47466", - "sha256:79fd5d3d62238c4f583b75d48d53cdae759fe04d4fb18fe8b371d88ad2b6f8be", - "sha256:7fe3e2fde2bf1d7ce25ebcd2d3de3650b8d60d9a73ce6dcef36e20191291613d", - "sha256:81042a24f67b96e4287774014fa27220d8a4d91af1043389e4d73892efc89ac6", - "sha256:81326f1095c53111f8afc95da281e1414185f4a538609a77ca50bdfa39a6c207", - "sha256:8873dc0d8f42142ea9f20c27bbdc485190fff93823c6795be661703369e5877d", - "sha256:88d2cbcb0a112f47eef71eb95460b6995da18e6f8ca50c264585abc2c473154b", - "sha256:91f2491aeab9599956c45a77c5666d323efdec790bfe23fcceafcd91105d585a", - "sha256:979daa8655ae5a51e8e7a24e7d34e250ae8309fd9719490df92cbb2fe2b0422b", - "sha256:9c871b006c878a890c6e44a5b2f3c6291335324b298c904dc0402ee92ee1f0be", - "sha256:a6d092545e5af53e960465f652e00efbf5357adad177b2630d63978d85e46a72", - "sha256:b5ed7837b923d1d71c4f587ae1539ccd96bfd6be9788f507dbe94dab5febbb5d", - "sha256:ba259f68250f16d2444cbbfaddaa0bb20e1560a4fdaad50bece25c199e6af864", - "sha256:be1d89614c6b6c36d7578496dc8625123bda2ff44f224cf8b1c45b810ee7383f", - "sha256:c1b030a79749aa8d1f1486885040114ee56933b15ccfc90049ba266e4aa2139f", - "sha256:c95bb147fab76f2ecde332d972d8f4138b8f2daee6c466af4ff3b4f29bd4c19e", - "sha256:d52c1c2d7e856cecc05aa0526453cb14574f821b7f413cc279b9514750d795c1", - "sha256:d609a6d564ad3d327e9509846c2c47f170456344521462b469e5cb39e48ba31c", - "sha256:e1bad043c12fb58e8c7d92b3d7f2f49977dcb80a08a6d1e7a5114a11bf819fca", - "sha256:e5a675f6829c53c87d79117a8eb656cc4a5f8918185a32fc93ba09778e90f6db", - "sha256:fec32646b98baf4a22fdceb08703965bd16dea09051fbeb31a04b5b6e72b846c" + "sha256:189aac76d6e0d7af15572c51892e7326ee451c076c5a50a9d266406cd6c49708", + "sha256:1bf7ba2af1d373a1750888724f84cffdfc697738f29a353c98195f98fc011509", + "sha256:1f4ee8e2e4243971618bc16fcc4478317405205f135e95226c2496e2a3b8dbbf", + "sha256:225e79a5d485bc1642cb7ba02281419c633c216cdc6b26c26494ba959f09e69f", + "sha256:23688ff75adfa8bfa2a67254d889f9bdf9302c27241d746e17547c42c732d3f4", + "sha256:28f7f73b34a05e23758e860a89a7f649b85c6749e252eff60ebb05532d180e86", + "sha256:2d0cb9b1fe6ad0d915d45ad3d87f03a38e979093a98597e755930db1f897afae", + "sha256:47874b4711c5aeb295c31b228a758ce3d096be83dc37bd56da48ed99efb8813b", + "sha256:511ec0c00840e12fb4e852e4db58fa6a01ca4da72f36a9766fae344c3d502033", + "sha256:53e7438fef0c97bc248f88ba1edd10268cd94d5609970aaf87abbe493691af87", + "sha256:569f9ee3025682afda6e9b0f5bb14897c0db03f1a1dc088b083dd36e743f92bb", + "sha256:593853aa1ac6dcc6405324d877544c596c9d948ef20d2e9512a0f5d2d3202356", + "sha256:5b0a07158360d22492f9abd02a0f2ee7981b33f0646bf796598b7673f6bbab14", + "sha256:7ca3db38a61f3655a2613ee2c190d63639215a7a736d3c64cc7bbdb002ce6310", + "sha256:7d1cc7acc9ce55179616cf72154f9e648136ea55987edf84addbcd9886ffeba2", + "sha256:88b51153657612aea68fa684a5b88037597925260392b7bb4509d4f9b0bdd889", + "sha256:955ec084f549128fa2702f0b2dc696392001d986b71acd8fd47424f28289a9c3", + "sha256:b251c7092cbb6d789d62dc9c9e7c4fb448c9138b51285c36aeb72462cad3600e", + "sha256:bd82b684bb498c60ef47bb1541a50e6d006dde8579934dcbdbc61d67d1ea70d9", + "sha256:bfe102659e2ec13b86c7f3b1db6c9a4e7beea4255058d006351339e6b342d5d2", + "sha256:c1e4e39e43057396a5e9d069bfbb6ffeee892e40c5d2effbd8cd71f34ee66c4d", + "sha256:cb2b74c123f65e8166f7e1265829a6c8ed755c3cd16d7f50e75a83456a5f3fd7", + "sha256:cca38ded59105f7705ef6ffe1e960b8db6c7d8279c1e71654a4775ab4454ca15", + "sha256:cf908840896f7aa62d0ec693beb53264b154f972eb8226fb864ac38975590c4f", + "sha256:d095a7b473f8a95f7efe821f92058c8a2ecfb18f8db6677ae3819e15dc11aaae", + "sha256:d22b4297e7e4225ccf01f1aa55e7a96412ea0796b532dd614c3fcbafa341128e", + "sha256:d4a2b578a7a70e0c71f662705262f87a456f1e6c1e40ada7ea699abaf070a76d", + "sha256:ddeb42a3d5419434742bf4cc71c9eaa22df3b76808e23a82bd0b0bd360f1a9f1", + "sha256:e65a5aa1670db6263f19fdc03daee1d7dbbadb5cb67fd0a1f16033659db13c1d", + "sha256:eaad65bd20955131bcdb3967a4dea66b4e4d4ca488efed7c00d91ee0173387e8", + "sha256:f45fba420b94165c17896861bb0e8b27fb7abdcedfeb154895d8553df90b7b00" ], - "version": "==5.0" + "version": "==5.0.2" }, "entrypoints": { "hashes": [ @@ -1241,10 +1249,10 @@ }, "packaging": { "hashes": [ - "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", - "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", + "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" ], - "version": "==19.2" + "version": "==20.0" }, "pluggy": { "hashes": [ @@ -1255,10 +1263,10 @@ }, "py": { "hashes": [ - "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", - "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", + "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" ], - "version": "==1.8.0" + "version": "==1.8.1" }, "pycodestyle": { "hashes": [ @@ -1276,10 +1284,10 @@ }, "pyparsing": { "hashes": [ - "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", - "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.5" + "version": "==2.4.6" }, "pytest": { "hashes": [ @@ -1316,10 +1324,10 @@ }, "wcwidth": { "hashes": [ - "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", - "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", + "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" ], - "version": "==0.1.7" + "version": "==0.1.8" }, "zipp": { "hashes": [ diff --git a/README.md b/README.md index d0296a8..fe37cd5 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). * [CVE advanced](misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). * [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. +* [Cytomic Orion](misp_modules/modules/expansion/cytomic_orion.py) - An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion. * [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. * [docx-enrich](misp_modules/modules/expansion/docx_enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). @@ -41,6 +42,8 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [GeoIP_City](misp_modules/modules/expansion/geoip_city.py) - a hover and expansion module to get GeoIP City information from geolite/maxmind. +* [GeoIP_ASN](misp_modules/modules/expansion/geoip_asn.py) - a hover and expansion module to get GeoIP ASN information from geolite/maxmind. * [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. * [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? @@ -69,6 +72,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. +* [SophosLabs Intelix](misp_modules/modules/expansion/sophoslabs_intelix.py) - SophosLabs Intelix is an API for Threat Intelligence and Analysis (free tier availible). [SophosLabs](https://aws.amazon.com/marketplace/pp/B07SLZPMCS) * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). @@ -89,27 +93,28 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj ### Export modules -* [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). -* [Cisco FireSight Manager ACL rule](misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL. -* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). -* [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. -* [PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. -* [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) module to export applicable attributes from an event to a mass EQL query. -* [Nexthink query format](misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format. -* [osquery](misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format. -* [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. -* [ThreatStream](misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format. +* [CEF](misp_modules/modules/export_mod/cef_export.py) - module to export Common Event Format (CEF). +* [Cisco FireSight Manager ACL rule](misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) - module to export as rule for the Cisco FireSight manager ACL. +* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) - module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). +* [Lite Export](misp_modules/modules/export_mod/liteexport.py) - module to export a lite event. +* [PDF export](misp_modules/modules/export_mod/pdfexport.py) - module to export an event in PDF. +* [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) - module to export applicable attributes from an event to a mass EQL query. +* [Nexthink query format](misp_modules/modules/export_mod/nexthinkexport.py) - module to export in Nexthink query format. +* [osquery](misp_modules/modules/export_mod/osqueryexport.py) - module to export in [osquery](https://osquery.io/) query format. +* [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) - module to export in ThreatConnect CSV format. +* [ThreatStream](misp_modules/modules/export_mod/threatStream_misp_export.py) - module to export in ThreatStream format. +* [VirusTotal Graph](misp_modules/modules/export_mod/vt_graph.py) - Module to create a VirusTotal graph out of an event. ### Import modules -* [CSV import](misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module. -* [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. -* [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. -* [GoAML import](misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. -* [Joe Sandbox import](misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report. -* [Lastline import](misp_modules/modules/import_mod/lastline_import.py) Module to import Lastline analysis reports. -* [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. -* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. +* [CSV import](misp_modules/modules/import_mod/csvimport.py) - Customizable CSV import module. +* [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) - Cuckoo JSON import. +* [Email Import](misp_modules/modules/import_mod/email_import.py) - Email import module for MISP to import basic metadata. +* [GoAML import](misp_modules/modules/import_mod/goamlimport.py) - Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. +* [Joe Sandbox import](misp_modules/modules/import_mod/joe_import.py) - Parse data from a Joe Sandbox json report. +* [Lastline import](misp_modules/modules/import_mod/lastline_import.py) - Module to import Lastline analysis reports. +* [OCR](misp_modules/modules/import_mod/ocr.py) - Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. +* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) - OpenIOC import based on PyMISP library. * [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. diff --git a/REQUIREMENTS b/REQUIREMENTS index ee6c7c1..40b4caf 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -106,3 +106,4 @@ xlsxwriter==1.2.6 yara-python==3.8.1 yarl==1.4.2 zipp==0.6.0 +vt-graph-api diff --git a/doc/README.md b/doc/README.md index 64df950..7e6bee3 100644 --- a/doc/README.md +++ b/doc/README.md @@ -532,11 +532,11 @@ Module to access intelmqs eventdb. Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History). - **features**: ->This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input. +>This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object. - **input**: >An IP address MISP attribute. - **output**: ->Text describing additional information about the input after a query on the IPASN-history database. +>Asn object(s) objects related to the IP address used as input. - **references**: >https://github.com/D4-project/IPASN-History - **requirements**: @@ -613,6 +613,7 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re Query Lastline with an analysis link and parse the report into MISP attributes and objects. The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module. - **features**: +>The module requires a Lastline Portal `username` and `password`. >The module uses the new format and it is able to return MISP attributes and objects. >The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module. - **input**: @@ -630,7 +631,7 @@ The analysis link can also be retrieved from the output of the [lastline_submit] Module to submit a file or URL to Lastline. - **features**: ->The module requires a Lastline API key and token (or username and password). +>The module requires a Lastline Analysis `api_token` and `key`. >When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module. - **input**: >File or URL to submit to Lastline. @@ -1586,6 +1587,26 @@ Module to export a structured CSV file for uploading to ThreatConnect. ----- +#### [vt_graph](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/vt_graph.py) + + + +This module is used to create a VirusTotal Graph from a MISP event. +- **features**: +>The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event. +> +>Once the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal. +- **input**: +>A MISP event. +- **output**: +>Link of the VirusTotal Graph created for the event. +- **references**: +>https://www.virustotal.com/gui/graph-overview +- **requirements**: +>vt_graph_api, the python library to query the VirusTotal graph API + +----- + ## Import Modules #### [csvimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) @@ -1681,6 +1702,7 @@ A module to import data from a Joe Sandbox analysis json report. Module to import and parse reports from Lastline analysis links. - **features**: +>The module requires a Lastline Portal `username` and `password`. >The module uses the new format and it is able to return MISP attributes and objects. >The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module. - **input**: diff --git a/doc/expansion/cytomic_orion.py b/doc/expansion/cytomic_orion.py new file mode 100644 index 0000000..6f87657 --- /dev/null +++ b/doc/expansion/cytomic_orion.py @@ -0,0 +1,9 @@ +{ + "description": "An expansion module to enrich attributes in MISP by quering the Cytomic Orion API", + "logo": "logos/cytomic_orion.png", + "requirements": ["Access (license) to Cytomic Orion"], + "input": "MD5, hash of the sample / malware to search for.", + "output": "MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines.", + "references": ["https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/", "https://www.cytomicmodel.com/solutions/"], + "features": "This module takes an MD5 hash and searches for occurrences of this hash in the Cytomic Orion database. Returns observed files and machines." +} diff --git a/doc/expansion/ipasn.json b/doc/expansion/ipasn.json index 68b10d1..8caed92 100644 --- a/doc/expansion/ipasn.json +++ b/doc/expansion/ipasn.json @@ -2,7 +2,7 @@ "description": "Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).", "requirements": ["pyipasnhistory: Python library to access IPASN-history instance"], "input": "An IP address MISP attribute.", - "output": "Text describing additional information about the input after a query on the IPASN-history database.", + "output": "Asn object(s) objects related to the IP address used as input.", "references": ["https://github.com/D4-project/IPASN-History"], - "features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input." + "features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object." } diff --git a/doc/expansion/lastline_query.json b/doc/expansion/lastline_query.json index 0d5da39..6165890 100644 --- a/doc/expansion/lastline_query.json +++ b/doc/expansion/lastline_query.json @@ -5,5 +5,5 @@ "input": "Link to a Lastline analysis.", "output": "MISP attributes and objects parsed from the analysis report.", "references": ["https://www.lastline.com"], - "features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." + "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." } diff --git a/doc/expansion/lastline_submit.json b/doc/expansion/lastline_submit.json index cf5ef57..d053f55 100644 --- a/doc/expansion/lastline_submit.json +++ b/doc/expansion/lastline_submit.json @@ -5,5 +5,5 @@ "input": "File or URL to submit to Lastline.", "output": "Link to the report generated by Lastline.", "references": ["https://www.lastline.com"], - "features": "The module requires a Lastline API key and token (or username and password).\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." + "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." } diff --git a/doc/export_mod/vt_graph.json b/doc/export_mod/vt_graph.json new file mode 100644 index 0000000..e317730 --- /dev/null +++ b/doc/export_mod/vt_graph.json @@ -0,0 +1,9 @@ +{ + "description": "This module is used to create a VirusTotal Graph from a MISP event.", + "logo": "logos/virustotal.png", + "requirements": ["vt_graph_api, the python library to query the VirusTotal graph API"], + "features": "The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event.\n\nOnce the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal.", + "references": ["https://www.virustotal.com/gui/graph-overview"], + "input": "A MISP event.", + "output": "Link of the VirusTotal Graph created for the event." +} diff --git a/doc/import_mod/lastline_import.json b/doc/import_mod/lastline_import.json index 1d4c15d..99414e0 100644 --- a/doc/import_mod/lastline_import.json +++ b/doc/import_mod/lastline_import.json @@ -5,5 +5,5 @@ "input": "Link to a Lastline analysis.", "output": "MISP attributes and objects parsed from the analysis report.", "references": ["https://www.lastline.com"], - "features": "The module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." + "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." } diff --git a/doc/logos/apivoid.png b/doc/logos/apivoid.png new file mode 100644 index 0000000..e4f84a7 Binary files /dev/null and b/doc/logos/apivoid.png differ diff --git a/doc/logos/cytomic_orion.png b/doc/logos/cytomic_orion.png new file mode 100644 index 0000000..45704e9 Binary files /dev/null and b/doc/logos/cytomic_orion.png differ diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py index 57a2505..c078cf7 100644 --- a/misp_modules/lib/__init__.py +++ b/misp_modules/lib/__init__.py @@ -1 +1,3 @@ +from .vt_graph_parser import * # noqa + all = ['joe_parser', 'lastline_api'] diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py index 00aa868..22a4918 100644 --- a/misp_modules/lib/joe_parser.py +++ b/misp_modules/lib/joe_parser.py @@ -51,12 +51,15 @@ signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'), class JoeParser(): - def __init__(self): + def __init__(self, config): self.misp_event = MISPEvent() self.references = defaultdict(list) self.attributes = defaultdict(lambda: defaultdict(set)) self.process_references = {} + self.import_pe = config["import_pe"] + self.create_mitre_attack = config["mitre_attack"] + def parse_data(self, data): self.data = data if self.analysis_type() == "file": @@ -72,7 +75,9 @@ class JoeParser(): if self.attributes: self.handle_attributes() - self.parse_mitre_attack() + + if self.create_mitre_attack: + self.parse_mitre_attack() def build_references(self): for misp_object in self.misp_event.objects: @@ -97,12 +102,12 @@ class JoeParser(): file_object = MISPObject('file') for key, mapping in dropped_file_mapping.items(): attribute_type, object_relation = mapping - file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key]}) + file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key], 'to_ids': False}) if droppedfile['@malicious'] == 'true': - file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious'}) + file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious', 'to_ids': False}) for h in droppedfile['value']: hash_type = dropped_hash_mapping[h['@algo']] - file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$']}) + file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$'], 'to_ids': False}) self.misp_event.add_object(**file_object) self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({ 'referenced_uuid': file_object.uuid, @@ -132,9 +137,12 @@ class JoeParser(): for object_relation, attribute in attributes.items(): network_connection_object.add_attribute(object_relation, **attribute) network_connection_object.add_attribute('first-packet-seen', - **{'type': 'datetime', 'value': min(tuple(min(timestamp) for timestamp in data.values()))}) + **{'type': 'datetime', + 'value': min(tuple(min(timestamp) for timestamp in data.values())), + 'to_ids': False}) for protocol in data.keys(): - network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol}) + network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), + **{'type': 'text', 'value': protocol, 'to_ids': False}) self.misp_event.add_object(**network_connection_object) self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, relationship_type='initiates')) @@ -143,8 +151,8 @@ class JoeParser(): network_connection_object = MISPObject('network-connection') for object_relation, attribute in attributes.items(): network_connection_object.add_attribute(object_relation, **attribute) - network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps)}) - network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol}) + network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps), 'to_ids': False}) + network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol, 'to_ids': False}) self.misp_event.add_object(**network_connection_object) self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, relationship_type='initiates')) @@ -154,7 +162,8 @@ class JoeParser(): if screenshotdata: screenshotdata = screenshotdata['interesting']['$'] attribute = {'type': 'attachment', 'value': 'screenshot.jpg', - 'data': screenshotdata, 'disable_correlation': True} + 'data': screenshotdata, 'disable_correlation': True, + 'to_ids': False} self.misp_event.add_attribute(**attribute) def parse_system_behavior(self): @@ -166,9 +175,9 @@ class JoeParser(): general = process['general'] process_object = MISPObject('process') for feature, relation in process_object_fields.items(): - process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature]}) + process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature], 'to_ids': False}) start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S') - process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time}) + process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time, 'to_ids': False}) self.misp_event.add_object(**process_object) for field, to_call in process_activities.items(): if process.get(field): @@ -203,7 +212,7 @@ class JoeParser(): url_object = MISPObject("url") self.analysisinfo_uuid = url_object.uuid - url_object.add_attribute("url", generalinfo["target"]["url"]) + url_object.add_attribute("url", generalinfo["target"]["url"], to_ids=False) self.misp_event.add_object(**url_object) def parse_fileinfo(self): @@ -213,10 +222,10 @@ class JoeParser(): self.analysisinfo_uuid = file_object.uuid for field in file_object_fields: - file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field]}) + file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field], 'to_ids': False}) for field, mapping in file_object_mapping.items(): attribute_type, object_relation = mapping - file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field]}) + file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field], 'to_ids': False}) arch = self.data['generalinfo']['arch'] if arch in arch_type_mapping: to_call = arch_type_mapping[arch] @@ -234,9 +243,9 @@ class JoeParser(): attribute_type = 'text' for comment, permissions in permission_lists.items(): permission_object = MISPObject('android-permission') - permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment)) + permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment, to_ids=False)) for permission in permissions: - permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission)) + permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission, to_ids=False)) self.misp_event.add_object(**permission_object) self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid, relationship_type='grants')) @@ -255,24 +264,24 @@ class JoeParser(): if elf.get('type'): # Haven't seen anything but EXEC yet in the files I tested attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type'] - elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value)) + elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value, to_ids=False)) for feature, relation in elf_object_mapping.items(): if elf.get(feature): - elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature])) + elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature], to_ids=False)) sections_number = len(fileinfo['sections']['section']) - elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number}) + elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) self.misp_event.add_object(**elf_object) for section in fileinfo['sections']['section']: section_object = MISPObject('elf-section') for feature in ('name', 'type'): if section.get(feature): - section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature])) + section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature], to_ids=False)) if section.get('size'): - section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16))) + section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16), to_ids=False)) for flag in section['flagsdesc']: try: attribute_value = elf_section_flags_mapping[flag] - section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value)) + section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value, to_ids=False)) except KeyError: print(f'Unknown elf section flag: {flag}') continue @@ -281,6 +290,8 @@ class JoeParser(): relationship_type=relationship)) def parse_pe(self, fileinfo, file_object): + if not self.import_pe: + return try: peinfo = fileinfo['pe'] except KeyError: @@ -292,8 +303,8 @@ class JoeParser(): self.misp_event.add_object(**file_object) for field, mapping in pe_object_fields.items(): attribute_type, object_relation = mapping - pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field]}) - pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16)}) + pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field], 'to_ids': False}) + pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16), 'to_ids': False}) program_name = fileinfo['filename'] if peinfo['versions']: for feature in peinfo['versions']['version']: @@ -301,18 +312,18 @@ class JoeParser(): if name == 'InternalName': program_name = feature['value'] if name in pe_object_mapping: - pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value']}) + pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value'], 'to_ids': False}) sections_number = len(peinfo['sections']['section']) - pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number}) + pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) signatureinfo = peinfo['signature'] if signatureinfo['signed']: signerinfo_object = MISPObject('authenticode-signerinfo') pe_object.add_reference(signerinfo_object.uuid, 'signed-by') self.misp_event.add_object(**pe_object) - signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name}) + signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name, 'to_ids': False}) for feature, mapping in signerinfo_object_mapping.items(): attribute_type, object_relation = mapping - signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature]}) + signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature], 'to_ids': False}) self.misp_event.add_object(**signerinfo_object) else: self.misp_event.add_object(**pe_object) @@ -327,7 +338,7 @@ class JoeParser(): for feature, mapping in pe_section_object_mapping.items(): if section.get(feature): attribute_type, object_relation = mapping - section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature]}) + section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature], 'to_ids': False}) return section_object def parse_network_interactions(self): @@ -339,13 +350,13 @@ class JoeParser(): for key, mapping in domain_object_mapping.items(): attribute_type, object_relation = mapping domain_object.add_attribute(object_relation, - **{'type': attribute_type, 'value': domain[key]}) + **{'type': attribute_type, 'value': domain[key], 'to_ids': False}) self.misp_event.add_object(**domain_object) reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts') self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) else: attribute = MISPAttribute() - attribute.from_dict(**{'type': 'domain', 'value': domain['@name']}) + attribute.from_dict(**{'type': 'domain', 'value': domain['@name'], 'to_ids': False}) self.misp_event.add_attribute(**attribute) reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts') self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) @@ -353,7 +364,7 @@ class JoeParser(): if ipinfo: for ip in ipinfo['ip']: attribute = MISPAttribute() - attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip']}) + attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip'], 'to_ids': False}) self.misp_event.add_attribute(**attribute) reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts') self.add_process_reference(ip['@targetid'], ip['@currentpath'], reference) @@ -363,7 +374,7 @@ class JoeParser(): target_id = int(url['@targetid']) current_path = url['@currentpath'] attribute = MISPAttribute() - attribute_dict = {'type': 'url', 'value': url['@name']} + attribute_dict = {'type': 'url', 'value': url['@name'], 'to_ids': False} if target_id != -1 and current_path != 'unknown': self.references[self.process_references[(target_id, current_path)]].append({ 'referenced_uuid': attribute.uuid, @@ -384,8 +395,8 @@ class JoeParser(): registry_key = MISPObject('registry-key') for field, mapping in regkey_object_mapping.items(): attribute_type, object_relation = mapping - registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field]}) - registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper())}) + registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field], 'to_ids': False}) + registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper()), 'to_ids': False}) self.misp_event.add_object(**registry_key) self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid, relationship_type=relationship)) @@ -398,7 +409,7 @@ class JoeParser(): def create_attribute(self, attribute_type, attribute_value): attribute = MISPAttribute() - attribute.from_dict(**{'type': attribute_type, 'value': attribute_value}) + attribute.from_dict(**{'type': attribute_type, 'value': attribute_value, 'to_ids': False}) self.misp_event.add_attribute(**attribute) return attribute.uuid @@ -419,5 +430,5 @@ class JoeParser(): attributes = {} for field, value in zip(network_behavior_fields, connection): attribute_type, object_relation = network_connection_object_mapping[field] - attributes[object_relation] = {'type': attribute_type, 'value': value} + attributes[object_relation] = {'type': attribute_type, 'value': value, 'to_ids': False} return attributes diff --git a/misp_modules/lib/lastline_api.py b/misp_modules/lib/lastline_api.py index a6912b8..83726ad 100644 --- a/misp_modules/lib/lastline_api.py +++ b/misp_modules/lib/lastline_api.py @@ -30,19 +30,21 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import ipaddress +import abc import logging +import io +import ipaddress +import pymisp import re import requests -import pymisp - from urllib import parse -DEFAULT_LASTLINE_API = "https://user.lastline.com/papi" +DEFAULT_LL_PORTAL_API_URL = "https://user.lastline.com/papi" +DEFAULT_LL_ANALYSIS_API_URL = "https://analysis.lastline.com" -HOSTED_LASTLINE_DOMAINS = frozenset([ +LL_HOSTED_DOMAINS = frozenset([ "user.lastline.com", "user.emea.lastline.com", ]) @@ -53,61 +55,66 @@ def purge_none(d): return {k: v for k, v in d.items() if v is not None} -def get_analysis_link(api_url, uuid): +def get_task_link(uuid, analysis_url=None, portal_url=None): """ - Get the analysis link of a task given the task uuid. + Get the task link given the task uuid and at least one API url. - :param str api_url: the URL :param str uuid: the task uuid + :param str|None analysis_url: the URL to the analysis API endpoint + :param str|None portal_url: the URL to the portal API endpoint :rtype: str - :return: the analysis link + :return: the task link + :raises ValueError: if not enough parameters have been provided """ + if not analysis_url and not portal_url: + raise ValueError("Neither analysis URL or portal URL have been specified") + if analysis_url: + portal_url = "{}/papi".format(analysis_url.replace("analysis.", "user.")) portal_url_path = "../portal#/analyst/task/{}/overview".format(uuid) - analysis_link = parse.urljoin(api_url, portal_url_path) - return analysis_link + return parse.urljoin(portal_url, portal_url_path) -def get_uuid_from_link(analysis_link): +def get_portal_url_from_task_link(task_link): """ - Return task uuid from link or raise ValueError exception. + Return the portal API url related to the provided task link. - :param str analysis_link: a link + :param str task_link: a link + :rtype: str + :return: the portal API url + """ + parsed_uri = parse.urlparse(task_link) + return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri) + + +def get_uuid_from_task_link(task_link): + """ + Return the uuid from a task link. + + :param str task_link: a link :rtype: str :return: the uuid :raises ValueError: if the link contains not task uuid """ try: - return re.findall("[a-fA-F0-9]{32}", analysis_link)[0] + return re.findall("[a-fA-F0-9]{32}", task_link)[0] except IndexError: raise ValueError("Link does not contain a valid task uuid") -def is_analysis_hosted(analysis_link): +def is_task_hosted(task_link): """ - Return whether the analysis link is pointing to a hosted submission. + Return whether the portal link is pointing to a hosted submission. - :param str analysis_link: a link + :param str task_link: a link :rtype: boolean - :return: whether the link is hosted + :return: whether the link points to a hosted analysis """ - for domain in HOSTED_LASTLINE_DOMAINS: - if domain in analysis_link: + for domain in LL_HOSTED_DOMAINS: + if domain in task_link: return True return False -def get_api_url_from_link(analysis_link): - """ - Return the API url related to the provided analysis link. - - :param str analysis_link: a link - :rtype: str - :return: the API url - """ - parsed_uri = parse.urlparse(analysis_link) - return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri) - - class InvalidArgument(Exception): """Error raised invalid.""" @@ -135,6 +142,572 @@ class ApiError(Error): return "{}{}".format(self.error_msg, error_code) +class LastlineAbstractClient(abc.ABC): + """"A very basic HTTP client providing basic functionality.""" + + __metaclass__ = abc.ABCMeta + + SUB_APIS = ('analysis', 'authentication', 'knowledgebase', 'login') + FORMATS = ["json", "xml"] + + @classmethod + def sanitize_login_params(cls, api_key, api_token, username, password): + """ + Return a dictionary with either API or USER credentials. + + :param str|None api_key: the API key + :param str|None api_token: the API token + :param str|None username: the username + :param str|None password: the password + :rtype: dict[str, str] + :return: the dictionary + :raises InvalidArgument: if too many values are invalid + """ + if api_key and api_token: + return { + "key": api_key, + "api_token": api_token, + } + elif username and password: + return { + "username": username, + "password": password, + } + else: + raise InvalidArgument("Arguments provided do not contain valid data") + + @classmethod + def get_login_params_from_dict(cls, d): + """ + Get the module configuration from a ConfigParser object. + + :param dict[str, str] d: the dictionary + :rtype: dict[str, str] + :return: the parsed configuration + """ + api_key = d.get("key") + api_token = d.get("api_token") + username = d.get("username") + password = d.get("password") + return cls.sanitize_login_params(api_key, api_token, username, password) + + @classmethod + def get_login_params_from_conf(cls, conf, section_name): + """ + Get the module configuration from a ConfigParser object. + + :param ConfigParser conf: the conf object + :param str section_name: the section name + :rtype: dict[str, str] + :return: the parsed configuration + """ + api_key = conf.get(section_name, "key", fallback=None) + api_token = conf.get(section_name, "api_token", fallback=None) + username = conf.get(section_name, "username", fallback=None) + password = conf.get(section_name, "password", fallback=None) + return cls.sanitize_login_params(api_key, api_token, username, password) + + @classmethod + def load_from_conf(cls, conf, section_name): + """ + Load client from a ConfigParser object. + + :param ConfigParser conf: the conf object + :param str section_name: the section name + :rtype: T <- LastlineAbstractClient + :return: the loaded client + """ + url = conf.get(section_name, "url") + return cls(url, cls.get_login_params_from_conf(conf, section_name)) + + def __init__(self, api_url, login_params, timeout=60, verify_ssl=True): + """ + Instantiate a Lastline mini client. + + :param str api_url: the URL of the API + :param dict[str, str]: the login parameters + :param int timeout: the timeout + :param boolean verify_ssl: whether to verify the SSL certificate + """ + self._url = api_url + self._login_params = login_params + self._timeout = timeout + self._verify_ssl = verify_ssl + self._session = None + self._logger = logging.getLogger(__name__) + + @abc.abstractmethod + def _login(self): + """Login using account-based or key-based methods.""" + + def _is_logged_in(self): + """Return whether we have an active session.""" + return self._session is not None + + @staticmethod + def _parse_response(response): + """ + Parse the response. + + :param requests.Response response: the response + :rtype: tuple(str|None, Error|ApiError) + :return: a tuple with mutually exclusive fields (either the response or the error) + """ + try: + ret = response.json() + if "success" not in ret: + return None, Error("no success field in response") + + if not ret["success"]: + error_msg = ret.get("error", "") + error_code = ret.get("error_code", None) + return None, ApiError(error_msg, error_code) + + if "data" not in ret: + return None, Error("no data field in response") + + return ret["data"], None + except ValueError as e: + return None, Error("Response not json {}".format(e)) + + def _handle_response(self, response, raw=False): + """ + Check a response for issues and parse the return. + + :param requests.Response response: the response + :param boolean raw: whether the raw body should be returned + :rtype: str + :return: if raw, return the response content; if not raw, the data field + :raises: CommunicationError, ApiError, Error + """ + # Check for HTTP errors, and re-raise in case + try: + response.raise_for_status() + except requests.RequestException as e: + _, err = self._parse_response(response) + if isinstance(err, ApiError): + err_msg = "{}: {}".format(e, err.error_msg) + else: + err_msg = "{}".format(e) + raise CommunicationError(err_msg) + + # Otherwise return the data (either parsed or not) but reraise if we have an API error + if raw: + return response.content + data, err = self._parse_response(response) + if err: + raise err + return data + + def _build_url(self, sub_api, parts, requested_format="json"): + if sub_api not in self.SUB_APIS: + raise InvalidArgument(sub_api) + if requested_format not in self.FORMATS: + raise InvalidArgument(requested_format) + num_parts = 2 + len(parts) + pattern = "/".join(["%s"] * num_parts) + ".%s" + params = [self._url, sub_api] + parts + [requested_format] + return pattern % tuple(params) + + def post(self, module, function, params=None, data=None, files=None, fmt="json"): + if isinstance(function, list): + functions = function + else: + functions = [function] if function else [] + url = self._build_url(module, functions, requested_format=fmt) + return self.do_request( + url=url, + method="POST", + params=params, + data=data, + files=files, + fmt=fmt, + ) + + def get(self, module, function, params=None, fmt="json"): + if isinstance(function, list): + functions = function + else: + functions = [function] if function else [] + url = self._build_url(module, functions, requested_format=fmt) + return self.do_request( + url=url, + method="GET", + params=params, + fmt=fmt, + ) + + def do_request( + self, + method, + url, + params=None, + data=None, + files=None, + fmt="json", + raw=False, + raw_response=False, + headers=None, + stream_response=False + ): + if raw_response: + raw = True + + if fmt: + fmt = fmt.lower().strip() + if fmt not in self.FORMATS: + raise InvalidArgument("Only json, xml, html and pdf supported") + elif not raw: + raise InvalidArgument("Unformatted response requires raw=True") + + if fmt != "json" and not raw: + raise InvalidArgument("Non-json format requires raw=True") + + if method not in ["POST", "GET"]: + raise InvalidArgument("Only POST and GET supported") + + if not self._is_logged_in(): + self._login() + + try: + try: + response = self._session.request( + method=method, + url=url, + data=data, + params=params, + files=files, + verify=self._verify_ssl, + timeout=self._timeout, + stream=stream_response, + headers=headers, + ) + except requests.RequestException as e: + raise CommunicationError(e) + + if raw_response: + return response + return self._handle_response(response, raw) + + except Error as e: + raise e + + except CommunicationError as e: + raise e + + +class AnalysisClient(LastlineAbstractClient): + + def _login(self): + """ + Creates auth session for malscape-service. + + Credentials are 'key' and 'api_token'. + """ + if self._session is None: + self._session = requests.session() + url = self._build_url("authentication", ["login"]) + self.do_request("POST", url, params=purge_none(self._login_params)) + + def get_progress(self, uuid): + """ + Get the completion progress of a given task. + :param str uuid: the unique identifier of the submitted task + :rtype: dict[str, int] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + url = self._build_url('analysis', ['get_progress']) + params = {'uuid': uuid} + return self.do_request("POST", url, params=params) + + def get_result(self, uuid): + """ + Get report results for a given task. + + :param str uuid: the unique identifier of the submitted task + :rtype: dict[str, any] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + # better: use 'get_results()' but that would break + # backwards-compatibility + url = self._build_url('analysis', ['get']) + params = {'uuid': uuid} + return self.do_request("GET", url, params=params) + + def submit_file( + self, + file_data, + file_name=None, + password=None, + analysis_env=None, + allow_network_traffic=True, + analysis_timeout=None, + bypass_cache=False, + ): + """ + Upload a file to be analyzed. + + :param bytes file_data: the data as a byte sequence + :param str|None file_name: if set, represents the name of the file to submit + :param str|None password: if set, use it to extract the sample + :param str|None analysis_env: if set, e.g windowsxp + :param boolean allow_network_traffic: if set to False, deny network connections + :param int|None analysis_timeout: if set limit the duration of the analysis + :param boolean bypass_cache: whether to re-process a file (requires special permissions) + :rtype: dict[str, any] + :return: a dictionary in the following form if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + file_stream = io.BytesIO(file_data) + api_url = self._build_url("analysis", ["submit", "file"]) + params = purge_none({ + "bypass_cache": bypass_cache and 1 or None, + "analysis_timeout": analysis_timeout, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic and 1 or None, + "filename": file_name, + "password": password, + "full_report_score": -1, + }) + + files = purge_none({ + # If an explicit filename was provided, we can pass it down to + # python-requests to use it in the multipart/form-data. This avoids + # having python-requests trying to guess the filename based on stream + # attributes. + # + # The problem with this is that, if the filename is not ASCII, then + # this triggers a bug in flask/werkzeug which means the file is + # thrown away. Thus, we just force an ASCII name + "file": ('dummy-ascii-name-for-file-param', file_stream), + }) + + return self.do_request("POST", api_url, params=params, files=files) + + def submit_url( + self, + url, + referer=None, + user_agent=None, + bypass_cache=False, + ): + """ + Upload an URL to be analyzed. + + :param str url: the url to analyze + :param str|None referer: the referer + :param str|None user_agent: the user agent + :param boolean bypass_cache: bypass_cache + :rtype: dict[str, any] + :return: a dictionary like the following if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + api_url = self._build_url("analysis", ["submit", "url"]) + params = purge_none({ + "url": url, + "referer": referer, + "bypass_cache": bypass_cache and 1 or None, + "user_agent": user_agent or None, + }) + return self.do_request("POST", api_url, params=params) + + +class PortalClient(LastlineAbstractClient): + + def _login(self): + """ + Login using account-based or key-based methods. + + Credentials are 'username' and 'password' + """ + if self._session is None: + self._session = requests.session() + self.post("login", function=None, data=self._login_params) + + def get_progress(self, uuid, analysis_instance=None): + """ + Get the completion progress of a given task. + + :param str uuid: the unique identifier of the submitted task + :param str analysis_instance: if set, defines the analysis instance to query + :rtype: dict[str, int] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance}) + return self.get("analysis", "get_progress", params=params) + + def get_result(self, uuid, analysis_instance=None): + """ + Get report results for a given task. + + :param str uuid: the unique identifier of the submitted task + :param str analysis_instance: if set, defines the analysis instance to query + :rtype: dict[str, any] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + params = purge_none( + { + "uuid": uuid, + "analysis_instance": analysis_instance, + "report_format": "json", + } + ) + return self.get("analysis", "get_result", params=params) + + def submit_url( + self, + url, + referer=None, + user_agent=None, + bypass_cache=False, + ): + """ + Upload an URL to be analyzed. + + :param str url: the url to analyze + :param str|None referer: the referer + :param str|None user_agent: the user agent + :param boolean bypass_cache: bypass_cache + :rtype: dict[str, any] + :return: a dictionary like the following if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + params = purge_none( + { + "url": url, + "bypass_cache": bypass_cache, + "referer": referer, + "user_agent": user_agent + } + ) + return self.post("analysis", "submit_url", params=params) + + def submit_file( + self, + file_data, + file_name=None, + password=None, + analysis_env=None, + allow_network_traffic=True, + analysis_timeout=None, + bypass_cache=False, + ): + """ + Upload a file to be analyzed. + + :param bytes file_data: the data as a byte sequence + :param str|None file_name: if set, represents the name of the file to submit + :param str|None password: if set, use it to extract the sample + :param str|None analysis_env: if set, e.g windowsxp + :param boolean allow_network_traffic: if set to False, deny network connections + :param int|None analysis_timeout: if set limit the duration of the analysis + :param boolean bypass_cache: whether to re-process a file (requires special permissions) + :rtype: dict[str, any] + :return: a dictionary in the following form if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + params = purge_none( + { + "filename": file_name, + "password": password, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic, + "analysis_timeout": analysis_timeout, + "bypass_cache": bypass_cache, + } + ) + files = {"file": (file_name, file_data, "application/octet-stream")} + return self.post("analysis", "submit_file", params=params, files=files) + + class LastlineResultBaseParser(object): """ This is a parser to extract *basic* information from a Lastline result dictionary. @@ -247,7 +820,7 @@ class LastlineResultBaseParser(object): # Add sandbox info like score and sandbox type o = pymisp.MISPObject(name="sandbox-report") - sandbox_type = "saas" if is_analysis_hosted(analysis_link) else "on-premise" + sandbox_type = "saas" if is_task_hosted(analysis_link) else "on-premise" o.add_attribute("score", result["score"]) o.add_attribute("sandbox-type", sandbox_type) o.add_attribute("{}-sandbox".format(sandbox_type), "lastline") @@ -266,408 +839,3 @@ class LastlineResultBaseParser(object): # Add mitre techniques for technique in self._get_mitre_techniques(result): self.misp_event.add_tag(technique) - - -class LastlineCommunityHTTPClient(object): - """"A very basic HTTP client providing basic functionality.""" - - @classmethod - def sanitize_login_params(cls, api_key, api_token, username, password): - """ - Return a dictionary with either API or USER credentials. - - :param str|None api_key: the API key - :param str|None api_token: the API token - :param str|None username: the username - :param str|None password: the password - :rtype: dict[str, str] - :return: the dictionary - :raises InvalidArgument: if too many values are invalid - """ - if api_key and api_token: - return { - "api_key": api_key, - "api_token": api_token, - } - elif username and password: - return { - "username": username, - "password": password, - } - else: - raise InvalidArgument("Arguments provided do not contain valid data") - - @classmethod - def get_login_params_from_conf(cls, conf, section_name): - """ - Get the module configuration from a ConfigParser object. - - :param ConfigParser conf: the conf object - :param str section_name: the section name - :rtype: dict[str, str] - :return: the parsed configuration - """ - api_key = conf.get(section_name, "api_key", fallback=None) - api_token = conf.get(section_name, "api_token", fallback=None) - username = conf.get(section_name, "username", fallback=None) - password = conf.get(section_name, "password", fallback=None) - return cls.sanitize_login_params(api_key, api_token, username, password) - - @classmethod - def get_login_params_from_request(cls, request): - """ - Get the module configuration from a ConfigParser object. - - :param dict[str, any] request: the request object - :rtype: dict[str, str] - :return: the parsed configuration - """ - api_key = request.get("config", {}).get("api_key") - api_token = request.get("config", {}).get("api_token") - username = request.get("config", {}).get("username") - password = request.get("config", {}).get("password") - return cls.sanitize_login_params(api_key, api_token, username, password) - - def __init__(self, api_url, login_params, timeout=60, verify_ssl=True): - """ - Instantiate a Lastline mini client. - - :param str api_url: the URL of the API - :param dict[str, str]: the login parameters - :param int timeout: the timeout - :param boolean verify_ssl: whether to verify the SSL certificate - """ - self.__url = api_url - self.__login_params = login_params - self.__timeout = timeout - self.__verify_ssl = verify_ssl - self.__session = None - self.__logger = logging.getLogger(__name__) - - def __login(self): - """Login using account-based or key-based methods.""" - if self.__session is None: - self.__session = requests.session() - - login_url = "/".join([self.__url, "login"]) - try: - response = self.__session.request( - method="POST", - url=login_url, - data=self.__login_params, - verify=self.__verify_ssl, - timeout=self.__timeout, - proxies=None, - ) - except requests.RequestException as e: - raise CommunicationError(e) - - self.__handle_response(response) - - def __is_logged_in(self): - """Return whether we have an active session.""" - return self.__session is not None - - @staticmethod - def __parse_response(response): - """ - Parse the response. - - :param requests.Response response: the response - :rtype: tuple(str|None, Error|ApiError) - :return: a tuple with mutually exclusive fields (either the response or the error) - """ - try: - ret = response.json() - if "success" not in ret: - return None, Error("no success field in response") - - if not ret["success"]: - error_msg = ret.get("error", "") - error_code = ret.get("error_code", None) - return None, ApiError(error_msg, error_code) - - if "data" not in ret: - return None, Error("no data field in response") - - return ret["data"], None - except ValueError as e: - return None, Error("Response not json {}".format(e)) - - def __handle_response(self, response, raw=False): - """ - Check a response for issues and parse the return. - - :param requests.Response response: the response - :param boolean raw: whether the raw body should be returned - :rtype: str - :return: if raw, return the response content; if not raw, the data field - :raises: CommunicationError, ApiError, Error - """ - # Check for HTTP errors, and re-raise in case - try: - response.raise_for_status() - except requests.RequestException as e: - _, err = self.__parse_response(response) - if isinstance(err, ApiError): - err_msg = "{}: {}".format(e, err.error_msg) - else: - err_msg = "{}".format(e) - raise CommunicationError(err_msg) - - # Otherwise return the data (either parsed or not) but reraise if we have an API error - if raw: - return response.content - data, err = self.__parse_response(response) - if err: - raise err - return data - - def do_request( - self, - method, - module, - function, - params=None, - data=None, - files=None, - url=None, - fmt="JSON", - raw=False, - raw_response=False, - headers=None, - stream_response=False - ): - if raw_response: - raw = True - - if fmt: - fmt = fmt.lower().strip() - if fmt not in ["json", "xml", "html", "pdf"]: - raise InvalidArgument("Only json, xml, html and pdf supported") - elif not raw: - raise InvalidArgument("Unformatted response requires raw=True") - - if fmt != "json" and not raw: - raise InvalidArgument("Non-json format requires raw=True") - - if method not in ["POST", "GET"]: - raise InvalidArgument("Only POST and GET supported") - - function = function.strip(" /") - if not function: - raise InvalidArgument("No function provided") - - # Login after we verified that all arguments are fine - if not self.__is_logged_in(): - self.__login() - - url_parts = [url or self.__url] - module = module.strip(" /") - if module: - url_parts.append(module) - if fmt: - function_part = "%s.%s" % (function, fmt) - else: - function_part = function - url_parts.append(function_part) - url = "/".join(url_parts) - - try: - try: - response = self.__session.request( - method=method, - url=url, - data=data, - params=params, - files=files, - verify=self.__verify_ssl, - timeout=self.__timeout, - stream=stream_response, - headers=headers, - ) - except requests.RequestException as e: - raise CommunicationError(e) - - if raw_response: - return response - return self.__handle_response(response, raw) - - except Error as e: - raise e - - except CommunicationError as e: - raise e - - -class LastlineCommunityAPIClient(object): - """"A very basic API client providing basic functionality.""" - - def __init__(self, api_url, login_params): - """ - Instantiate the API client. - - :param str api_url: the URL to the API server - :param dict[str, str] login_params: the login parameters - """ - self._client = LastlineCommunityHTTPClient(api_url, login_params) - self._logger = logging.getLogger(__name__) - - def _post(self, module, function, params=None, data=None, files=None, fmt="JSON"): - return self._client.do_request( - method="POST", - module=module, - function=function, - params=params, - data=data, - files=files, - fmt=fmt, - ) - - def _get(self, module, function, params=None, fmt="JSON"): - return self._client.do_request( - method="GET", - module=module, - function=function, - params=params, - fmt=fmt, - ) - - def get_progress(self, uuid, analysis_instance=None): - """ - Get the completion progress of a given task. - - :param str uuid: the unique identifier of the submitted task - :param str analysis_instance: if set, defines the analysis instance to query - :rtype: dict[str, int] - :return: a dictionary like the the following: - { - "completed": 1, - "progress": 100 - } - """ - params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance}) - return self._get("analysis", "get_progress", params=params) - - def get_result(self, uuid, analysis_instance=None): - """ - Get report results for a given task. - - :param str uuid: the unique identifier of the submitted task - :param str analysis_instance: if set, defines the analysis instance to query - :rtype: dict[str, any] - :return: a dictionary like the the following: - { - "completed": 1, - "progress": 100 - } - """ - params = purge_none( - { - "uuid": uuid, - "analysis_instance": analysis_instance, - "report_format": "json", - } - ) - return self._get("analysis", "get_result", params=params) - - def submit_url( - self, - url, - referer=None, - user_agent=None, - bypass_cache=False, - ): - """ - Upload an URL to be analyzed. - - :param str url: the url to analyze - :param str|None referer: the referer - :param str|None user_agent: the user agent - :param boolean bypass_cache: bypass_cache - :rtype: dict[str, any] - :return: a dictionary like the following if the analysis is already available: - { - "submission": "2019-11-17 09:33:23", - "child_tasks": [...], - "reports": [...], - "submission_timestamp": "2019-11-18 16:11:04", - "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", - "score": 0, - "analysis_subject": { - "url": "https://www.google.com" - }, - "last_submission_timestamp": "2019-11-18 16:11:04" - } - - OR the following if the analysis is still pending: - - { - "submission_timestamp": "2019-11-18 13:59:25", - "task_uuid": "f3c0ae115d51001017ff8da768fa6049", - } - """ - params = purge_none( - { - "url": url, - "bypass_cache": bypass_cache, - "referer": referer, - "user_agent": user_agent - } - ) - return self._post(module="analysis", function="submit_url", params=params) - - def submit_file( - self, - file_data, - file_name=None, - password=None, - analysis_env=None, - allow_network_traffic=True, - analysis_timeout=None, - bypass_cache=False, - ): - """ - Upload a file to be analyzed. - - :param bytes file_data: the data as a byte sequence - :param str|None file_name: if set, represents the name of the file to submit - :param str|None password: if set, use it to extract the sample - :param str|None analysis_env: if set, e.g windowsxp - :param boolean allow_network_traffic: if set to False, deny network connections - :param int|None analysis_timeout: if set limit the duration of the analysis - :param boolean bypass_cache: whether to re-process a file (requires special permissions) - :rtype: dict[str, any] - :return: a dictionary in the following form if the analysis is already available: - { - "submission": "2019-11-17 09:33:23", - "child_tasks": [...], - "reports": [...], - "submission_timestamp": "2019-11-18 16:11:04", - "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", - "score": 0, - "analysis_subject": { - "url": "https://www.google.com" - }, - "last_submission_timestamp": "2019-11-18 16:11:04" - } - - OR the following if the analysis is still pending: - - { - "submission_timestamp": "2019-11-18 13:59:25", - "task_uuid": "f3c0ae115d51001017ff8da768fa6049", - } - """ - params = purge_none( - { - "filename": file_name, - "password": password, - "analysis_env": analysis_env, - "allow_network_traffic": allow_network_traffic, - "analysis_timeout": analysis_timeout, - "bypass_cache": bypass_cache, - } - ) - files = {"file": (file_name, file_data, "application/octet-stream")} - return self._post(module="analysis", function="submit_file", params=params, files=files) diff --git a/misp_modules/lib/vt_graph_parser/__init__.py b/misp_modules/lib/vt_graph_parser/__init__.py new file mode 100644 index 0000000..abc02c5 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/__init__.py @@ -0,0 +1,8 @@ +"""vt_graph_parser. + +This module provides methods to import graph from misp. +""" + + +from .helpers import * # noqa +from .importers import * # noqa diff --git a/misp_modules/lib/vt_graph_parser/errors.py b/misp_modules/lib/vt_graph_parser/errors.py new file mode 100644 index 0000000..a7e18e9 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/errors.py @@ -0,0 +1,20 @@ +"""vt_graph_parser.errors. + +This module provides custom errors for data importers. +""" + + +class GraphImportError(Exception): + pass + + +class InvalidFileFormatError(Exception): + pass + + +class MispEventNotFoundError(Exception): + pass + + +class ServerError(Exception): + pass diff --git a/misp_modules/lib/vt_graph_parser/helpers/__init__.py b/misp_modules/lib/vt_graph_parser/helpers/__init__.py new file mode 100644 index 0000000..8f9f660 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/__init__.py @@ -0,0 +1,7 @@ +"""vt_graph_parser.helpers. + +This modules provides functions and attributes to help MISP importers. +""" + + +__all__ = ["parsers", "rules", "wrappers"] diff --git a/misp_modules/lib/vt_graph_parser/helpers/parsers.py b/misp_modules/lib/vt_graph_parser/helpers/parsers.py new file mode 100644 index 0000000..8ca5745 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/parsers.py @@ -0,0 +1,88 @@ +"""vt_graph_parser.helpers.parsers. + +This module provides parsers for MISP inputs. +""" + + +from vt_graph_parser.helpers.wrappers import MispAttribute + + +MISP_INPUT_ATTR = [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "url", + "filename|md5", + "filename", + "target-user", + "target-email" +] + +VIRUSTOTAL_GRAPH_LINK_PREFIX = "https://www.virustotal.com/graph/" + + +def _parse_data(attributes, objects): + """Parse MISP event attributes and objects data. + + Args: + attributes (dict): dictionary which contains the MISP event attributes data. + objects (dict): dictionary which contains the MISP event objects data. + + Returns: + ([MispAttribute], str): MISP attributes and VTGraph link if exists. + Link defaults to "". + """ + attributes_data = [] + vt_graph_link = "" + + # Get simple MISP event attributes. + attributes_data += ( + [attr for attr in attributes + if attr.get("type") in MISP_INPUT_ATTR]) + + # Get attributes from MISP objects too. + if objects: + for object_ in objects: + object_attrs = object_.get("Attribute", []) + attributes_data += ( + [attr for attr in object_attrs + if attr.get("type") in MISP_INPUT_ATTR]) + + # Check if there is any VirusTotal Graph computed in MISP event. + vt_graph_links = ( + attr for attr in attributes if attr.get("type") == "link" + and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX)) + + # MISP could have more than one VirusTotal Graph, so we will take + # the last one. + current_id = 0 # MISP attribute id is the number of the attribute. + vt_graph_link = "" + for link in vt_graph_links: + if int(link.get("id")) > current_id: + current_id = int(link.get("id")) + vt_graph_link = link.get("value") + + attributes = [ + MispAttribute(data["type"], data["category"], data["value"]) + for data in attributes_data] + return (attributes, + vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, "")) + + +def parse_pymisp_response(payload): + """Get event attributes and VirusTotal Graph id from pymisp response. + + Args: + payload (dict): dictionary which contains pymisp response. + + Returns: + ([MispAttribute], str): MISP attributes and VTGraph link if exists. + Link defaults to "". + """ + event_attrs = payload.get("Attribute", []) + objects = payload.get("Object") + return _parse_data(event_attrs, objects) diff --git a/misp_modules/lib/vt_graph_parser/helpers/rules.py b/misp_modules/lib/vt_graph_parser/helpers/rules.py new file mode 100644 index 0000000..e3ed7f8 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/rules.py @@ -0,0 +1,304 @@ +"""vt_graph_parser.helpers.rules. + +This module provides rules that helps MISP importers to connect MISP attributes +between them using VirusTotal relationship. Check all available relationship +here: + +- File: https://developers.virustotal.com/v3/reference/#files-relationships +- URL: https://developers.virustotal.com/v3/reference/#urls-relationships +- Domain: https://developers.virustotal.com/v3/reference/#domains-relationships +- IP: https://developers.virustotal.com/v3/reference/#ip-relationships +""" + + +import abc + + +class MispEventRule(object): + """Rules for MISP event nodes connection object wrapper.""" + + def __init__(self, last_rule=None, node=None): + """Create a MispEventRule instance. + + MispEventRule is a collection of rules that can infer the relationships + between nodes from MISP events. + + Args: + last_rule (MispEventRule): previous rule. + node (Node): actual node. + """ + self.last_rule = last_rule + self.node = node + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def get_last_different_rule(self): + """Search the last rule whose event was different from actual. + + Returns: + MispEventRule: the last different rule. + """ + if not isinstance(self, self.last_rule.__class__): + return self.last_rule + else: + return self.last_rule.get_last_different_rule() + + def resolve_relation(self, graph, node, misp_category): + """Try to infer a relationship between two nodes. + + This method is based on a non-deterministic finite automaton for + this reason the future rule only depends on the actual rule and the input + node. + + For example if the actual rule is a MISPEventDomainRule and the given node + is an ip_address node, the connection type between them will be + `resolutions` and the this rule will transit to MISPEventIPRule. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + if node.node_type in self.relation_event: + return self.relation_event[node.node_type](graph, node, misp_category) + else: + return self.manual_link(graph, node) + + def manual_link(self, graph, node): + """Creates a manual link between self.node and the given node. + + We accept MISP types that VirusTotal does not know how to link, so we create + a end to end relationship instead of create an unknown relationship node. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + + Returns: + MispEventRule: the transited rule. + """ + graph.add_link(self.node.node_id, node.node_id, "manual") + return self + + @abc.abstractmethod + def __file_transition(self, graph, node, misp_category): + """Make a new transition due to file attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __ip_transition(self, graph, node, misp_category): + """Make a new transition due to ip attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __url_transition(self, graph, node, misp_category): + """Make a new transition due to url attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __domain_transition(self, graph, node, misp_category): + """Make a new transition due to domain attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + +class MispEventURLRule(MispEventRule): + """Rule for URL event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventURLRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "downloaded_files") + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_ips") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_domains") + return MispEventDomainRule(self, node) + + +class MispEventIPRule(MispEventRule): + """Rule for IP event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventIPRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + connection_type = "communicating_files" + if misp_category == "Artifacts dropped": + connection_type = "downloaded_files" + graph.add_link(self.node.node_id, node.node_id, connection_type) + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "resolutions") + return MispEventDomainRule(self, node) + + +class MispEventDomainRule(MispEventRule): + """Rule for domain event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventDomainRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + connection_type = "communicating_files" + if misp_category == "Artifacts dropped": + connection_type = "downloaded_files" + graph.add_link(self.node.node_id, node.node_id, connection_type) + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "resolutions") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + graph.add_link(self.node.node_id, node.node_id, "siblings") + return MispEventDomainRule(self, node) + + +class MispEventFileRule(MispEventRule): + """Rule for File event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventFileRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_ips") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_domains") + return MispEventDomainRule(self, node) + + +class MispEventInitialRule(MispEventRule): + """Initial rule.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventInitialRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + return MispEventDomainRule(self, node) diff --git a/misp_modules/lib/vt_graph_parser/helpers/wrappers.py b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py new file mode 100644 index 0000000..d376d43 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py @@ -0,0 +1,58 @@ +"""vt_graph_parser.helpers.wrappers. + +This module provides a Python object wrapper for MISP objects. +""" + + +class MispAttribute(object): + """Python object wrapper for MISP attribute. + + Attributes: + type (str): VirusTotal node type. + category (str): MISP attribute category. + value (str): node id. + label (str): node name. + misp_type (str): MISP node type. + """ + + MISP_TYPES_REFERENCE = { + "hostname": "domain", + "domain": "domain", + "ip-src": "ip_address", + "ip-dst": "ip_address", + "url": "url", + "filename|X": "file", + "filename": "file", + "md5": "file", + "sha1": "file", + "sha256": "file", + "target-user": "victim", + "target-email": "email" + } + + def __init__(self, misp_type, category, value, label=""): + """Constructor for a MispAttribute. + + Args: + misp_type (str): MISP type attribute. + category (str): MISP category attribute. + value (str): attribute value. + label (str): attribute label. + """ + if misp_type.startswith("filename|"): + label, value = value.split("|") + misp_type = "filename|X" + if misp_type == "filename": + label = value + + self.type = self.MISP_TYPES_REFERENCE.get(misp_type) + self.category = category + self.value = value + self.label = label + self.misp_type = misp_type + + def __eq__(self, other): + return (isinstance(other, self.__class__) and self.value == other.value and self.type == other.type) + + def __repr__(self): + return 'MispAttribute("{type}", "{category}", "{value}")'.format(type=self.type, category=self.category, value=self.value) diff --git a/misp_modules/lib/vt_graph_parser/importers/__init__.py b/misp_modules/lib/vt_graph_parser/importers/__init__.py new file mode 100644 index 0000000..c59197c --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/__init__.py @@ -0,0 +1,7 @@ +"""vt_graph_parser.importers. + +This module provides methods to import graphs from MISP. +""" + + +__all__ = ["base", "pymisp_response"] diff --git a/misp_modules/lib/vt_graph_parser/importers/base.py b/misp_modules/lib/vt_graph_parser/importers/base.py new file mode 100644 index 0000000..ed5c0fc --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/base.py @@ -0,0 +1,98 @@ +"""vt_graph_parser.importers.base. + +This module provides a common method to import graph from misp attributes. +""" + + +import vt_graph_api +from vt_graph_parser.helpers.rules import MispEventInitialRule + + +def import_misp_graph( + misp_attributes, graph_id, vt_api_key, fetch_information, name, + private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, + group_viewers, use_vt_to_connect_the_graph, max_api_quotas, + max_search_depth): + """Import VirusTotal Graph from MISP. + + Args: + misp_attributes ([MispAttribute]): list with the MISP attributes which + will be added to the returned graph. + graph_id: if supplied, the graph will be loaded instead of compute it again. + vt_api_key (str): VT API Key. + fetch_information (bool): whether the script will fetch + information for added nodes in VT. Defaults to True. + name (str): graph title. Defaults to "". + private (bool): True for private graphs. You need to have + Private Graph premium features enabled in your subscription. Defaults + to False. + fetch_vt_enterprise (bool, optional): if True, the graph will search any + available information using VirusTotal Intelligence for the node if there + is no normal information for it. Defaults to False. + user_editors ([str]): usernames that can edit the graph. + Defaults to None. + user_viewers ([str]): usernames that can view the graph. + Defaults to None. + group_editors ([str]): groups that can edit the graph. + Defaults to None. + group_viewers ([str]): groups that can view the graph. + Defaults to None. + use_vt_to_connect_the_graph (bool): if True, graph nodes will + be linked using VirusTotal API. Otherwise, the links will be generated + using production rules based on MISP attributes order. Defaults to + False. + max_api_quotas (int): maximum number of api quotas that could + be consumed to resolve graph using VirusTotal API. Defaults to 20000. + max_search_depth (int, optional): max search depth to explore + relationship between nodes when use_vt_to_connect_the_graph is True. + Defaults to 3. + + If use_vt_to_connect_the_graph is True, it will take some time to compute + graph. + + Returns: + vt_graph_api.graph.VTGraph: the imported graph. + """ + + rule = MispEventInitialRule() + + # Check if the event has been already computed in VirusTotal Graph. Otherwise + # a new graph will be created. + if not graph_id: + graph = vt_graph_api.VTGraph( + api_key=vt_api_key, name=name, private=private, + user_editors=user_editors, user_viewers=user_viewers, + group_editors=group_editors, group_viewers=group_viewers) + else: + graph = vt_graph_api.VTGraph.load_graph(graph_id, vt_api_key) + + attributes_to_add = [attr for attr in misp_attributes + if not graph.has_node(attr.value)] + + total_expandable_attrs = max(sum( + 1 for attr in attributes_to_add + if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES), + 1) + + max_quotas_per_search = max( + int(max_api_quotas / total_expandable_attrs), 1) + + previous_node_id = "" + for attr in attributes_to_add: + # Add the current attr as node to the graph. + added_node = graph.add_node( + attr.value, attr.type, fetch_information, fetch_vt_enterprise, + attr.label) + # If use_vt_to_connect_the_grap is True the nodes will be connected using + # VT API. + if use_vt_to_connect_the_graph: + if (attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and previous_node_id): + graph.add_link(previous_node_id, attr.value, "manual") + else: + graph.connect_with_graph( + attr.value, max_quotas_per_search, max_search_depth, + fetch_info_collected_nodes=fetch_information) + else: + rule = rule.resolve_relation(graph, added_node, attr.category) + + return graph diff --git a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py new file mode 100644 index 0000000..e0e834b --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py @@ -0,0 +1,73 @@ +"""vt_graph_parser.importers.pymisp_response. + +This modules provides a graph importer method for MISP event by using the +response payload giving by MISP API directly. +""" + + +from vt_graph_parser.helpers.parsers import parse_pymisp_response +from vt_graph_parser.importers.base import import_misp_graph + + +def from_pymisp_response( + payload, vt_api_key, fetch_information=True, + private=False, fetch_vt_enterprise=False, user_editors=None, + user_viewers=None, group_editors=None, group_viewers=None, + use_vt_to_connect_the_graph=False, max_api_quotas=1000, + max_search_depth=3, expand_node_one_level=False): + """Import VirusTotal Graph from MISP JSON file. + + Args: + payload (dict): dictionary which contains the request payload. + vt_api_key (str): VT API Key. + fetch_information (bool, optional): whether the script will fetch + information for added nodes in VT. Defaults to True. + name (str, optional): graph title. Defaults to "". + private (bool, optional): True for private graphs. You need to have + Private Graph premium features enabled in your subscription. Defaults + to False. + fetch_vt_enterprise (bool, optional): if True, the graph will search any + available information using VirusTotal Intelligence for the node if there + is no normal information for it. Defaults to False. + user_editors ([str], optional): usernames that can edit the graph. + Defaults to None. + user_viewers ([str], optional): usernames that can view the graph. + Defaults to None. + group_editors ([str], optional): groups that can edit the graph. + Defaults to None. + group_viewers ([str], optional): groups that can view the graph. + Defaults to None. + use_vt_to_connect_the_graph (bool, optional): if True, graph nodes will + be linked using VirusTotal API. Otherwise, the links will be generated + using production rules based on MISP attributes order. Defaults to + False. + max_api_quotas (int, optional): maximum number of api quotas that could + be consumed to resolve graph using VirusTotal API. Defaults to 20000. + max_search_depth (int, optional): max search depth to explore + relationship between nodes when use_vt_to_connect_the_graph is True. + Defaults to 3. + expand_one_level (bool, optional): expand entire graph one level. + Defaults to False. + + If use_vt_to_connect_the_graph is True, it will take some time to compute + graph. + + Raises: + LoaderError: if JSON file is invalid. + + Returns: + [vt_graph_api.graph.VTGraph: the imported graph]. + """ + graphs = [] + for event_payload in payload['data']: + misp_attrs, graph_id = parse_pymisp_response(event_payload) + name = "Graph created from MISP event" + graph = import_misp_graph( + misp_attrs, graph_id, vt_api_key, fetch_information, name, + private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, + group_viewers, use_vt_to_connect_the_graph, max_api_quotas, + max_search_depth) + if expand_node_one_level: + graph.expand_n_level(1) + graphs.append(graph) + return graphs diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 12c2ab6..2a99050 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -6,7 +6,7 @@ sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/') __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', - 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', + 'whois', 'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', @@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus', 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', - 'lastline_query', 'lastline_submit'] + 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion'] diff --git a/misp_modules/modules/expansion/_ransomcoindb/__init__.py b/misp_modules/modules/expansion/_ransomcoindb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py new file mode 100755 index 0000000..9723ed6 --- /dev/null +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +''' +Cytomic Orion MISP Module +An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion + + +''' + +from pymisp import MISPAttribute, MISPEvent, MISPObject +import json +import requests +import sys + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['md5'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.3', 'author': 'Koen Van Impe', + 'description': 'an expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion', + 'module-type': ['expansion']} +moduleconfig = ['api_url', 'token_url', 'clientid', 'clientsecret', 'clientsecret', 'username', 'password', 'upload_timeframe', 'upload_tag', 'delete_tag', 'upload_ttlDays', 'upload_threat_level_id', 'limit_upload_events', 'limit_upload_attributes'] +# There are more config settings in this module than used by the enrichment +# There is also a PyMISP module which reuses the module config, and requires additional configuration, for example used for pushing indicators to the API + + +class CytomicParser(): + def __init__(self, attribute, config_object): + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.misp_event.add_attribute(**self.attribute) + + self.config_object = config_object + + if self.config_object: + self.token = self.get_token() + else: + sys.exit('Missing configuration') + + def get_token(self): + try: + scope = self.config_object['scope'] + grant_type = self.config_object['grant_type'] + username = self.config_object['username'] + password = self.config_object['password'] + token_url = self.config_object['token_url'] + clientid = self.config_object['clientid'] + clientsecret = self.config_object['clientsecret'] + + if scope and grant_type and username and password: + data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password} + + if token_url and clientid and clientsecret: + access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret)) + tokens = json.loads(access_token_response.text) + if 'access_token' in tokens: + return tokens['access_token'] + else: + self.result = {'error': 'No token received.'} + return + else: + self.result = {'error': 'No token_url, clientid or clientsecret supplied.'} + return + else: + self.result = {'error': 'No scope, grant_type, username or password supplied.'} + return + except Exception: + self.result = {'error': 'Unable to connect to token_url.'} + return + + def get_results(self): + if hasattr(self, 'result'): + return self.result + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + def parse(self, searchkey): + + if self.token: + + endpoint_fileinformation = self.config_object['endpoint_fileinformation'] + endpoint_machines = self.config_object['endpoint_machines'] + endpoint_machines_client = self.config_object['endpoint_machines_client'] + query_machines = self.config_object['query_machines'] + query_machine_info = self.config_object['query_machine_info'] + + # Update endpoint URLs + query_endpoint_fileinformation = endpoint_fileinformation.format(md5=searchkey) + query_endpoint_machines = endpoint_machines.format(md5=searchkey) + + # API calls + api_call_headers = {'Authorization': 'Bearer ' + self.token} + result_query_endpoint_fileinformation = requests.get(query_endpoint_fileinformation, headers=api_call_headers, verify=False) + json_result_query_endpoint_fileinformation = json.loads(result_query_endpoint_fileinformation.text) + + if json_result_query_endpoint_fileinformation: + + cytomic_object = MISPObject('cytomic-orion-file') + + cytomic_object.add_attribute('fileName', type='text', value=json_result_query_endpoint_fileinformation['fileName']) + cytomic_object.add_attribute('fileSize', type='text', value=json_result_query_endpoint_fileinformation['fileSize']) + cytomic_object.add_attribute('last-seen', type='datetime', value=json_result_query_endpoint_fileinformation['lastSeen']) + cytomic_object.add_attribute('first-seen', type='datetime', value=json_result_query_endpoint_fileinformation['firstSeen']) + cytomic_object.add_attribute('classification', type='text', value=json_result_query_endpoint_fileinformation['classification']) + cytomic_object.add_attribute('classificationName', type='text', value=json_result_query_endpoint_fileinformation['classificationName']) + self.misp_event.add_object(**cytomic_object) + + result_query_endpoint_machines = requests.get(query_endpoint_machines, headers=api_call_headers, verify=False) + json_result_query_endpoint_machines = json.loads(result_query_endpoint_machines.text) + + if query_machines and json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: + for machine in json_result_query_endpoint_machines: + + if query_machine_info and machine['muid']: + query_endpoint_machines_client = endpoint_machines_client.format(muid=machine['muid']) + result_endpoint_machines_client = requests.get(query_endpoint_machines_client, headers=api_call_headers, verify=False) + json_result_endpoint_machines_client = json.loads(result_endpoint_machines_client.text) + + if json_result_endpoint_machines_client: + + cytomic_machine_object = MISPObject('cytomic-orion-machine') + + clienttag = [{'name': json_result_endpoint_machines_client['clientName']}] + + cytomic_machine_object.add_attribute('machineName', type='target-machine', value=json_result_endpoint_machines_client['machineName'], Tag=clienttag) + cytomic_machine_object.add_attribute('machineMuid', type='text', value=machine['muid']) + cytomic_machine_object.add_attribute('clientName', type='target-org', value=json_result_endpoint_machines_client['clientName'], Tag=clienttag) + cytomic_machine_object.add_attribute('clientId', type='text', value=machine['clientId']) + cytomic_machine_object.add_attribute('machinePath', type='text', value=machine['lastPath']) + cytomic_machine_object.add_attribute('first-seen', type='datetime', value=machine['firstSeen']) + cytomic_machine_object.add_attribute('last-seen', type='datetime', value=machine['lastSeen']) + cytomic_machine_object.add_attribute('creationDate', type='datetime', value=json_result_endpoint_machines_client['creationDate']) + cytomic_machine_object.add_attribute('clientCreationDateUTC', type='datetime', value=json_result_endpoint_machines_client['clientCreationDateUTC']) + cytomic_machine_object.add_attribute('lastSeenUtc', type='datetime', value=json_result_endpoint_machines_client['lastSeenUtc']) + self.misp_event.add_object(**cytomic_machine_object) + else: + self.result = {'error': 'No (valid) token.'} + return + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('attribute'): + return {'error': 'Unsupported input.'} + + attribute = request['attribute'] + if not any(input_type == attribute['type'] for input_type in mispattributes['input']): + return {'error': 'Unsupported attributes type'} + + if not request.get('config'): + return {'error': 'Missing configuration'} + + config_object = { + 'clientid': request["config"].get("clientid"), + 'clientsecret': request["config"].get("clientsecret"), + 'scope': 'orion.api', + 'password': request["config"].get("password"), + 'username': request["config"].get("username"), + 'grant_type': 'password', + 'token_url': request["config"].get("token_url"), + 'endpoint_fileinformation': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/info'), + 'endpoint_machines': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/muids'), + 'endpoint_machines_client': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/muid/{muid}/info'), + 'query_machines': True, + 'query_machine_info': True + } + + cytomic_parser = CytomicParser(attribute, config_object) + cytomic_parser.parse(attribute['value']) + + return cytomic_parser.get_results() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/geoip_asn.py b/misp_modules/modules/expansion/geoip_asn.py new file mode 100644 index 0000000..95d7ee7 --- /dev/null +++ b/misp_modules/modules/expansion/geoip_asn.py @@ -0,0 +1,64 @@ +import json +import geoip2.database +import sys +import logging + +log = logging.getLogger('geoip_asn') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} +moduleconfig = ['local_geolite_db'] +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '0.1', 'author': 'GlennHD', + 'description': 'Query a local copy of the Maxmind Geolite ASN database (MMDB format)', + 'module-type': ['expansion', 'hover']} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('local_geolite_db'): + return {'error': 'Please specify the path of your local copy of the Maxmind Geolite ASN database'} + path_to_geolite = request['config']['local_geolite_db'] + + if request.get('ip-dst'): + toquery = request['ip-dst'] + elif request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('domain|ip'): + toquery = request['domain|ip'].split('|')[1] + else: + return False + + try: + reader = geoip2.database.Reader(path_to_geolite) + except FileNotFoundError: + return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + log.debug(toquery) + try: + answer = reader.asn(toquery) + stringmap = 'ASN=' + str(answer.autonomous_system_number) + ', AS Org=' + str(answer.autonomous_system_organization) + except Exception as e: + misperrors['error'] = f"GeoIP resolving error: {e}" + return misperrors + + r = {'results': [{'types': mispattributes['output'], 'values': stringmap}]} + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/geoip_city.py b/misp_modules/modules/expansion/geoip_city.py new file mode 100644 index 0000000..01c0627 --- /dev/null +++ b/misp_modules/modules/expansion/geoip_city.py @@ -0,0 +1,65 @@ +import json +import geoip2.database +import sys +import logging + +log = logging.getLogger('geoip_city') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} +moduleconfig = ['local_geolite_db'] +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '0.1', 'author': 'GlennHD', + 'description': 'Query a local copy of the Maxmind Geolite City database (MMDB format)', + 'module-type': ['expansion', 'hover']} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('local_geolite_db'): + return {'error': 'Please specify the path of your local copy of Maxminds Geolite database'} + path_to_geolite = request['config']['local_geolite_db'] + + if request.get('ip-dst'): + toquery = request['ip-dst'] + elif request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('domain|ip'): + toquery = request['domain|ip'].split('|')[1] + else: + return False + + try: + reader = geoip2.database.Reader(path_to_geolite) + except FileNotFoundError: + return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + log.debug(toquery) + try: + answer = reader.city(toquery) + stringmap = 'Continent=' + str(answer.continent.name) + ', Country=' + str(answer.country.name) + ', Subdivision=' + str(answer.subdivisions.most_specific.name) + ', City=' + str(answer.city.name) + + except Exception as e: + misperrors['error'] = f"GeoIP resolving error: {e}" + return misperrors + + r = {'results': [{'types': mispattributes['output'], 'values': stringmap}]} + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 8ba012b..d28e570 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -59,5 +59,5 @@ def introspection(): def version(): - # moduleinfo['config'] = moduleconfig + moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/google_search.py b/misp_modules/modules/expansion/google_search.py new file mode 100644 index 0000000..b7b4e7a --- /dev/null +++ b/misp_modules/modules/expansion/google_search.py @@ -0,0 +1,34 @@ +import json +try: + from google import google +except ImportError: + print("GoogleAPI not installed. Command : pip install git+https://github.com/abenassi/Google-Search-API") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['url'], 'output': ['text']} +moduleinfo = {'author': 'Oun & Gindt', 'module-type': ['hover'], + 'description': 'An expansion hover module to expand google search information about an URL'} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('url'): + return {'error': "Unsupported attributes type"} + num_page = 1 + res = "" + search_results = google.search(request['url'], num_page) + for i in range(3): + res += "("+str(i+1)+")" + '\t' + res += json.dumps(search_results[i].description, ensure_ascii=False) + res += '\n\n' + return {'results': [{'types': mispattributes['output'], 'values':res}]} + + +def introspection(): + return mispattributes + + +def version(): + return moduleinfo diff --git a/misp_modules/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py index 8489aa0..3c6867c 100755 --- a/misp_modules/modules/expansion/ipasn.py +++ b/misp_modules/modules/expansion/ipasn.py @@ -2,22 +2,40 @@ import json from pyipasnhistory import IPASNHistory +from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} -moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', +mispattributes = {'input': ['ip-src', 'ip-dst'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.2', 'author': 'Raphaël Vinot', 'description': 'Query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git)', 'module-type': ['expansion', 'hover']} +def parse_result(attribute, values): + event = MISPEvent() + initial_attribute = MISPAttribute() + initial_attribute.from_dict(**attribute) + event.add_attribute(**initial_attribute) + mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')} + print(values) + for last_seen, response in values['response'].items(): + asn = MISPObject('asn') + asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen}) + for feature, attribute_fields in mapping.items(): + attribute_type, object_relation = attribute_fields + asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]}) + asn.add_reference(initial_attribute.uuid, 'related-to') + event.add_object(**asn) + event = json.loads(event.to_json()) + return {key: event[key] for key in ('Attribute', 'Object')} + + def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip-dst'): - toquery = request['ip-dst'] + if request.get('attribute') and request['attribute'].get('type') in mispattributes['input']: + toquery = request['attribute']['value'] else: misperrors['error'] = "Unsupported attributes type" return misperrors @@ -28,7 +46,7 @@ def handler(q=False): if not values: misperrors['error'] = 'Unable to find the history of this IP' return misperrors - return {'results': [{'types': mispattributes['output'], 'values': values}]} + return {'results': parse_result(request['attribute'], values)} def introspection(): diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py index dce63ea..1ace259 100644 --- a/misp_modules/modules/expansion/joesandbox_query.py +++ b/misp_modules/modules/expansion/joesandbox_query.py @@ -4,12 +4,13 @@ import json from joe_parser import JoeParser misperrors = {'error': 'Error'} -mispattributes = {'input': ['link'], 'format': 'misp_standard'} -moduleinfo = {'version': '0.1', 'author': 'Christian Studer', +inputSource = ['link'] + +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', 'description': 'Query Joe Sandbox API with a report URL to get the parsed data.', 'module-type': ['expansion']} -moduleconfig = ['apiurl', 'apikey'] +moduleconfig = ['apiurl', 'apikey', 'import_pe', 'import_mitre_attack'] def handler(q=False): @@ -18,6 +19,11 @@ def handler(q=False): request = json.loads(q) apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api' apikey = request['config'].get('apikey') + parser_config = { + "import_pe": request["config"].get('import_pe', "false") == "true", + "mitre_attack": request["config"].get('import_mitre_attack', "false") == "true", + } + if not apikey: return {'error': 'No API key provided'} @@ -41,7 +47,7 @@ def handler(q=False): analysis_webid = joe_info['most_relevant_analysis']['webid'] - joe_parser = JoeParser() + joe_parser = JoeParser(parser_config) joe_data = json.loads(joe.analysis_download(analysis_webid, 'jsonfixed')[1]) joe_parser.parse_data(joe_data['analysis']) joe_parser.finalize_results() @@ -50,7 +56,19 @@ def handler(q=False): def introspection(): - return mispattributes + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['input'] = inputSource + except NameError: + pass + modulesetup['format'] = 'misp_standard' + return modulesetup def version(): diff --git a/misp_modules/modules/expansion/lastline_query.py b/misp_modules/modules/expansion/lastline_query.py index 4019b92..4ce4e47 100644 --- a/misp_modules/modules/expansion/lastline_query.py +++ b/misp_modules/modules/expansion/lastline_query.py @@ -27,10 +27,9 @@ moduleinfo = { } moduleconfig = [ - "api_key", - "api_token", "username", "password", + "verify_ssl", ] @@ -51,24 +50,25 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) + config = request["config"] + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) analysis_link = request['attribute']['value'] # The API url changes based on the analysis link host name - api_url = lastline_api.get_api_url_from_link(analysis_link) + api_url = lastline_api.get_portal_url_from_task_link(analysis_link) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - task_uuid = lastline_api.get_uuid_from_link(analysis_link) + task_uuid = lastline_api.get_uuid_from_task_link(analysis_link) except (KeyError, ValueError) as e: misperrors["error"] = "Error processing input parameters: {}".format(e) return misperrors # Make the API calls try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.PortalClient(api_url, auth_data, verify_ssl=config.get('verify_ssl', True).lower() in ("true")) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") @@ -108,7 +108,7 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( { diff --git a/misp_modules/modules/expansion/lastline_submit.py b/misp_modules/modules/expansion/lastline_submit.py index 0ae475a..1572955 100644 --- a/misp_modules/modules/expansion/lastline_submit.py +++ b/misp_modules/modules/expansion/lastline_submit.py @@ -33,13 +33,9 @@ moduleinfo = { } moduleconfig = [ - "api_url", - "api_key", + "url", "api_token", - "username", - "password", - # Module options - "bypass_cache", + "key", ] @@ -75,31 +71,31 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) - api_url = request.get("config", {}).get("api_url", lastline_api.DEFAULT_LASTLINE_API) + config = request.get("config", {}) + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) + api_url = config.get("url", lastline_api.DEFAULT_LL_ANALYSIS_API_URL) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - bypass_cache = request.get("config", {}).get("bypass_cache", False) - call_args = {"bypass_cache": __str_to_bool(bypass_cache)} + call_args = {} if "url" in request: # URLs are text strings - api_method = lastline_api.LastlineCommunityAPIClient.submit_url + api_method = lastline_api.AnalysisClient.submit_url call_args["url"] = request.get("url") else: data = request.get("data") # Malware samples are zip-encrypted and then base64 encoded if "malware-sample" in request: - api_method = lastline_api.LastlineCommunityAPIClient.submit_file + api_method = lastline_api.AnalysisClient.submit_file call_args["file_data"] = __unzip(base64.b64decode(data), DEFAULT_ZIP_PASSWORD) call_args["file_name"] = request.get("malware-sample").split("|", 1)[0] call_args["password"] = DEFAULT_ZIP_PASSWORD # Attachments are just base64 encoded elif "attachment" in request: - api_method = lastline_api.LastlineCommunityAPIClient.submit_file + api_method = lastline_api.AnalysisClient.submit_file call_args["file_data"] = base64.b64decode(data) call_args["file_name"] = request.get("attachment") @@ -112,7 +108,7 @@ def handler(q=False): # Make the API call try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.AnalysisClient(api_url, auth_data) response = api_method(api_client, **call_args) task_uuid = response.get("task_uuid") if not task_uuid: @@ -127,7 +123,7 @@ def handler(q=False): return misperrors # Assemble and return - analysis_link = lastline_api.get_analysis_link(api_url, task_uuid) + analysis_link = lastline_api.get_task_link(task_uuid, analysis_url=api_url) return { "results": [ @@ -152,12 +148,12 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( { "config": a, - "url": "https://www.google.com", + "url": "https://www.google.exe.com", } ) print(json.dumps(handler(j), indent=4, sort_keys=True)) diff --git a/misp_modules/modules/expansion/ransomcoindb.py b/misp_modules/modules/expansion/ransomcoindb.py index 3bac983..2b9b566 100644 --- a/misp_modules/modules/expansion/ransomcoindb.py +++ b/misp_modules/modules/expansion/ransomcoindb.py @@ -26,6 +26,8 @@ def handler(q=False): return False q = json.loads(q) + if "config" not in q or "api-key" not in q["config"]: + return {"error": "Ransomcoindb API key is missing"} api_key = q["config"]["api-key"] r = {"results": []} diff --git a/misp_modules/modules/expansion/sophoslabs_intelix.py b/misp_modules/modules/expansion/sophoslabs_intelix.py new file mode 100644 index 0000000..017683a --- /dev/null +++ b/misp_modules/modules/expansion/sophoslabs_intelix.py @@ -0,0 +1,124 @@ +from pymisp import MISPEvent, MISPObject +import json +import requests +import base64 +from urllib.parse import quote + +moduleinfo = {'version': '1.0', + 'author': 'Ben Verschaeren', + 'description': 'SOPHOSLabs Intelix Integration', + 'module-type': ['expansion']} + +moduleconfig = ['client_id', 'client_secret'] + +misperrors = {'error': 'Error'} + +misp_types_in = ['sha256', 'ip', 'ip-src', 'ip-dst', 'uri', 'url', 'domain', 'hostname'] + +mispattributes = {'input': misp_types_in, + 'format': 'misp_standard'} + + +class SophosLabsApi(): + def __init__(self, client_id, client_secret): + self.misp_event = MISPEvent() + self.client_id = client_id + self.client_secret = client_secret + self.authToken = f"{self.client_id}:{self.client_secret}" + self.baseurl = 'de.api.labs.sophos.com' + d = {'grant_type': 'client_credentials'} + h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}", + 'Content-Type': 'application/x-www-form-urlencoded'} + r = requests.post('https://api.labs.sophos.com/oauth2/token', headers=h, data=d) + if r.status_code == 200: + j = json.loads(r.text) + self.accessToken = j['access_token'] + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + def hash_lookup(self, filehash): + sophos_object = MISPObject('SOPHOSLabs Intelix SHA256 Report') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/files/v1/{filehash}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'reputationScore' in j: + sophos_object.add_attribute('Reputation Score', type='text', value=j['reputationScore']) + if 0 <= j['reputationScore'] <= 19: + sophos_object.add_attribute('Decision', type='text', value='This file is malicious') + if 20 <= j['reputationScore'] <= 29: + sophos_object.add_attribute('Decision', type='text', value='This file is potentially unwanted') + if 30 <= j['reputationScore'] <= 69: + sophos_object.add_attribute('Decision', type='text', value='This file is unknown and suspicious') + if 70 <= j['reputationScore'] <= 100: + sophos_object.add_attribute('Decision', type='text', value='This file is known good') + if 'detectionName' in j: + sophos_object.add_attribute('Detection Name', type='text', value=j['detectionName']) + else: + sophos_object.add_attribute('Detection Name', type='text', value='No name associated with this IoC') + self.misp_event.add_object(**sophos_object) + + def ip_lookup(self, ip): + sophos_object = MISPObject('SOPHOSLabs Intelix IP Category Lookup') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/ips/v1/{ip}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'category' in j: + for c in j['category']: + sophos_object.add_attribute('IP Address Categorisation', type='text', value=c) + else: + sophos_object.add_attribute('IP Address Categorisation', type='text', value='No category assocaited with IoC') + self.misp_event.add_object(**sophos_object) + + def url_lookup(self, url): + sophos_object = MISPObject('SOPHOSLabs Intelix URL Lookup') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/urls/v1/{quote(url, safe='')}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'productivityCategory' in j: + sophos_object.add_attribute('URL Categorisation', type='text', value=j['productivityCategory']) + else: + sophos_object.add_attribute('URL Categorisation', type='text', value='No category assocaited with IoC') + + if 'riskLevel' in j: + sophos_object.add_attribute('URL Risk Level', type='text', value=j['riskLevel']) + else: + sophos_object.add_attribute('URL Risk Level', type='text', value='No risk level associated with IoC') + + if 'securityCategory' in j: + sophos_object.add_attribute('URL Security Category', type='text', value=j['securityCategory']) + else: + sophos_object.add_attribute('URL Security Category', type='text', value='No Security Category associated with IoC') + self.misp_event.add_object(**sophos_object) + + +def handler(q=False): + if q is False: + return False + j = json.loads(q) + if not j.get('config') or not j['config'].get('client_id') or not j['config'].get('client_secret'): + misperrors['error'] = "Missing client_id or client_secret value for SOPHOSLabs Intelix. \ + It's free to sign up here https://aws.amazon.com/marketplace/pp/B07SLZPMCS." + return misperrors + client = SophosLabsApi(j['config']['client_id'], j['config']['client_secret']) + if j['attribute']['type'] == "sha256": + client.hash_lookup(j['attribute']['value1']) + if j['attribute']['type'] in ['ip-dst', 'ip-src', 'ip']: + client.ip_lookup(j["attribute"]["value1"]) + if j['attribute']['type'] in ['uri', 'url', 'domain', 'hostname']: + client.url_lookup(j["attribute"]["value1"]) + return client.get_result() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py index 30b78ee..baaaaf6 100644 --- a/misp_modules/modules/expansion/urlhaus.py +++ b/misp_modules/modules/expansion/urlhaus.py @@ -35,6 +35,11 @@ class URLhaus(): results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {'results': results} + def parse_error(self, query_status): + if query_status == 'no_results': + return {'error': f'No results found on URLhaus for this {self.attribute.type} attribute'} + return {'error': f'Error encountered during the query of URLhaus: {query_status}'} + class HostQuery(URLhaus): def __init__(self, attribute): @@ -45,9 +50,12 @@ class HostQuery(URLhaus): def query_api(self): response = requests.post(self.url, data={'host': self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) if 'urls' in response and response['urls']: for url in response['urls']: self.misp_event.add_attribute(type='url', value=url['url']) + return self.get_result() class PayloadQuery(URLhaus): @@ -63,6 +71,8 @@ class PayloadQuery(URLhaus): if hasattr(self.attribute, 'object_id') and hasattr(self.attribute, 'event_id') and self.attribute.event_id != '0': file_object.id = self.attribute.object_id response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256' for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')): if response[key]: @@ -81,6 +91,7 @@ class PayloadQuery(URLhaus): file_object.add_attribute(_filename_, **{'type': _filename_, 'value': url[_filename_]}) if any((file_object.attributes, file_object.references)): self.misp_event.add_object(**file_object) + return self.get_result() class UrlQuery(URLhaus): @@ -100,6 +111,8 @@ class UrlQuery(URLhaus): def query_api(self): response = requests.post(self.url, data={'url': self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) if 'payloads' in response and response['payloads']: for payload in response['payloads']: file_object = self._create_file_object(payload) @@ -109,6 +122,7 @@ class UrlQuery(URLhaus): self.misp_event.add_object(**vt_object) if any((file_object.attributes, file_object.references)): self.misp_event.add_object(**file_object) + return self.get_result() _misp_type_mapping = {'url': UrlQuery, 'md5': PayloadQuery, 'sha256': PayloadQuery, @@ -122,8 +136,7 @@ def handler(q=False): request = json.loads(q) attribute = request['attribute'] urlhaus_parser = _misp_type_mapping[attribute['type']](attribute) - urlhaus_parser.query_api() - return urlhaus_parser.get_result() + return urlhaus_parser.query_api() def introspection(): diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index 77a99a2..f47a2e3 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -3,12 +3,12 @@ import json import requests misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512", "url"], +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], 'format': 'misp_standard'} # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '4', 'author': 'Hannah Ward', - 'description': 'Get information from virustotal', + 'description': 'Get information from VirusTotal', 'module-type': ['expansion']} # config fields that your code expects from the site admin @@ -25,8 +25,7 @@ class VirusTotalParser(object): self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip, 'domain': self.parse_domain, 'hostname': self.parse_domain, 'md5': self.parse_hash, 'sha1': self.parse_hash, - 'sha256': self.parse_hash, 'sha512': self.parse_hash, - 'url': self.parse_url} + 'sha256': self.parse_hash, 'url': self.parse_url} def query_api(self, attribute): self.attribute = MISPAttribute() diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index 69c2c85..e7c2e96 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -3,10 +3,10 @@ import json import requests misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512", "url"], +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], 'format': 'misp_standard'} moduleinfo = {'version': '1', 'author': 'Christian Studer', - 'description': 'Get information from virustotal public API v2.', + 'description': 'Get information from VirusTotal public API v2.', 'module-type': ['expansion', 'hover']} moduleconfig = ['apikey'] @@ -85,8 +85,10 @@ class DomainQuery(VirusTotalParser): whois_object = MISPObject(whois) whois_object.add_attribute('text', type='text', value=query_result[whois]) self.misp_event.add_object(**whois_object) - siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) - self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) + if 'domain_siblings' in query_result: + siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) + if 'subdomains' in query_result: + self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) self.parse_urls(query_result) def parse_siblings(self, domain): @@ -153,7 +155,7 @@ ip = ('ip', IpQuery) file = ('resource', HashQuery) misp_type_mapping = {'domain': domain, 'hostname': domain, 'ip-src': ip, 'ip-dst': ip, 'md5': file, 'sha1': file, 'sha256': file, - 'sha512': file, 'url': ('resource', UrlQuery)} + 'url': ('resource', UrlQuery)} def parse_error(status_code): diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 77dec0d..1b0e1d0 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ __all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', - 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport'] + 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph'] diff --git a/misp_modules/modules/export_mod/vt_graph.py b/misp_modules/modules/export_mod/vt_graph.py new file mode 100644 index 0000000..70c1952 --- /dev/null +++ b/misp_modules/modules/export_mod/vt_graph.py @@ -0,0 +1,113 @@ +'''Export MISP event to VirusTotal Graph.''' + + +import base64 +import json +from vt_graph_parser.importers.pymisp_response import from_pymisp_response + + +misperrors = { + 'error': 'Error' +} +moduleinfo = { + 'version': '0.1', + 'author': 'VirusTotal', + 'description': 'Send event to VirusTotal Graph', + 'module-type': ['export'] +} +mispattributes = { + 'input': [ + 'hostname', + 'domain', + 'ip-src', + 'ip-dst', + 'md5', + 'sha1', + 'sha256', + 'url', + 'filename|md5', + 'filename' + ] +} +moduleconfig = [ + 'vt_api_key', + 'fetch_information', + 'private', + 'fetch_vt_enterprise', + 'expand_one_level', + 'user_editors', + 'user_viewers', + 'group_editors', + 'group_viewers' +] + + +def handler(q=False): + """Expansion handler. + + Args: + q (bool, optional): module data. Defaults to False. + + Returns: + [str]: VirusTotal graph links + """ + if not q: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('vt_api_key'): + misperrors['error'] = 'A VirusTotal api key is required for this module.' + return misperrors + + config = request['config'] + + api_key = config.get('vt_api_key') + fetch_information = config.get('fetch_information') or False + private = config.get('private') or False + fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False + expand_one_level = config.get('expand_one_level') or False + + user_editors = config.get('user_editors') + if user_editors: + user_editors = user_editors.split(',') + user_viewers = config.get('user_viewers') + if user_viewers: + user_viewers = user_viewers.split(',') + group_editors = config.get('group_editors') + if group_editors: + group_editors = group_editors.split(',') + group_viewers = config.get('group_viewers') + if group_viewers: + group_viewers = group_viewers.split(',') + + graphs = from_pymisp_response( + request, api_key, fetch_information=fetch_information, + private=private, fetch_vt_enterprise=fetch_vt_enterprise, + user_editors=user_editors, user_viewers=user_viewers, + group_editors=group_editors, group_viewers=group_viewers, + expand_node_one_level=expand_one_level) + links = [] + + for graph in graphs: + graph.save_graph() + links.append(graph.get_ui_link()) + + # This file will contains one VirusTotal graph link for each exported event + file_data = str(base64.b64encode( + bytes('\n'.join(links), 'utf-8')), 'utf-8') + return {'response': [], 'data': file_data} + + +def introspection(): + modulesetup = { + 'responseType': 'application/txt', + 'outputFileExtension': 'txt', + 'userConfig': {}, + 'inputSource': [] + } + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 96e42b1..e4dc2e5 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -34,7 +34,7 @@ misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fi class CsvParser(): - def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes): + def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes, categories): self.misp_event = MISPEvent() self.header = header self.has_header = has_header @@ -42,11 +42,16 @@ class CsvParser(): self.data = data self.from_misp = from_misp self.MISPtypes = MISPtypes + self.categories = categories self.fields_number = len(self.header) - self.__score_mapping = {0: self.__create_standard_misp, + self.__score_mapping = {0: self.__create_standard_attribute, 1: self.__create_attribute_with_ids, 2: self.__create_attribute_with_tags, - 3: self.__create_attribute_with_ids_and_tags} + 3: self.__create_attribute_with_ids_and_tags, + 4: self.__create_attribute_check_category, + 5: self.__create_attribute_check_category_and_ids, + 6: self.__create_attribute_check_category_and_tags, + 7: self.__create_attribute_check_category_with_ids_and_tags} def parse_csv(self): if self.from_misp: @@ -165,35 +170,68 @@ class CsvParser(): # Utility functions # ################################################################################ + def __create_attribute_check_category(self, line, indexes): + attribute = self.__create_standard_attribute(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_and_ids(self, line, indexes): + attribute = self.__create_attribute_with_ids(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_and_tags(self, line, indexes): + attribute = self.__create_attribute_with_tags(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_with_ids_and_tags(self, line, indexes): + attribute = self.__create_attribute_with_ids_and_tags(line, indexes) + self.__check_category(attribute) + return attribute + def __create_attribute_with_ids(self, line, indexes): - attribute = self.__create_standard_misp(line, indexes) - return self.__deal_with_ids(attribute) + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_ids(attribute) + return attribute def __create_attribute_with_ids_and_tags(self, line, indexes): - attribute = self.__deal_with_ids(self.__create_standard_misp(line, indexes)) - return self.__deal_with_tags(attribute) + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_ids(attribute) + self.__deal_with_tags(attribute) + return attribute def __create_attribute_with_tags(self, line, indexes): - attribute = self.__create_standard_misp(line, indexes) - return self.__deal_with_tags(attribute) + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_tags(attribute) + return attribute - def __create_standard_misp(self, line, indexes): + def __create_standard_attribute(self, line, indexes): return {self.header[index]: line[index] for index in indexes if line[index]} + def __check_category(self, attribute): + category = attribute['category'] + if category in self.categories: + return + if category.capitalize() in self.categories: + attribute['category'] = category.capitalize() + return + del attribute['category'] + @staticmethod def __deal_with_ids(attribute): attribute['to_ids'] = True if attribute['to_ids'] == '1' else False - return attribute @staticmethod def __deal_with_tags(attribute): attribute['Tag'] = [{'name': tag.strip()} for tag in attribute['Tag'].split(',')] - return attribute def __get_score(self): score = 1 if 'to_ids' in self.header else 0 if 'attribute_tag' in self.header: score += 2 + if 'category' in self.header: + score += 4 return score def __finalize_results(self): @@ -218,7 +256,11 @@ def handler(q=False): return False request = json.loads(q) if request.get('data'): - data = base64.b64decode(request['data']).decode('utf-8') + try: + data = base64.b64decode(request['data']).decode('utf-8') + except UnicodeDecodeError: + misperrors['error'] = "Input is not valid UTF-8" + return misperrors else: misperrors['error'] = "Unsupported attributes type" return misperrors @@ -241,7 +283,8 @@ def handler(q=False): header = misp_standard_csv_header descFilename = os.path.join(pymisp_path[0], 'data/describeTypes.json') with open(descFilename, 'r') as f: - MISPtypes = json.loads(f.read())['result'].get('types') + description = json.loads(f.read())['result'] + MISPtypes = description['types'] for h in header: if not any((h in MISPtypes, h in misp_extended_csv_header, h in ('', ' ', '_', 'object_id'))): misperrors['error'] = 'Wrong header field: {}. Please use a header value that can be recognized by MISP (or alternatively skip it using a whitespace).'.format(h) @@ -256,7 +299,7 @@ def handler(q=False): wrong_types = tuple(wrong_type for wrong_type in ('type', 'value') if wrong_type in header) misperrors['error'] = 'Error with the following header: {}. It contains the following field(s): {}, which is(are) already provided by the usage of at least on MISP attribute type in the header.'.format(header, 'and'.join(wrong_types)) return misperrors - csv_parser = CsvParser(header, has_header, delimiter, data, from_misp, MISPtypes) + csv_parser = CsvParser(header, has_header, delimiter, data, from_misp, MISPtypes, description['categories']) # build the attributes result = csv_parser.parse_csv() if 'error' in result: diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py index d1c4d19..0753167 100644 --- a/misp_modules/modules/import_mod/joe_import.py +++ b/misp_modules/modules/import_mod/joe_import.py @@ -4,10 +4,20 @@ import json from joe_parser import JoeParser misperrors = {'error': 'Error'} -userConfig = {} +userConfig = { + "Import PE": { + "type": "Boolean", + "message": "Import PE Information", + }, + "Mitre Att&ck": { + "type": "Boolean", + "message": "Import Mitre Att&ck techniques", + }, +} + inputSource = ['file'] -moduleinfo = {'version': '0.1', 'author': 'Christian Studer', +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', 'description': 'Import for Joe Sandbox JSON reports', 'module-type': ['import']} @@ -18,10 +28,16 @@ def handler(q=False): if q is False: return False q = json.loads(q) + config = { + "import_pe": bool(int(q["config"]["Import PE"])), + "mitre_attack": bool(int(q["config"]["Mitre Att&ck"])), + } + data = base64.b64decode(q.get('data')).decode('utf-8') if not data: return json.dumps({'success': 0}) - joe_parser = JoeParser() + + joe_parser = JoeParser(config) joe_parser.parse_data(json.loads(data)['analysis']) joe_parser.finalize_results() return {'results': joe_parser.results} diff --git a/misp_modules/modules/import_mod/lastline_import.py b/misp_modules/modules/import_mod/lastline_import.py index ff26b93..37f6249 100644 --- a/misp_modules/modules/import_mod/lastline_import.py +++ b/misp_modules/modules/import_mod/lastline_import.py @@ -29,10 +29,9 @@ moduleinfo = { } moduleconfig = [ - "api_key", - "api_token", "username", "password", + "verify_ssl", ] @@ -65,24 +64,25 @@ def handler(q=False): # Parse the init parameters try: - auth_data = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_request(request) + config = request["config"] + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) analysis_link = request["config"]["analysis_link"] # The API url changes based on the analysis link host name - api_url = lastline_api.get_api_url_from_link(analysis_link) + api_url = lastline_api.get_portal_url_from_task_link(analysis_link) except Exception as e: misperrors["error"] = "Error parsing configuration: {}".format(e) return misperrors # Parse the call parameters try: - task_uuid = lastline_api.get_uuid_from_link(analysis_link) + task_uuid = lastline_api.get_uuid_from_task_link(analysis_link) except (KeyError, ValueError) as e: misperrors["error"] = "Error processing input parameters: {}".format(e) return misperrors # Make the API calls try: - api_client = lastline_api.LastlineCommunityAPIClient(api_url, auth_data) + api_client = lastline_api.PortalClient(api_url, auth_data, verify_ssl=config.get('verify_ssl', True).lower() in ("true")) response = api_client.get_progress(task_uuid) if response.get("completed") != 1: raise ValueError("Analysis is not finished yet.") @@ -122,7 +122,7 @@ if __name__ == "__main__": args = parser.parse_args() c = configparser.ConfigParser() c.read(args.config_file) - a = lastline_api.LastlineCommunityHTTPClient.get_login_params_from_conf(c, args.section_name) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) j = json.dumps( { diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 93ee69d..801769a 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -99,7 +99,7 @@ class TestExpansions(unittest.TestCase): def test_bgpranking(self): query = {"module": "bgpranking", "AS": "13335"} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET - Cloudflare, Inc., US') + self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET, US') def test_btc_steroids(self): query = {"module": "btc_steroids", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} @@ -235,11 +235,12 @@ class TestExpansions(unittest.TestCase): self.assertTrue(value.startswith('{"ip":"1.1.1.1","status":"ok"')) def test_ipasn(self): - query = {"module": "ipasn", "ip-dst": "1.1.1.1"} + query = {"module": "ipasn", + "attribute": {"type": "ip-src", + "value": "149.13.33.14", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} response = self.misp_modules_post(query) - key = list(self.get_values(response)['response'].keys())[0] - entry = self.get_values(response)['response'][key]['asn'] - self.assertEqual(entry, '13335') + self.assertEqual(self.get_object(response), 'asn') def test_macaddess_io(self): module_name = 'macaddress_io' @@ -362,6 +363,15 @@ class TestExpansions(unittest.TestCase): response = self.misp_modules_post(query) self.assertEqual(self.get_values(response), '1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh') + def test_ransomcoindb(self): + query = {"module": "ransomcoindb", + "attributes": {"type": "btc", + "value": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + if 'ransomcoindb' not in self.configs: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), "Ransomcoindb API key is missing") + def test_rbl(self): query = {"module": "rbl", "ip-src": "8.8.8.8"} response = self.misp_modules_post(query)