diff --git a/.gitignore b/.gitignore index cde1a60..3d994af 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,9 @@ __pycache__ build/ dist/ misp_modules.egg-info/ + +# For MKDOCS +docs/expansion* +docs/import_mod* +docs/export_mod* +site* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 0a3a912..4d551b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,31 +6,51 @@ services: cache: pip python: - - "3.4" - - "3.5" - - "3.5-dev" - "3.6" - "3.6-dev" + - "3.7-dev" + - "3.8-dev" + +before_install: + - docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/ install: - - pip install -U nose codecov pytest - - pip install -U -r REQUIREMENTS + - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr libfuzzy-dev libcaca-dev liblua5.3-dev + - pip install pipenv + - pipenv install --dev + # install gtcaca + - git clone git://github.com/stricaud/gtcaca.git + - mkdir -p gtcaca/build + - pushd gtcaca/build + - cmake .. && make + - sudo make install + - popd + # install pyfaup + - git clone https://github.com/stricaud/faup.git + - pushd faup/build + - cmake .. && make + - sudo make install + - popd + - sudo ldconfig + - pushd faup/src/lib/bindings/python - pip install . + - popd script: - - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 & + - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 & - pid=$! - sleep 5 - - nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - pipenv run nosetests --with-coverage --cover-package=misp_modules + - kill -s KILL $pid - pushd ~/ - - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & + - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & - pid=$! - popd - sleep 5 - - nosetests --with-coverage --cover-package=misp_modules - - kill -s INT $pid + - pipenv run nosetests --with-coverage --cover-package=misp_modules + - kill -s KILL $pid + - pipenv run flake8 --ignore=E501,W503,E226 misp_modules after_success: - - coverage combine .coverage* - - codecov + - pipenv run coverage combine .coverage* + - pipenv run codecov diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1cff13f --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +# https://www.mkdocs.org/user-guide/deploying-your-docs/ + +.PHONY: prepare_docs generate_docs ci_generate_docs test_docs + +prepare_docs: + cd doc; python generate_documentation.py + mkdir -p docs/expansion/logos docs/export_mod/logos docs/import_mod/logos + cp -R doc/logos/* docs/expansion/logos + cp -R doc/logos/* docs/export_mod/logos + cp -R doc/logos/* docs/import_mod/logos + cp LICENSE docs/license.md + +install_requirements: + pip install -r docs/REQUIREMENTS.txt + +generate_docs: prepare_docs + mkdocs build + +deploy: + mkdocs gh-deploy + +test_docs: prepare_docs + mkdocs serve + + +# DOCKER make commands +generate_docs_docker: prepare_docs + docker run --rm -it -v $(PWD):/docs squidfunk/mkdocs-material build + +deploy_docker: + docker run --rm -it -v $(PWD):/docs -v /home/$(whoami)/.docker:/root/.docker:ro squidfunk/mkdocs-material gh-deploy + +test_docs_docker: prepare_docs + docker run --rm -it -p 8000:8000 -v $(PWD):/docs squidfunk/mkdocs-material diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..1169368 --- /dev/null +++ b/Pipfile @@ -0,0 +1,66 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +nose = "*" +codecov = "*" +pytest = "*" +flake8 = "*" + +[packages] +dnspython = "*" +requests = {extras = ["security"],version = "*"} +urlarchiver = "*" +passivetotal = "*" +pypdns = "*" +pypssl = "*" +pyeupi = "*" +uwhois = {editable = true,git = "https://github.com/Rafiot/uwhoisd.git",ref = "testing",subdirectory = "client"} +pymisp = {editable = true,extras = ["fileobjects,openioc,pdfexport"],git = "https://github.com/MISP/PyMISP.git"} +pyonyphe = {editable = true,git = "https://github.com/sebdraven/pyonyphe"} +pydnstrails = {editable = true,git = "https://github.com/sebdraven/pydnstrails"} +pytesseract = "*" +pygeoip = "*" +beautifulsoup4 = "*" +oauth2 = "*" +yara-python = "==3.8.1" +sigmatools = "*" +stix2-patterns = "*" +maclookup = "*" +vulners = "*" +blockchain = "*" +reportlab = "*" +pyintel471 = {editable = true,git = "https://github.com/MISP/PyIntel471.git"} +shodan = "*" +Pillow = "*" +Wand = "*" +SPARQLWrapper = "*" +domaintools_api = "*" +misp-modules = {editable = true,path = "."} +pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} +pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} +backscatter = "*" +pyzbar = "*" +opencv-python = "*" +np = "*" +ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git/"} +python-pptx = "*" +python-docx = "*" +ezodf = "*" +pandas = "*" +pandas_ods_reader = "*" +pdftotext = "*" +lxml = "*" +xlrd = "*" +idna-ssl = {markers = "python_version < '3.7'"} +jbxapi = "*" +geoip2 = "*" +apiosintDS = "*" +assemblyline_client = "*" +vt-graph-api = "*" +trustar = "*" + +[requires] +python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..73aeaed --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,1363 @@ +{ + "_meta": { + "hash": { + "sha256": "c2d937b384431e4b313b29bb02db0bd1d3a866ddcb7c6e91cbfa34f88d351b59" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "aiohttp": { + "hashes": [ + "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e", + "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326", + "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a", + "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654", + "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a", + "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4", + "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17", + "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec", + "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd", + "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48", + "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59", + "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965" + ], + "markers": "python_full_version >= '3.5.3'", + "version": "==3.6.2" + }, + "antlr4-python3-runtime": { + "hashes": [ + "sha256:15793f5d0512a372b4e7d2284058ad32ce7dd27126b105fb0b2245130445db33" + ], + "markers": "python_version >= '3'", + "version": "==4.8" + }, + "apiosintds": { + "hashes": [ + "sha256:d8ab4dcf75a9989572cd6808773b56fdf535b6080d6041d98e911e6c5eb31f3c" + ], + "index": "pypi", + "version": "==1.8.3" + }, + "argparse": { + "hashes": [ + "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", + "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" + ], + "version": "==1.4.0" + }, + "assemblyline-client": { + "hashes": [ + "sha256:6f45cab3be3ec60984a5c2049d46dac80d4e3d4f3d9538220518a44c7a6ddb15", + "sha256:971371065f2b41027325bf9fa9c72960262a446c7e08bda57865d34dcc4108b0" + ], + "index": "pypi", + "version": "==3.7.3" + }, + "async-timeout": { + "hashes": [ + "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", + "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" + ], + "markers": "python_full_version >= '3.5.3'", + "version": "==3.0.1" + }, + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.3.0" + }, + "backscatter": { + "hashes": [ + "sha256:7a0d1aa3661635de81e2a09b15d53e35cbe399a111cc58a70925f80e6874abd3", + "sha256:afb0efcf5d2551dac953ec4c38fb710b274b8e811775650e02c1ef42cafb14c8" + ], + "index": "pypi", + "version": "==0.2.4" + }, + "beautifulsoup4": { + "hashes": [ + "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", + "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", + "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" + ], + "index": "pypi", + "version": "==4.9.1" + }, + "blockchain": { + "hashes": [ + "sha256:dbaa3eebb6f81b4245005739da802c571b09f98d97eb66520afd95d9ccafebe2" + ], + "index": "pypi", + "version": "==1.4.4" + }, + "certifi": { + "hashes": [ + "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", + "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + ], + "version": "==2020.6.20" + }, + "cffi": { + "hashes": [ + "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff", + "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b", + "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac", + "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0", + "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384", + "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26", + "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6", + "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b", + "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e", + "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd", + "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2", + "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66", + "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc", + "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8", + "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55", + "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4", + "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5", + "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d", + "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78", + "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa", + "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793", + "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f", + "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a", + "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f", + "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30", + "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f", + "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3", + "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c" + ], + "version": "==1.14.0" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "click-plugins": { + "hashes": [ + "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", + "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" + ], + "version": "==1.1.1" + }, + "colorama": { + "hashes": [ + "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", + "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.3" + }, + "configparser": { + "hashes": [ + "sha256:2ca44140ee259b5e3d8aaf47c79c36a7ab0d5e94d70bd4105c03ede7a20ea5a1", + "sha256:cffc044844040c7ce04e9acd1838b5f2e5fa3170182f6fda4d2ea8b0099dbadd" + ], + "markers": "python_version >= '3.6'", + "version": "==5.0.0" + }, + "cryptography": { + "hashes": [ + "sha256:091d31c42f444c6f519485ed528d8b451d1a0c7bf30e8ca583a0cac44b8a0df6", + "sha256:18452582a3c85b96014b45686af264563e3e5d99d226589f057ace56196ec78b", + "sha256:1dfa985f62b137909496e7fc182dac687206d8d089dd03eaeb28ae16eec8e7d5", + "sha256:1e4014639d3d73fbc5ceff206049c5a9a849cefd106a49fa7aaaa25cc0ce35cf", + "sha256:22e91636a51170df0ae4dcbd250d318fd28c9f491c4e50b625a49964b24fe46e", + "sha256:3b3eba865ea2754738616f87292b7f29448aec342a7c720956f8083d252bf28b", + "sha256:651448cd2e3a6bc2bb76c3663785133c40d5e1a8c1a9c5429e4354201c6024ae", + "sha256:726086c17f94747cedbee6efa77e99ae170caebeb1116353c6cf0ab67ea6829b", + "sha256:844a76bc04472e5135b909da6aed84360f522ff5dfa47f93e3dd2a0b84a89fa0", + "sha256:88c881dd5a147e08d1bdcf2315c04972381d026cdb803325c03fe2b4a8ed858b", + "sha256:96c080ae7118c10fcbe6229ab43eb8b090fccd31a09ef55f83f690d1ef619a1d", + "sha256:a0c30272fb4ddda5f5ffc1089d7405b7a71b0b0f51993cb4e5dbb4590b2fc229", + "sha256:bb1f0281887d89617b4c68e8db9a2c42b9efebf2702a3c5bf70599421a8623e3", + "sha256:c447cf087cf2dbddc1add6987bbe2f767ed5317adb2d08af940db517dd704365", + "sha256:c4fd17d92e9d55b84707f4fd09992081ba872d1a0c610c109c18e062e06a2e55", + "sha256:d0d5aeaedd29be304848f1c5059074a740fa9f6f26b84c5b63e8b29e73dfc270", + "sha256:daf54a4b07d67ad437ff239c8a4080cfd1cc7213df57d33c97de7b4738048d5e", + "sha256:e993468c859d084d5579e2ebee101de8f5a27ce8e2159959b6673b418fd8c785", + "sha256:f118a95c7480f5be0df8afeb9a11bd199aa20afab7a96bcf20409b411a3a85f0" + ], + "version": "==2.9.2" + }, + "decorator": { + "hashes": [ + "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", + "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" + ], + "version": "==4.4.2" + }, + "deprecated": { + "hashes": [ + "sha256:525ba66fb5f90b07169fdd48b6373c18f1ee12728ca277ca44567a367d9d7f74", + "sha256:a766c1dccb30c5f6eb2b203f87edd1d8588847709c78589e1521d769addc8218" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.2.10" + }, + "dnspython": { + "hashes": [ + "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01", + "sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d" + ], + "index": "pypi", + "version": "==1.16.0" + }, + "domaintools-api": { + "hashes": [ + "sha256:62e2e688d14dbd7ca51a44bd0a8490aa69c712895475598afbdbb1e1e15bf2f2", + "sha256:fe75e3cc86e7e2904b06d8e94b1986e721fdce85d695c87d1140403957e4c989" + ], + "index": "pypi", + "version": "==0.5.2" + }, + "enum-compat": { + "hashes": [ + "sha256:3677daabed56a6f724451d585662253d8fb4e5569845aafa8bb0da36b1a8751e", + "sha256:88091b617c7fc3bbbceae50db5958023c48dc40b50520005aa3bf27f8f7ea157" + ], + "version": "==0.0.3" + }, + "ez-setup": { + "hashes": [ + "sha256:303c5b17d552d1e3fb0505d80549f8579f557e13d8dc90e5ecef3c07d7f58642" + ], + "version": "==0.9" + }, + "ezodf": { + "hashes": [ + "sha256:000da534f689c6d55297a08f9e2ed7eada9810d194d31d164388162fb391122d" + ], + "index": "pypi", + "version": "==0.3.2" + }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.18.2" + }, + "futures": { + "hashes": [ + "sha256:3a44f286998ae64f0cc083682fcfec16c406134a81a589a5de445d7bb7c2751b", + "sha256:51ecb45f0add83c806c68e4b06106f90db260585b25ef2abfcda0bd95c0132fd", + "sha256:c4884a65654a7c45435063e14ae85280eb1f111d94e542396717ba9828c4337f" + ], + "version": "==3.1.1" + }, + "geoip2": { + "hashes": [ + "sha256:5869e987bc54c0d707264fec4710661332cc38d2dca5a7f9bb5362d0308e2ce0", + "sha256:99ec12d2f1271a73a0a4a2b663fe6ce25fd02289c0a6bef05c0a1c3b30ee95a4" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "httplib2": { + "hashes": [ + "sha256:8af66c1c52c7ffe1aa5dc4bcd7c769885254b0756e6e69f953c7f0ab49a70ba3", + "sha256:ca2914b015b6247791c4866782fa6042f495b94401a0f0bd3e1d6e0ba2236782" + ], + "version": "==0.18.1" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "idna-ssl": { + "hashes": [ + "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c" + ], + "markers": "python_version < '3.7'", + "version": "==1.1.0" + }, + "isodate": { + "hashes": [ + "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", + "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" + ], + "version": "==0.6.0" + }, + "jbxapi": { + "hashes": [ + "sha256:58eb7d77a52169309e2322ce874c0f00a7900a515d1d0798ff85973cdb2766e3" + ], + "index": "pypi", + "version": "==3.8.0" + }, + "jsonschema": { + "hashes": [ + "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", + "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" + ], + "version": "==3.2.0" + }, + "lief": { + "hashes": [ + "sha256:276cc63ec12a21bdf01b8d30962692c17499788234f0765247ca7a35872097ec", + "sha256:3e6baaeb52bdc339b5f19688b58fd8d5778b92e50221f920cedfa2bec1f4d5c2", + "sha256:45e5c592b57168c447698381d927eb2386ffdd52afe0c48245f848d4cc7ee05a", + "sha256:6547752b5db105cd41c9fa65d0d7452a4d7541b77ffee716b46246c6d81e172f", + "sha256:83b51e01627b5982662f9550ac1230758aa56945ed86829e4291932d98417da3", + "sha256:895599194ea7495bf304e39317b04df20cccf799fc2751867cc1aa4997cfcdae", + "sha256:8a91cee2568306fe1d2bf84341b459c85368317d01d7105fa49e4f4ede837076", + "sha256:913b36a67707dc2afa72f117bab9856ea3f434f332b04a002a0f9723c8779320", + "sha256:9f604a361a3b1b3ed5fdafed0321c5956cb3b265b5efe2250d1bf8911a80c65b", + "sha256:a487fe7234c04bccd58223dbb79214421176e2629814c7a4a887764cceb5be7c", + "sha256:bc8488fb0661cb436fe4bb4fe947d0f9aa020e9acaed233ccf01ab04d888c68a", + "sha256:bddbf333af62310a10cb738a1df1dc2b140dd9c663b55ba3500c10c249d416d2", + "sha256:cce48d7c97cef85e01e6cfeff55f2068956b5c0257eb9c2d2c6d15e33dd1e4fc", + "sha256:f8b3f66956c56b582b3adc573bf2a938c25fb21c8894b373a113e24c494fc982" + ], + "version": "==0.10.1" + }, + "lxml": { + "hashes": [ + "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", + "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", + "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", + "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", + "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", + "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", + "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", + "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", + "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", + "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", + "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", + "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", + "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", + "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", + "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", + "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", + "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", + "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", + "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", + "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", + "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", + "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", + "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", + "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", + "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", + "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", + "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" + ], + "index": "pypi", + "version": "==4.5.1" + }, + "maclookup": { + "hashes": [ + "sha256:33bf8eaebe3b1e4ab4ae9277dd93c78024e0ebf6b3c42f76c37695bc26ce287a", + "sha256:795e792cd3e03c9bdad77e52904d43ff71d3ac03b360443f99d4bae08a6bffef" + ], + "index": "pypi", + "version": "==1.0.3" + }, + "maxminddb": { + "hashes": [ + "sha256:f4d28823d9ca23323d113dc7af8db2087aa4f657fafc64ff8f7a8afda871425b" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.5.4" + }, + "misp-modules": { + "editable": true, + "path": "." + }, + "multidict": { + "hashes": [ + "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", + "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", + "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", + "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", + "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", + "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", + "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", + "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", + "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", + "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", + "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", + "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", + "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", + "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", + "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", + "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", + "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" + ], + "markers": "python_version >= '3.5'", + "version": "==4.7.6" + }, + "np": { + "hashes": [ + "sha256:781265283f3823663ad8fb48741aae62abcf4c78bc19f908f8aa7c1d3eb132f8" + ], + "index": "pypi", + "version": "==1.0.2" + }, + "numpy": { + "hashes": [ + "sha256:13af0184177469192d80db9bd02619f6fa8b922f9f327e077d6f2a6acb1ce1c0", + "sha256:26a45798ca2a4e168d00de75d4a524abf5907949231512f372b217ede3429e98", + "sha256:26f509450db547e4dfa3ec739419b31edad646d21fb8d0ed0734188b35ff6b27", + "sha256:30a59fb41bb6b8c465ab50d60a1b298d1cd7b85274e71f38af5a75d6c475d2d2", + "sha256:33c623ef9ca5e19e05991f127c1be5aeb1ab5cdf30cb1c5cf3960752e58b599b", + "sha256:356f96c9fbec59974a592452ab6a036cd6f180822a60b529a975c9467fcd5f23", + "sha256:3c40c827d36c6d1c3cf413694d7dc843d50997ebffbc7c87d888a203ed6403a7", + "sha256:4d054f013a1983551254e2379385e359884e5af105e3efe00418977d02f634a7", + "sha256:63d971bb211ad3ca37b2adecdd5365f40f3b741a455beecba70fd0dde8b2a4cb", + "sha256:658624a11f6e1c252b2cd170d94bf28c8f9410acab9f2fd4369e11e1cd4e1aaf", + "sha256:76766cc80d6128750075378d3bb7812cf146415bd29b588616f72c943c00d598", + "sha256:7b57f26e5e6ee2f14f960db46bd58ffdca25ca06dd997729b1b179fddd35f5a3", + "sha256:7b852817800eb02e109ae4a9cef2beda8dd50d98b76b6cfb7b5c0099d27b52d4", + "sha256:8cde829f14bd38f6da7b2954be0f2837043e8b8d7a9110ec5e318ae6bf706610", + "sha256:a2e3a39f43f0ce95204beb8fe0831199542ccab1e0c6e486a0b4947256215632", + "sha256:a86c962e211f37edd61d6e11bb4df7eddc4a519a38a856e20a6498c319efa6b0", + "sha256:a8705c5073fe3fcc297fb8e0b31aa794e05af6a329e81b7ca4ffecab7f2b95ef", + "sha256:b6aaeadf1e4866ca0fdf7bb4eed25e521ae21a7947c59f78154b24fc7abbe1dd", + "sha256:be62aeff8f2f054eff7725f502f6228298891fd648dc2630e03e44bf63e8cee0", + "sha256:c2edbb783c841e36ca0fa159f0ae97a88ce8137fb3a6cd82eae77349ba4b607b", + "sha256:cbe326f6d364375a8e5a8ccb7e9cd73f4b2f6dc3b2ed205633a0db8243e2a96a", + "sha256:d34fbb98ad0d6b563b95de852a284074514331e6b9da0a9fc894fb1cdae7a79e", + "sha256:d97a86937cf9970453c3b62abb55a6475f173347b4cde7f8dcdb48c8e1b9952d", + "sha256:dd53d7c4a69e766e4900f29db5872f5824a06827d594427cf1a4aa542818b796", + "sha256:df1889701e2dfd8ba4dc9b1a010f0a60950077fb5242bb92c8b5c7f1a6f2668a", + "sha256:fa1fe75b4a9e18b66ae7f0b122543c42debcf800aaafa0212aaff3ad273c2596" + ], + "markers": "python_version >= '3.6'", + "version": "==1.19.0" + }, + "oauth2": { + "hashes": [ + "sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6", + "sha256:c006a85e7c60107c7cc6da1b184b5c719f6dd7202098196dfa6e55df669b59bf" + ], + "index": "pypi", + "version": "==1.9.0.post1" + }, + "odtreader": { + "editable": true, + "git": "https://github.com/cartertemm/ODTReader.git/", + "ref": "49d6938693f6faa3ff09998f86dba551ae3a996b" + }, + "opencv-python": { + "hashes": [ + "sha256:068928b9907b3d3acd53b129062557d6b0b8b324bfade77f028dbe4dfe482bf2", + "sha256:0e7c91718351449877c2d4141abd64eee1f9c8701bcfaf4e8627bd023e303368", + "sha256:1ab92d807427641ec45d28d5907426aa06b4ffd19c5b794729c74d91cd95090e", + "sha256:31d634dea1b47c231b88d384f90605c598214d0c596443c9bb808e11761829f5", + "sha256:5fdfc0bed37315f27d30ae5ae9bad47ec0a0a28c323739d39c8177b7e0929238", + "sha256:6fa8fac14dd5af4819d475f74af12d65fbbfa391d3110c3a972934a5e6507c24", + "sha256:78cc89ebc808886eb190626ee71ab65e47f374121975f86e4d5f7c0e3ce6bed9", + "sha256:7c7ba11720d01cb572b4b6945d115cb103462c0a28996b44d4e540d06e6a90fd", + "sha256:a37ee82f1b8ed4b4645619c504311e71ce845b78f40055e78d71add5fab7da82", + "sha256:aa3ca1f54054e1c6439fdf1edafa2a2b940a9eaac04a7b422a1cba9b2d7b9690", + "sha256:b9de3dd956574662712da8e285f0f54327959a4e95b96a2847d3c3f5ee7b96e2", + "sha256:c0087b428cef9a32d977390656d91b02245e0e91f909870492df7e39202645dd", + "sha256:d87e506ab205799727f0efa34b3888949bf029a3ada5eb000ff632606370ca6e", + "sha256:d8a55585631f9c9eca4b1a996e9732ae023169cf2f46f69e4518d67d96198226", + "sha256:dcb8da8c5ebaa6360c8555547a4c7beb6cd983dd95ba895bb78b86cc8cf3de2b", + "sha256:e2206bb8c17c0f212f1f356d82d72dd090ff4651994034416da9bf0c29732825", + "sha256:e3c57d6579e5bf85f564d6d48d8ee89868b92879a9232b9975d072c346625e92", + "sha256:ef89cbf332b9a735d8a82e9ff79cc743eeeb775ad1cd7100bc2aa2429b496f07", + "sha256:f45c1c3cdda1857bedd4dfe0bbd49c9419af0cc57f33490341edeae97d18f037", + "sha256:fb3c855347310788e4286b867997be354c55535597966ed5dac876d9166013a4" + ], + "index": "pypi", + "version": "==4.2.0.34" + }, + "pandas": { + "hashes": [ + "sha256:02f1e8f71cd994ed7fcb9a35b6ddddeb4314822a0e09a9c5b2d278f8cb5d4096", + "sha256:13f75fb18486759da3ff40f5345d9dd20e7d78f2a39c5884d013456cec9876f0", + "sha256:35b670b0abcfed7cad76f2834041dcf7ae47fd9b22b63622d67cdc933d79f453", + "sha256:4c73f373b0800eb3062ffd13d4a7a2a6d522792fa6eb204d67a4fad0a40f03dc", + "sha256:5759edf0b686b6f25a5d4a447ea588983a33afc8a0081a0954184a4a87fd0dd7", + "sha256:5a7cf6044467c1356b2b49ef69e50bf4d231e773c3ca0558807cdba56b76820b", + "sha256:69c5d920a0b2a9838e677f78f4dde506b95ea8e4d30da25859db6469ded84fa8", + "sha256:8778a5cc5a8437a561e3276b85367412e10ae9fff07db1eed986e427d9a674f8", + "sha256:9871ef5ee17f388f1cb35f76dc6106d40cb8165c562d573470672f4cdefa59ef", + "sha256:9c31d52f1a7dd2bb4681d9f62646c7aa554f19e8e9addc17e8b1b20011d7522d", + "sha256:ab8173a8efe5418bbe50e43f321994ac6673afc5c7c4839014cf6401bbdd0705", + "sha256:ae961f1f0e270f1e4e2273f6a539b2ea33248e0e3a11ffb479d757918a5e03a9", + "sha256:b3c4f93fcb6e97d993bf87cdd917883b7dab7d20c627699f360a8fb49e9e0b91", + "sha256:c9410ce8a3dee77653bc0684cfa1535a7f9c291663bd7ad79e39f5ab58f67ab3", + "sha256:f69e0f7b7c09f1f612b1f8f59e2df72faa8a6b41c5a436dde5b615aaf948f107", + "sha256:faa42a78d1350b02a7d2f0dbe3c80791cf785663d6997891549d0f86dc49125e" + ], + "index": "pypi", + "version": "==1.0.5" + }, + "pandas-ods-reader": { + "hashes": [ + "sha256:d2d6e4f9cd2850da32808bbc68d433a337911058387992026d3987ead1f4a7c8", + "sha256:d4d6781cc46e782e265b48681416f636e7659343dec948c6fccc4236af6fa1e6" + ], + "index": "pypi", + "version": "==0.0.7" + }, + "passivetotal": { + "hashes": [ + "sha256:2944974d380a41f19f8fbb3d7cbfc8285479eb81092940b57bf0346d66706a05", + "sha256:a0cbea84b0bd6e9f3694ddeb447472b3d6f09e28940a7a0388456b8cf6a8e478", + "sha256:e35bf2cbccb385795a67d66f180d14ce9136cf1611b1c3da8a1055a1aced6264" + ], + "index": "pypi", + "version": "==1.0.31" + }, + "pdftotext": { + "hashes": [ + "sha256:d37864049581fb13cdcf7b23d4ea23dac7ca2e9c646e8ecac1a39275ab1cae03" + ], + "index": "pypi", + "version": "==2.1.4" + }, + "pillow": { + "hashes": [ + "sha256:0295442429645fa16d05bd567ef5cff178482439c9aad0411d3f0ce9b88b3a6f", + "sha256:06aba4169e78c439d528fdeb34762c3b61a70813527a2c57f0540541e9f433a8", + "sha256:09d7f9e64289cb40c2c8d7ad674b2ed6105f55dc3b09aa8e4918e20a0311e7ad", + "sha256:0a80dd307a5d8440b0a08bd7b81617e04d870e40a3e46a32d9c246e54705e86f", + "sha256:1ca594126d3c4def54babee699c055a913efb01e106c309fa6b04405d474d5ae", + "sha256:25930fadde8019f374400f7986e8404c8b781ce519da27792cbe46eabec00c4d", + "sha256:431b15cffbf949e89df2f7b48528be18b78bfa5177cb3036284a5508159492b5", + "sha256:52125833b070791fcb5710fabc640fc1df07d087fc0c0f02d3661f76c23c5b8b", + "sha256:5e51ee2b8114def244384eda1c82b10e307ad9778dac5c83fb0943775a653cd8", + "sha256:612cfda94e9c8346f239bf1a4b082fdd5c8143cf82d685ba2dba76e7adeeb233", + "sha256:6d7741e65835716ceea0fd13a7d0192961212fd59e741a46bbed7a473c634ed6", + "sha256:6edb5446f44d901e8683ffb25ebdfc26988ee813da3bf91e12252b57ac163727", + "sha256:725aa6cfc66ce2857d585f06e9519a1cc0ef6d13f186ff3447ab6dff0a09bc7f", + "sha256:8dad18b69f710bf3a001d2bf3afab7c432785d94fcf819c16b5207b1cfd17d38", + "sha256:94cf49723928eb6070a892cb39d6c156f7b5a2db4e8971cb958f7b6b104fb4c4", + "sha256:97f9e7953a77d5a70f49b9a48da7776dc51e9b738151b22dacf101641594a626", + "sha256:9ad7f865eebde135d526bb3163d0b23ffff365cf87e767c649550964ad72785d", + "sha256:a060cf8aa332052df2158e5a119303965be92c3da6f2d93b6878f0ebca80b2f6", + "sha256:c79f9c5fb846285f943aafeafda3358992d64f0ef58566e23484132ecd8d7d63", + "sha256:c92302a33138409e8f1ad16731568c55c9053eee71bb05b6b744067e1b62380f", + "sha256:d08b23fdb388c0715990cbc06866db554e1822c4bdcf6d4166cf30ac82df8c41", + "sha256:d350f0f2c2421e65fbc62690f26b59b0bcda1b614beb318c81e38647e0f673a1", + "sha256:ec29604081f10f16a7aea809ad42e27764188fc258b02259a03a8ff7ded3808d", + "sha256:edf31f1150778abd4322444c393ab9c7bd2af271dd4dafb4208fb613b1f3cdc9", + "sha256:f7e30c27477dffc3e85c2463b3e649f751789e0f6c8456099eea7ddd53be4a8a", + "sha256:ffe538682dc19cc542ae7c3e504fdf54ca7f86fb8a135e59dd6bc8627eae6cce" + ], + "index": "pypi", + "version": "==7.2.0" + }, + "progressbar2": { + "hashes": [ + "sha256:13f228cf357f94cdef933c91c1e771e52e1b1931dbae48267be8fcdc2ae2ce36", + "sha256:27abf038efe5b1b5dd91ecc5f704bc88683c1e2a0b2c0fee04de80a648634a0c" + ], + "version": "==3.51.4" + }, + "psutil": { + "hashes": [ + "sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058", + "sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953", + "sha256:60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4", + "sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e", + "sha256:73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f", + "sha256:75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38", + "sha256:a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e", + "sha256:d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8", + "sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26", + "sha256:e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5", + "sha256:f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==5.7.0" + }, + "pybgpranking": { + "editable": true, + "git": "https://github.com/D4-project/BGP-Ranking.git/", + "ref": "fd9c0e03af9b61d4bf0b67ac73c7208a55178a54", + "subdirectory": "client" + }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.20" + }, + "pycryptodome": { + "hashes": [ + "sha256:02e51e1d5828d58f154896ddfd003e2e7584869c275e5acbe290443575370fba", + "sha256:03d5cca8618620f45fd40f827423f82b86b3a202c8d44108601b0f5f56b04299", + "sha256:0e24171cf01021bc5dc17d6a9d4f33a048f09d62cc3f62541e95ef104588bda4", + "sha256:132a56abba24e2e06a479d8e5db7a48271a73a215f605017bbd476d31f8e71c1", + "sha256:1e655746f539421d923fd48df8f6f40b3443d80b75532501c0085b64afed9df5", + "sha256:2b998dc45ef5f4e5cf5248a6edfcd8d8e9fb5e35df8e4259b13a1b10eda7b16b", + "sha256:360955eece2cd0fa694a708d10303c6abd7b39614fa2547b6bd245da76198beb", + "sha256:39ef9fb52d6ec7728fce1f1693cb99d60ce302aeebd59bcedea70ca3203fda60", + "sha256:4350a42028240c344ee855f032c7d4ad6ff4f813bfbe7121547b7dc579ecc876", + "sha256:50348edd283afdccddc0938cdc674484533912ba8a99a27c7bfebb75030aa856", + "sha256:54bdedd28476dea8a3cd86cb67c0df1f0e3d71cae8022354b0f879c41a3d27b2", + "sha256:55eb61aca2c883db770999f50d091ff7c14016f2769ad7bca3d9b75d1d7c1b68", + "sha256:6276478ada411aca97c0d5104916354b3d740d368407912722bd4d11aa9ee4c2", + "sha256:67dcad1b8b201308586a8ca2ffe89df1e4f731d5a4cdd0610cc4ea790351c739", + "sha256:709b9f144d23e290b9863121d1ace14a72e01f66ea9c903fbdc690520dfdfcf0", + "sha256:8063a712fba642f78d3c506b0896846601b6de7f5c3d534e388ad0cc07f5a149", + "sha256:80d57177a0b7c14d4594c62bbb47fe2f6309ad3b0a34348a291d570925c97a82", + "sha256:a207231a52426de3ff20f5608f0687261a3329d97a036c51f7d4c606a6f30c23", + "sha256:abc2e126c9490e58a36a0f83516479e781d83adfb134576a5cbe5c6af2a3e93c", + "sha256:b56638d58a3a4be13229c6a815cd448f9e3ce40c00880a5398471b42ee86f50e", + "sha256:bcd5b8416e73e4b0d48afba3704d8c826414764dafaed7a1a93c442188d90ccc", + "sha256:bec2bcdf7c9ce7f04d718e51887f3b05dc5c1cfaf5d2c2e9065ecddd1b2f6c9a", + "sha256:c8bf40cf6e281a4378e25846924327e728a887e8bf0ee83b2604a0f4b61692e8", + "sha256:d8074c8448cfd0705dfa71ca333277fce9786d0b9cac75d120545de6253f996a", + "sha256:dd302b6ae3965afeb5ef1b0d92486f986c0e65183cd7835973f0b593800590e6", + "sha256:de6e1cd75677423ff64712c337521e62e3a7a4fc84caabbd93207752e831a85a", + "sha256:ef39c98d9b8c0736d91937d193653e47c3b19ddf4fc3bccdc5e09aaa4b0c5d21", + "sha256:f521178e5a991ffd04182ed08f552daca1affcb826aeda0e1945cd989a9d4345", + "sha256:f78a68c2c820e4731e510a2df3eef0322f24fde1781ced970bf497b6c7d92982", + "sha256:fbe65d5cfe04ff2f7684160d50f5118bdefb01e3af4718eeb618bfed40f19d94" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.9.8" + }, + "pycryptodomex": { + "hashes": [ + "sha256:06f5a458624c9b0e04c0086c7f84bcc578567dab0ddc816e0476b3057b18339f", + "sha256:1714675fb4ac29a26ced38ca22eb8ffd923ac851b7a6140563863194d7158422", + "sha256:17272d06e4b2f6455ee2cbe93e8eb50d9450a5dc6223d06862ee1ea5d1235861", + "sha256:2199708ebeed4b82eb45b10e1754292677f5a0df7d627ee91ea01290b9bab7e6", + "sha256:2275a663c9e744ee4eace816ef2d446b3060554c5773a92fbc79b05bf47debda", + "sha256:2710fc8d83b3352b370db932b3710033b9d630b970ff5aaa3e7458b5336e3b32", + "sha256:35b9c9177a9fe7288b19dd41554c9c8ca1063deb426dd5a02e7e2a7416b6bd11", + "sha256:3caa32cf807422adf33c10c88c22e9e2e08b9d9d042f12e1e25fe23113dd618f", + "sha256:48cc2cfc251f04a6142badeb666d1ff49ca6fdfc303fd72579f62b768aaa52b9", + "sha256:4ae6379350a09339109e9b6f419bb2c3f03d3e441f4b0f5b8ca699d47cc9ff7e", + "sha256:4e0b27697fa1621c6d3d3b4edeec723c2e841285de6a8d378c1962da77b349be", + "sha256:58e19560814dabf5d788b95a13f6b98279cf41a49b1e49ee6cf6c79a57adb4c9", + "sha256:8044eae59301dd392fbb4a7c5d64e1aea8ef0be2540549807ecbe703d6233d68", + "sha256:89be1bf55e50116fe7e493a7c0c483099770dd7f81b87ac8d04a43b1a203e259", + "sha256:8fcdda24dddf47f716400d54fc7f75cadaaba1dd47cc127e59d752c9c0fc3c48", + "sha256:914fbb18e29c54585e6aa39d300385f90d0fa3b3cc02ed829b08f95c1acf60c2", + "sha256:93a75d1acd54efed314b82c952b39eac96ce98d241ad7431547442e5c56138aa", + "sha256:9fd758e5e2fe02d57860b85da34a1a1e7037155c4eadc2326fc7af02f9cae214", + "sha256:a2bc4e1a2e6ca3a18b2e0be6131a23af76fecb37990c159df6edc7da6df913e3", + "sha256:a2ee8ba99d33e1a434fcd27d7d0aa7964163efeee0730fe2efc9d60edae1fc71", + "sha256:b2d756620078570d3f940c84bc94dd30aa362b795cce8b2723300a8800b87f1c", + "sha256:c0d085c8187a1e4d3402f626c9e438b5861151ab132d8761d9c5ce6491a87761", + "sha256:c990f2c58f7c67688e9e86e6557ed05952669ff6f1343e77b459007d85f7df00", + "sha256:ccbbec59bf4b74226170c54476da5780c9176bae084878fc94d9a2c841218e34", + "sha256:dc2bed32c7b138f1331794e454a953360c8cedf3ee62ae31f063822da6007489", + "sha256:e070a1f91202ed34c396be5ea842b886f6fa2b90d2db437dc9fb35a26c80c060", + "sha256:e42860fbe1292668b682f6dabd225fbe2a7a4fa1632f0c39881c019e93dea594", + "sha256:e4e1c486bf226822c8dceac81d0ec59c0a2399dbd1b9e04f03c3efa3605db677", + "sha256:ea4d4b58f9bc34e224ef4b4604a6be03d72ef1f8c486391f970205f6733dbc46", + "sha256:f60b3484ce4be04f5da3777c51c5140d3fe21cdd6674f2b6568f41c8130bcdeb" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.9.8" + }, + "pydeep": { + "hashes": [ + "sha256:22866eb422d1d5907f8076ee792da65caecb172425d27576274e2a8eacf6afc1" + ], + "version": "==0.4" + }, + "pydnstrails": { + "editable": true, + "git": "https://github.com/sebdraven/pydnstrails", + "ref": "48c1f740025c51289f43a24863d1845ff12fd21a" + }, + "pyeupi": { + "hashes": [ + "sha256:2309c61ac2ef0eafabd6e9f32a0078069ffbba0e113ebc6b51cffc1869094472", + "sha256:a0798a4a52601b0840339449a1bbf2aa2bc180d8f82a979022954e05fcb5bfba" + ], + "index": "pypi", + "version": "==1.1" + }, + "pygeoip": { + "hashes": [ + "sha256:1938b9dac7b00d77f94d040b9465ea52c938f3fcdcd318b5537994f3c16aef96", + "sha256:f22c4e00ddf1213e0fae36dc60b46ee7c25a6339941ec1a975539014c1f9a96d" + ], + "index": "pypi", + "version": "==0.3.2" + }, + "pyintel471": { + "editable": true, + "git": "https://github.com/MISP/PyIntel471.git", + "ref": "0df8d51f1c1425de66714b3a5a45edb69b8cc2fc" + }, + "pyipasnhistory": { + "editable": true, + "git": "https://github.com/D4-project/IPASN-History.git/", + "ref": "fc5e48608afc113e101ca6421bf693b7b9753f9e", + "subdirectory": "client" + }, + "pymisp": { + "editable": true, + "extras": [ + "fileobjects", + "openioc", + "pdfexport" + ], + "git": "https://github.com/MISP/PyMISP.git", + "ref": "ec28820cf491ca7d385477996afa0547eb6b6830" + }, + "pyonyphe": { + "editable": true, + "git": "https://github.com/sebdraven/pyonyphe", + "ref": "1ce15581beebb13e841193a08a2eb6f967855fcb" + }, + "pyopenssl": { + "hashes": [ + "sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504", + "sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507" + ], + "version": "==19.1.0" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" + }, + "pypdns": { + "hashes": [ + "sha256:640a7e08c3e1e6d6cf378bc7bf48225d847a9c86583c196994fb15acc20ec6f4", + "sha256:9cd2d42ed5e9e4ff7ea29b3947b133a74b0fe0f548ca4c9fac26c0b8f8b750d5" + ], + "index": "pypi", + "version": "==1.5.1" + }, + "pypssl": { + "hashes": [ + "sha256:4dbe772aefdf4ab18934d83cde79e2fc5d5ba9d2b4153dc419a63faab3432643" + ], + "index": "pypi", + "version": "==2.1" + }, + "pyrsistent": { + "hashes": [ + "sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3" + ], + "version": "==0.16.0" + }, + "pytesseract": { + "hashes": [ + "sha256:afd8a5cdf8ab5d35690efbe71cbf5f89419f668ea8dde7649149815d5c5a899a" + ], + "index": "pypi", + "version": "==0.3.4" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.1" + }, + "python-docx": { + "hashes": [ + "sha256:bc76ecac6b2d00ce6442a69d03a6f35c71cd72293cd8405a7472dfe317920024" + ], + "index": "pypi", + "version": "==0.8.10" + }, + "python-magic": { + "hashes": [ + "sha256:356efa93c8899047d1eb7d3eb91e871ba2f5b1376edbaf4cc305e3c872207355", + "sha256:b757db2a5289ea3f1ced9e60f072965243ea43a2221430048fd8cacab17be0ce" + ], + "version": "==0.4.18" + }, + "python-pptx": { + "hashes": [ + "sha256:a857d69e52d7e8a8fb32fca8182fdd4a3c68c689de8d4e4460e9b4a95efa7bc4" + ], + "index": "pypi", + "version": "==0.6.18" + }, + "python-utils": { + "hashes": [ + "sha256:ebaadab29d0cb9dca0a82eab9c405f5be5125dbbff35b8f32cc433fa498dbaa7", + "sha256:f21fc09ff58ea5ebd1fd2e8ef7f63e39d456336900f26bdc9334a03a3f7d8089" + ], + "version": "==2.4.0" + }, + "pytz": { + "hashes": [ + "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", + "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + ], + "version": "==2019.3" + }, + "pyyaml": { + "hashes": [ + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" + ], + "version": "==5.3.1" + }, + "pyzbar": { + "hashes": [ + "sha256:0e204b904e093e5e75aa85e0203bb0e02888105732a509b51f31cff400f34265", + "sha256:496249b546be70ec98c0ff0ad9151e73daaffff129266df86150a15dcd8dac4c", + "sha256:7d6c01d2c0a352fa994aa91b5540d1caeaeaac466656eb41468ca5df33be9f2e" + ], + "index": "pypi", + "version": "==0.1.8" + }, + "pyzipper": { + "hashes": [ + "sha256:49813f1d415bdd7c87064009b9270c6dd0a96da770cfe57df2c6d2d84a6c085a", + "sha256:bfdc65f616278b38ef03c6ea5a1aca7499caf98cbfcd47fc44f73e68f4307145" + ], + "markers": "python_version >= '3.5'", + "version": "==0.3.3" + }, + "rdflib": { + "hashes": [ + "sha256:78149dd49d385efec3b3adfbd61c87afaf1281c30d3fcaf1b323b34f603fb155", + "sha256:88208ea971a87886d60ae2b1a4b2cdc263527af0454c422118d43fe64b357877" + ], + "version": "==5.0.0" + }, + "redis": { + "hashes": [ + "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", + "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==3.5.3" + }, + "reportlab": { + "hashes": [ + "sha256:0f0c2d98e213d51ae527c0301364d3376cb05f6c47251368a9abd4c3197fcefa", + "sha256:1425c7ea60b8691a881ae21ea0f6907a1dc480d84204ccbfea6da41fbee8f594", + "sha256:204f1d245875ab3d076b37c1a18ac8d2e3222842e13cfa282bcd95282be239e5", + "sha256:21627b57249303bf9b5a633099d058ae9f8625fd6f90cfe79348c48fd5a242cd", + "sha256:2e8e3242f80b79f2470f1b5979abbdb41f31b1333543b830749100342f837d40", + "sha256:2eced06dec3f36135c626b9823649ef9cac95c5634d1bc743a15ee470027483b", + "sha256:3472aa0b74a3b2f252dce823f3c3ba6af8a24de0c1729441deaaf50bed6de9f9", + "sha256:3f0353ffefd3afc0061f4794ef608d6c6f32e69816885f4d45c625c20d8eaf5b", + "sha256:4a9f4540a8eddf56d900ceeb8136bd0ca866c208ba3dcbcde73f07405dbadfba", + "sha256:4eea1afb4aa89780734f44175508edff82928fdf460c9bd60bc719dd99041dc3", + "sha256:5803ffebd36de1ada417f50ce65d379ea5a0bf1a2e8f5d5710a031b3b349b726", + "sha256:58f5f72fc8e5932dedcf24789908a81c6b1e13ea4d63bd9a9a39dc698d8c3321", + "sha256:5b588e5f251c76a8d3589023d1c369c7968e0efe2b38ad5948f665edbf6f9e8b", + "sha256:5d922768fe11a58d80694852aba7389d613c15eb1871c5581a2f075996873d57", + "sha256:5d98f297c5cdd5bc0ccf5697c20b03602ee3378c97938d20312662b27cd9a1d6", + "sha256:66d1d96e97a562614943ecb9daf438e392b3d0b033bd5f4a8098ab616dd877da", + "sha256:670650970c7ba7164cf6340bcd182e7e933eff5d65183af98ee77b40cc25a438", + "sha256:67bb95af7bc8ad7925d299f310d15d556d3e7026fe1b60d8e290454604ae0a85", + "sha256:9c999f5d1a600c4970ba293789b6da14e02e3763a8d3d9abe42dcafa8a5318e9", + "sha256:9d62bef5347063a984e63410fa5a69f1d2cc2fdf8d6ed3d0b9d4ea2ccb4b4154", + "sha256:a14a0d603727b6be2e549c52dd42678ab2d06d2721d4580199e3161843e59298", + "sha256:a3a17b46ff1a15eb29370e11796d8914ef4ea67471bdbc4aa9a9eb9284f4e44c", + "sha256:a6d3e20beeba3fd68cec73b8c0785bfa648c06ac76d1f142c60ccb1a8d2506b6", + "sha256:ad7d7003c732f2be42580e3906e92bd9d2aca5e098898c597554be9ca627fad5", + "sha256:af0ee7b50b85543b68b043e61271963ff5671e564e1d620a404c24a24d4f537c", + "sha256:b3eec55274f5ead7e3af2bf0c01b481ffe1b4c6a7dae42b63d85543e9f2f9a0f", + "sha256:b48c21d43a7ab956954591ce3f71db92ce542bb7428db09734425e2b77ac3142", + "sha256:b761905ab85beb79cf7929c9a019f30ad65664e5733d57a30a995e7b9bef06d1", + "sha256:bbae2f054d0f234c3382076efa337802997aca0f3f664e314f65eefb9d694fa9", + "sha256:bd4157d0bc40fb72bb676fc745fdd648022cccaf4ccfbb291af7f48831d0d5d9", + "sha256:bf74cfabf332034f42a54938eb335543cbf92790170300dbe236ba83b7601cd0", + "sha256:c253c8571db2df3886e390a2bfbe917222953054f4643437373b824f64b013cd", + "sha256:ce1277a6acbc62e9966f410f2596ac533ee0cd5df9b69d5fe4406338a169b7d8", + "sha256:ce8f56987e0e456063e311f066a81496b8b9626c846f2cb0ebb554d1a5f40839", + "sha256:d6264a0589ba8032d9c3bdca9a3e87a897ede09b7f6a8ad5e83b57573212e01e", + "sha256:e6fa0c97e3929d00db27e8cf3b2b5771e94f5f179086c4b0e3213dff53637372", + "sha256:f0930f2b6dddd477b3331ec670171a4662336aac1a778e1a30e980a5cbf40b17", + "sha256:f8cb2b4b925ca6b6e4fdefd288a707776ac686c45034f34d4c952f122d11c40b", + "sha256:f9b71539f518323d95850405c49c01fc3d2f0f0b9f3e157de6d2786804fb28a4", + "sha256:fc488e661f99c915362e0373218f8727cecf888eb1b0eb3a8fe1af624a1b9776" + ], + "index": "pypi", + "version": "==3.5.44" + }, + "requests": { + "extras": [ + "security" + ], + "hashes": [ + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + ], + "index": "pypi", + "version": "==2.24.0" + }, + "requests-cache": { + "hashes": [ + "sha256:813023269686045f8e01e2289cc1e7e9ae5ab22ddd1e2849a9093ab3ab7270eb", + "sha256:81e13559baee64677a7d73b85498a5a8f0639e204517b5d05ff378e44a57831a" + ], + "version": "==0.5.2" + }, + "shodan": { + "hashes": [ + "sha256:31b0740ffaf7c5196a26a0b1edf7d271dffe54ea52bb1b34ba87aa231b5c339b" + ], + "index": "pypi", + "version": "==1.23.0" + }, + "sigmatools": { + "hashes": [ + "sha256:5453717e452aa7860c5e6ac80bcee4f398d70956fc2ee9859bc7255067da8736", + "sha256:cdfeb8200c09c0a40ea1a015e57f3b8e2ba62a28352ca05fa015674f640871e3" + ], + "index": "pypi", + "version": "==0.17.0" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" + }, + "socketio-client": { + "hashes": [ + "sha256:540d8ab209154d1d9cdb97c170c589a14f7d7f17e19c14e2f59f0307e6175485" + ], + "version": "==0.5.6" + }, + "soupsieve": { + "hashes": [ + "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", + "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" + ], + "markers": "python_version >= '3.5'", + "version": "==2.0.1" + }, + "sparqlwrapper": { + "hashes": [ + "sha256:17ec44b08b8ae2888c801066249f74fe328eec25d90203ce7eadaf82e64484c7", + "sha256:357ee8a27bc910ea13d77836dbddd0b914991495b8cc1bf70676578155e962a8", + "sha256:8cf6c21126ed76edc85c5c232fd6f77b9f61f8ad1db90a7147cdde2104aff145", + "sha256:c7f9c9d8ebb13428771bc3b6dee54197422507dcc3dea34e30d5dcfc53478dec", + "sha256:d6a66b5b8cda141660e07aeb00472db077a98d22cb588c973209c7336850fb3c" + ], + "index": "pypi", + "version": "==1.8.5" + }, + "stix2-patterns": { + "hashes": [ + "sha256:587a82545680311431e5610036dd6c8c247347a24243fafdafaae2df4d6d7799", + "sha256:7fcb2fa67efeac2a8c493d367c93d0ce6243a10e2eff715ae9f2983e6b32b95d" + ], + "index": "pypi", + "version": "==1.3.0" + }, + "tabulate": { + "hashes": [ + "sha256:ac64cb76d53b1231d364babcd72abbb16855adac7de6665122f97b593f1eb2ba", + "sha256:db2723a20d04bcda8522165c73eea7c300eda74e0ce852d9022e0159d7895007" + ], + "version": "==0.8.7" + }, + "tornado": { + "hashes": [ + "sha256:0fe2d45ba43b00a41cd73f8be321a44936dc1aba233dee979f17a042b83eb6dc", + "sha256:22aed82c2ea340c3771e3babc5ef220272f6fd06b5108a53b4976d0d722bcd52", + "sha256:2c027eb2a393d964b22b5c154d1a23a5f8727db6fda837118a776b29e2b8ebc6", + "sha256:5217e601700f24e966ddab689f90b7ea4bd91ff3357c3600fa1045e26d68e55d", + "sha256:5618f72e947533832cbc3dec54e1dffc1747a5cb17d1fd91577ed14fa0dc081b", + "sha256:5f6a07e62e799be5d2330e68d808c8ac41d4a259b9cea61da4101b83cb5dc673", + "sha256:c58d56003daf1b616336781b26d184023ea4af13ae143d9dda65e31e534940b9", + "sha256:c952975c8ba74f546ae6de2e226ab3cc3cc11ae47baf607459a6728585bb542a", + "sha256:c98232a3ac391f5faea6821b53db8db461157baa788f5d6222a193e9456e1740" + ], + "markers": "python_version >= '3.5'", + "version": "==6.0.4" + }, + "trustar": { + "hashes": [ + "sha256:73336b94012427b66ee61db65fc3c2cea2ed743beaa56cdd5a4c1674ef1a7660" + ], + "index": "pypi", + "version": "==0.3.29" + }, + "tzlocal": { + "hashes": [ + "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44", + "sha256:e2cb6c6b5b604af38597403e9852872d7f534962ae2954c7f35efcb1ccacf4a4" + ], + "version": "==2.1" + }, + "unicodecsv": { + "hashes": [ + "sha256:018c08037d48649a0412063ff4eda26eaa81eff1546dbffa51fa5293276ff7fc" + ], + "version": "==0.14.1" + }, + "url-normalize": { + "hashes": [ + "sha256:1709cb4739e496f9f807a894e361915792f273538e250b1ab7da790544a665c3", + "sha256:1bd7085349dcdf06e52194d0f75ff99fff2eeed0da85a50e4cc2346452c1b8bc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.4.2" + }, + "urlarchiver": { + "hashes": [ + "sha256:652e0890dab58bf62a759656671dcfb9a40eb4a77aac8a8d93154f00360238b5" + ], + "index": "pypi", + "version": "==0.2" + }, + "urllib3": { + "hashes": [ + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.25.9" + }, + "uwhois": { + "editable": true, + "git": "https://github.com/Rafiot/uwhoisd.git", + "ref": "783bba09b5a6964f25566089826a1be4b13f2a22", + "subdirectory": "client" + }, + "validators": { + "hashes": [ + "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" + ], + "version": "==0.14.0" + }, + "vt-graph-api": { + "hashes": [ + "sha256:200c4f5a7c0a518502e890c4f4508a5ea042af9407d2889ef16a17ef11b7d25c", + "sha256:223c1cf32d69e10b5d3e178ec315589c7dfa7d43ccff6630a11ed5c5f498715c" + ], + "index": "pypi", + "version": "==1.0.1" + }, + "vulners": { + "hashes": [ + "sha256:00ff8744d07f398880afc1efcab6dac4abb614c84553fa31b2d439f986b8e0db", + "sha256:90a855915b4fb4dbd0325643d9e643602975fcb931162e5dc2e7778d1daa2fd8", + "sha256:f230bfcd42663326b7c9b8fa117752e26cad4ccca528caaab531c5b592af8cb5" + ], + "index": "pypi", + "version": "==1.5.5" + }, + "wand": { + "hashes": [ + "sha256:d5b75ac13d7485032970926415648586eafeeb1eb62ed6ebd0778358cf9d70e0", + "sha256:df0780b1b54938a43d29279a6588fde11e349550c8958a673d57c26a3e6de7f1" + ], + "index": "pypi", + "version": "==0.6.1" + }, + "websocket-client": { + "hashes": [ + "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549", + "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010" + ], + "version": "==0.57.0" + }, + "wrapt": { + "hashes": [ + "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" + ], + "version": "==1.12.1" + }, + "xlrd": { + "hashes": [ + "sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2", + "sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde" + ], + "index": "pypi", + "version": "==1.2.0" + }, + "xlsxwriter": { + "hashes": [ + "sha256:828b3285fc95105f5b1946a6a015b31cf388bd5378fdc6604e4d1b7839df2e77", + "sha256:82a3b0e73e3913483da23791d1a25e4d2dbb3837d1be4129473526b9a270a5cc" + ], + "version": "==1.2.9" + }, + "yara-python": { + "hashes": [ + "sha256:03e5c5e333c8572e7994b0b11964d515d61a393f23c5e272f8d0e4229f368c58", + "sha256:0423e08bd618752a028ac0405ff8e0103f3a8fd607dde7618a64a4c010c3757b", + "sha256:0a0dd632dcdb347d1a9a8b1f6a83b3a77d5e63f691357ea4021fb1cf1d7ff0a4", + "sha256:728b99627a8072a877eaaa4dafb4eff39d1b14ff4fd70d39f18899ce81e29625", + "sha256:7cb0d5724eccfa52e1bcd352a56cb4dc422aa51f5f6d0945d4f830783927513b", + "sha256:8c76531e89806c0309586dd4863a972d12f1d5d63261c6d4b9331a99859fd1d8", + "sha256:9472676583e212bc4e17c2236634e02273d53c872b350f0571b48e06183de233", + "sha256:9735b680a7d95c1d3f255c351bb067edc62cdb3c0999f7064278cb2c85245405", + "sha256:997f104590167220a9af5564c042ec4d6534261e7b8a5b49655d8dffecc6b8a2", + "sha256:a48e071d02a3699363e628ac899b5b7237803bcb4b512c92ebcb4fb9b1488497", + "sha256:b67c0d75a6519ca357b4b85ede9768c96a81fff20fbc169bd805ff009ddee561" + ], + "index": "pypi", + "version": "==3.8.1" + }, + "yarl": { + "hashes": [ + "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce", + "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6", + "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce", + "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae", + "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d", + "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f", + "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b", + "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b", + "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb", + "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462", + "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea", + "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70", + "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1", + "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a", + "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b", + "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080", + "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2" + ], + "markers": "python_version >= '3.5'", + "version": "==1.4.2" + } + }, + "develop": { + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.3.0" + }, + "certifi": { + "hashes": [ + "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", + "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + ], + "version": "==2020.6.20" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "codecov": { + "hashes": [ + "sha256:491938ad774ea94a963d5d16354c7299e90422a33a353ba0d38d0943ed1d5091", + "sha256:b67bb8029e8340a7bf22c71cbece5bd18c96261fdebc2f105ee4d5a005bc8728", + "sha256:d8b8109f44edad03b24f5f189dac8de9b1e3dc3c791fa37eeaf8c7381503ec34" + ], + "index": "pypi", + "version": "==2.1.7" + }, + "coverage": { + "hashes": [ + "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a", + "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355", + "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65", + "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7", + "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9", + "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1", + "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0", + "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55", + "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c", + "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6", + "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef", + "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019", + "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e", + "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0", + "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf", + "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24", + "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2", + "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c", + "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4", + "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0", + "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd", + "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04", + "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e", + "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730", + "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2", + "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768", + "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796", + "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7", + "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a", + "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489", + "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==5.1" + }, + "flake8": { + "hashes": [ + "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c", + "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208" + ], + "index": "pypi", + "version": "==3.8.3" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "more-itertools": { + "hashes": [ + "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5", + "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2" + ], + "markers": "python_version >= '3.5'", + "version": "==8.4.0" + }, + "nose": { + "hashes": [ + "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", + "sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a", + "sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98" + ], + "index": "pypi", + "version": "==1.3.7" + }, + "packaging": { + "hashes": [ + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.4" + }, + "pluggy": { + "hashes": [ + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.13.1" + }, + "py": { + "hashes": [ + "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", + "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.9.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", + "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.6.0" + }, + "pyflakes": { + "hashes": [ + "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", + "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.2.0" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" + }, + "pytest": { + "hashes": [ + "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1", + "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8" + ], + "index": "pypi", + "version": "==5.4.3" + }, + "requests": { + "extras": [ + "security" + ], + "hashes": [ + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + ], + "index": "pypi", + "version": "==2.24.0" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" + }, + "urllib3": { + "hashes": [ + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.25.9" + }, + "wcwidth": { + "hashes": [ + "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", + "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" + ], + "version": "==0.2.5" + } + } +} diff --git a/README.md b/README.md index 14840ea..67573da 100644 --- a/README.md +++ b/README.md @@ -11,76 +11,171 @@ without modifying core components. The API is available via a simple REST API wh MISP modules support is included in MISP starting from version 2.4.28. -For more information: [Extending MISP with Python modules](https://www.circl.lu/assets/files/misp-training/switch2016/2-misp-modules.pdf) slides from MISP training. +For more information: [Extending MISP with Python modules](https://www.misp-project.org/misp-training/3.1-misp-modules.pdf) slides from MISP training. ## Existing MISP modules ### Expansion modules -* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history. +* [apiosintDS](misp_modules/modules/expansion/apiosintds.py) - a hover and expansion module to query the OSINT.digitalside.it API. +* [API Void](misp_modules/modules/expansion/apivoid.py) - an expansion and hover module to query API Void with a domain attribute. +* [AssemblyLine submit](misp_modules/modules/expansion/assemblyline_submit.py) - an expansion module to submit samples and urls to AssemblyLine. +* [AssemblyLine query](misp_modules/modules/expansion/assemblyline_query.py) - an expansion module to query AssemblyLine and parse the full submission report. +* [Backscatter.io](misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations. +* [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. +* [RansomcoinDB check](misp_modules/modules/expansion/ransomcoindb.py) - An expansion hover module to query the [ransomcoinDB](https://ransomcoindb.concinnity-risks.com): it contains mapping between BTC addresses and malware hashes. Enrich MISP by querying for BTC -> hash or hash -> BTC addresses. +* [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused. +* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. +* [Censys-enrich](misp_modules/modules/expansion/censys_enrich.py) - An expansion and module to retrieve information from censys.io about a particular IP or certificate. * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. -* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. +* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate(s) seen. * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). +* [CVE advanced](misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). +* [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. +* [Cytomic Orion](misp_modules/modules/expansion/cytomic_orion.py) - An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion. +* [DBL Spamhaus](misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. * [DNS](misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. +* [docx-enrich](misp_modules/modules/expansion/docx_enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). * [DomainTools](misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. +* [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate event query language (EQL) from an attribute. [Event Query Language](https://eql.readthedocs.io/en/latest/) * [EUPI](misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). * [Farsight DNSDB Passive DNS](misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [GeoIP](misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [GeoIP_City](misp_modules/modules/expansion/geoip_city.py) - a hover and expansion module to get GeoIP City information from geolite/maxmind. +* [GeoIP_ASN](misp_modules/modules/expansion/geoip_asn.py) - a hover and expansion module to get GeoIP ASN information from geolite/maxmind. +* [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. +* [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? +* [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. +* [Joe Sandbox submit](misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox. +* [Joe Sandbox query](misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data. +* [Lastline submit](misp_modules/modules/expansion/lastline_submit.py) - Submit files and URLs to Lastline. +* [Lastline query](misp_modules/modules/expansion/lastline_query.py) - Query Lastline with the link to an analysis and parse the report. +* [macaddress.io](misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). +* [macvendors](misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. +* [MALWAREbazaar](misp_modules/modules/expansion/malwarebazaar.py) - an expansion module to query MALWAREbazaar with some payload. +* [ocr-enrich](misp_modules/modules/expansion/ocr_enrich.py) - an enrichment module to get OCRized data from images into MISP. +* [ods-enrich](misp_modules/modules/expansion/ods_enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). +* [odt-enrich](misp_modules/modules/expansion/odt_enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). * [onyphe](misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. * [onyphe_full](misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. * [OTX](misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). * [passivetotal](misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [pdf-enrich](misp_modules/modules/expansion/pdf_enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser). +* [pptx-enrich](misp_modules/modules/expansion/pptx_enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser). +* [qrcode](misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values. * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. +* [securitytrails](misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/). * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. +* [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. +* [SophosLabs Intelix](misp_modules/modules/expansion/sophoslabs_intelix.py) - SophosLabs Intelix is an API for Threat Intelligence and Analysis (free tier availible). [SophosLabs](https://aws.amazon.com/marketplace/pp/B07SLZPMCS) * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. +* [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). -* [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key) +* [urlhaus](misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url. +* [urlscan](misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io). +* [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference)) +* [virustotal_public](misp_modules/modules/expansion/virustotal_public.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a public key and a low request rate limit. (More details about the API: [here](https://developers.virustotal.com/reference)) * [VMray](misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray. * [VulnDB](misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/). -* [whois](misp_modules/modules/expansion) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd). +* [Vulners](misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API. +* [whois](misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd). * [wikidata](misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. * [xforce](misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. +* [xlsx-enrich](misp_modules/modules/expansion/xlsx_enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser). +* [YARA query](misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes. * [YARA syntax validator](misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator. ### Export modules -* [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). -* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). -* [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. -* [Simple PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export in PDF (required: asciidoctor-pdf). -* [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. -* [ThreatStream](misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format. +* [CEF](misp_modules/modules/export_mod/cef_export.py) - module to export Common Event Format (CEF). +* [Cisco FireSight Manager ACL rule](misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) - module to export as rule for the Cisco FireSight manager ACL. +* [GoAML export](misp_modules/modules/export_mod/goamlexport.py) - module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). +* [Lite Export](misp_modules/modules/export_mod/liteexport.py) - module to export a lite event. +* [PDF export](misp_modules/modules/export_mod/pdfexport.py) - module to export an event in PDF. +* [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) - module to export applicable attributes from an event to a mass EQL query. +* [Nexthink query format](misp_modules/modules/export_mod/nexthinkexport.py) - module to export in Nexthink query format. +* [osquery](misp_modules/modules/export_mod/osqueryexport.py) - module to export in [osquery](https://osquery.io/) query format. +* [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) - module to export in ThreatConnect CSV format. +* [ThreatStream](misp_modules/modules/export_mod/threatStream_misp_export.py) - module to export in ThreatStream format. +* [VirusTotal Graph](misp_modules/modules/export_mod/vt_graph.py) - Module to create a VirusTotal graph out of an event. ### Import modules -* [CSV import](misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module. -* [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. -* [Email Import](misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. -* [GoAML import](misp_modules/modules/import_mod/) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. -* [OCR](misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. -* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. +* [CSV import](misp_modules/modules/import_mod/csvimport.py) - Customizable CSV import module. +* [Cuckoo JSON](misp_modules/modules/import_mod/cuckooimport.py) - Cuckoo JSON import. +* [Email Import](misp_modules/modules/import_mod/email_import.py) - Email import module for MISP to import basic metadata. +* [GoAML import](misp_modules/modules/import_mod/goamlimport.py) - Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. +* [Joe Sandbox import](misp_modules/modules/import_mod/joe_import.py) - Parse data from a Joe Sandbox json report. +* [Lastline import](misp_modules/modules/import_mod/lastline_import.py) - Module to import Lastline analysis reports. +* [OCR](misp_modules/modules/import_mod/ocr.py) - Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. +* [OpenIOC](misp_modules/modules/import_mod/openiocimport.py) - OpenIOC import based on PyMISP library. * [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. -## How to install and start MISP modules? +## How to install and start MISP modules in a Python virtualenv? (recommended) ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick virtualenv libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev build-essential -y +sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ +chown -R www-data . sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -sudo pip3 install -I -r REQUIREMENTS -sudo pip3 install -I . -sudo vi /etc/rc.local, add this line: `sudo -u www-data misp-modules -s &` -misp-modules #to start the modules +sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS +sudo -u www-data /var/www/MISP/venv/bin/pip install . +# Start misp-modules as a service +sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now misp-modules +/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules +~~~~ + +## How to install and start MISP modules on RHEL-based distributions ? +As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository. + +~~~~bash +sudo yum install rh-ruby22 +sudo yum install openjpeg-devel +sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel gcc-c++ pkgconfig poppler-cpp-devel python-devel redhat-rpm-config +cd /var/www/MISP +git clone https://github.com/MISP/misp-modules.git +cd misp-modules +sudo -u apache /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" +sudo -u apache /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS +sudo -u apache /var/www/MISP/venv/bin/pip install -U . +~~~~ + +Create the service file /etc/systemd/system/misp-modules.service : +~~~~ +echo "[Unit] +Description=MISP's modules +After=misp-workers.service + +[Service] +Type=simple +User=apache +Group=apache +ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s' +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service +~~~~ + +The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service. +Then, enable the misp-modules service and start it: +~~~~bash +systemctl daemon-reload +systemctl enable --now misp-modules ~~~~ ## How to add your own MISP modules? @@ -93,6 +188,8 @@ Create your module in [misp_modules/modules/expansion/](misp_modules/modules/exp Don't forget to return an error key and value if an error is raised to propagate it to the MISP user-interface. +Your module's script name should also be added in the `__all__` list of `/__init__.py` in order for it to be loaded. + ~~~python ... # Checking for required value @@ -184,6 +281,19 @@ def handler(q=False): codecs.encode(src, "rot-13")} ~~~ +#### export module + +For an export module, the `request["data"]` object corresponds to a list of events (dictionaries) to handle. + +Iterating over events attributes is performed using their `Attribute` key. + +~~~python +... +for event in request["data"]: + for attribute in event["Attribute"]: + # do stuff w/ attribute['type'], attribute['value'], ... +... + ### Returning Binary Data If you want to return a file or other data you need to add a data attribute. @@ -380,7 +490,7 @@ Recommended Plugin.Import_ocr_enabled true Enable or disable the ocr In this same menu set any other plugin settings that are required for testing. ## Install misp-module on an offline instance. -First, you need to grab all necessery packages for example like this : +First, you need to grab all necessary packages for example like this : Use pip wheel to create an archive ~~~ @@ -388,7 +498,7 @@ mkdir misp-modules-offline pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/* ~~~ -On offline machine : +On offline machine : ~~~ mkdir misp-modules-bundle tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle @@ -409,7 +519,7 @@ Download a pre-built virtual image from the [MISP training materials](https://ww - Create a Host-Only adapter in VirtualBox - Set your Misp OVA to that Host-Only adapter - Start the virtual machine -- Get the IP address of the virutal machine +- Get the IP address of the virtual machine - SSH into the machine (Login info on training page) - Go into the misp-modules directory @@ -427,16 +537,18 @@ sudo git checkout MyModBranch Remove the contents of the build directory and re-install misp-modules. -~~~python +~~~bash sudo rm -fr build/* -sudo pip3 install --upgrade . +sudo -u www-data /var/www/MISP/venv/bin/pip install --upgrade . ~~~ SSH in with a different terminal and run `misp-modules` with debugging enabled. -~~~python -sudo killall misp-modules -misp-modules -d +~~~bash +# In case misp-modules is not a service do: +# sudo killall misp-modules +sudo systemctl disable --now misp-modules +sudo -u www-data /var/www/MISP/venv/bin/misp-modules -d ~~~ @@ -447,3 +559,14 @@ cd tests/ curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @MY_TEST_FILE.json -X POST cd ../ ~~~ + +## Documentation + +In order to provide documentation about some modules that require specific input / output / configuration, the [doc](doc) directory contains detailed information about the general purpose, requirements, features, input and ouput of each of these modules: + +- ***description** - quick description of the general purpose of the module, as the one given by the moduleinfo +- **requirements** - special libraries needed to make the module work +- **features** - description of the way to use the module, with the required MISP features to make the module give the intended result +- **references** - link(s) giving additional information about the format concerned in the module +- **input** - description of the format of data used in input +- **output** - description of the format given as the result of the module execution diff --git a/REQUIREMENTS b/REQUIREMENTS index a8baf52..73b002a 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,23 +1,112 @@ -tornado -dnspython -requests -urlarchiver -passivetotal -PyPDNS -pypssl -redis -pyeupi -ipasn-redis -asnhistory -git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client -git+https://github.com/MISP/PyMISP.git#egg=pymisp -git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe -pillow -pytesseract -SPARQLWrapper -domaintools_api -pygeoip -bs4 -oauth2 -yara -sigmatools +-i https://pypi.org/simple +-e . +-e git+https://github.com/D4-project/BGP-Ranking.git/@fd9c0e03af9b61d4bf0b67ac73c7208a55178a54#egg=pybgpranking&subdirectory=client +-e git+https://github.com/D4-project/IPASN-History.git/@fc5e48608afc113e101ca6421bf693b7b9753f9e#egg=pyipasnhistory&subdirectory=client +-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 +-e git+https://github.com/MISP/PyMISP.git@b5b40ae2c5225a4b349c26294cfc012309a61352#egg=pymisp[fileobjects,openioc,virustotal,pdfexport] +-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client +-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader +-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails +-e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe +-e git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python +aiohttp==3.4.4 +antlr4-python3-runtime==4.8 ; python_version >= '3' +apiosintds==1.8.3 +argparse==1.4.0 +assemblyline-client==3.7.3 +async-timeout==3.0.1 +attrs==19.3.0 +backscatter==0.2.4 +beautifulsoup4==4.8.2 +blockchain==1.4.4 +censys==0.0.8 +certifi==2019.11.28 +cffi==1.14.0 +chardet==3.0.4 +click-plugins==1.1.1 +click==7.1.1 +colorama==0.4.3 +cryptography==2.8 +decorator==4.4.2 +deprecated==1.2.7 +dnspython==1.16.0 +domaintools-api==0.3.3 +enum-compat==0.0.3 +ez-setup==0.9 +ezodf==0.3.2 +future==0.18.2 +futures==3.1.1 +geoip2==3.0.0 +httplib2==0.17.0 +idna-ssl==1.1.0 ; python_version < '3.7' +idna==2.9 +importlib-metadata==1.6.0 ; python_version < '3.8' +isodate==0.6.0 +jbxapi==3.4.0 +jsonschema==3.2.0 +lief==0.10.1 +lxml==4.5.0 +maclookup==1.0.3 +maxminddb==1.5.2 +multidict==4.7.5 +np==1.0.2 +numpy==1.18.2 +oauth2==1.9.0.post1 +opencv-python==4.2.0.32 +pandas-ods-reader==0.0.7 +pandas==1.0.3 +passivetotal==1.0.31 +pdftotext==2.1.4 +pillow==7.0.0 +progressbar2==3.50.1 +psutil==5.7.0 +pycparser==2.20 +pycryptodome==3.9.7 +pycryptodomex==3.9.7 +pydeep==0.4 +pyeupi==1.0 +pygeoip==0.3.2 +pyopenssl==19.1.0 +pyparsing==2.4.6 +pypdns==1.5.1 +pypssl==2.1 +pyrsistent==0.16.0 +pytesseract==0.3.3 +python-dateutil==2.8.1 +python-docx==0.8.10 +python-magic==0.4.15 +python-pptx==0.6.18 +python-utils==2.4.0 +pytz==2019.3 +pyyaml==5.3.1 +pyzbar==0.1.8 +pyzipper==0.3.1 ; python_version >= '3.5' +rdflib==4.2.2 +redis==3.4.1 +reportlab==3.5.42 +requests-cache==0.5.2 +requests[security]==2.23.0 +shodan==1.22.0 +sigmatools==0.16.0 +six==1.14.0 +socketio-client==0.5.6 +soupsieve==2.0 +sparqlwrapper==1.8.5 +stix2-patterns==1.3.0 +tabulate==0.8.7 +tornado==6.0.4 +trustar==0.3.28 +url-normalize==1.4.1 +urlarchiver==0.2 +urllib3==1.25.8 +validators==0.14.0 +vt-graph-api==1.0.1 +vulners==1.5.5 +wand==0.5.9 +websocket-client==0.57.0 +wrapt==1.12.1 +xlrd==1.2.0 +xlsxwriter==1.2.8 +yara-python==3.8.1 +yarl==1.4.2 +zipp==3.1.0 diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..e173ad4 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,1868 @@ +# MISP modules documentation + +## Expansion Modules + +#### [apiosintds](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/apiosintds.py) + +On demand query API for OSINT.digitalside.it project. +- **features**: +>The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute. +> +>The result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls. +> +>Furthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it +- **input**: +>A domain, ip, url or hash attribute. +- **output**: +>Hashes and urls resulting from the query to OSINT.digitalside.it +- **references**: +>https://osint.digitalside.it/#About +- **requirements**: +>The apiosintDS python library to query the OSINT.digitalside.it API. + +----- + +#### [apivoid](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/apivoid.py) + + + +Module to query APIVoid with some domain attributes. +- **features**: +>This module takes a domain name and queries API Void to get the related DNS records and the SSL certificates. It returns then those pieces of data as MISP objects that can be added to the event. +> +>To make it work, a valid API key and enough credits to proceed 2 queries (0.06 + 0.07 credits) are required. +- **input**: +>A domain attribute. +- **output**: +>DNS records and SSL certificates related to the domain. +- **references**: +>https://www.apivoid.com/ +- **requirements**: +>A valid APIVoid API key with enough credits to proceed 2 queries + +----- + +#### [assemblyline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/assemblyline_query.py) + + + +A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it. +- **features**: +>The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the used-ID and an API key or the password associated to the user-ID. +> +>The submission ID extracted from the submission link is then used to query AssemblyLine and get the full submission report. This report is parsed to extract file objects and the associated IPs, domains or URLs the files are connecting to. +> +>Some more data may be parsed in the future. +- **input**: +>Link of an AssemblyLine submission report. +- **output**: +>MISP attributes & objects parsed from the AssemblyLine submission. +- **references**: +>https://www.cyber.cg.ca/en/assemblyline +- **requirements**: +>assemblyline_client: Python library to query the AssemblyLine rest API. + +----- + +#### [assemblyline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/assemblyline_submit.py) + + + +A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission. +- **features**: +>The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the user-ID and an API key or the password associated to the user-ID. +> +>If the sample or url is correctly submitted, you get then the link of the submission. +- **input**: +>Sample, or url to submit to AssemblyLine. +- **output**: +>Link of the report generated in AssemblyLine. +- **references**: +>https://www.cyber.gc.ca/en/assemblyline +- **requirements**: +>assemblyline_client: Python library to query the AssemblyLine rest API. + +----- + +#### [backscatter_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) + + + +Query backscatter.io (https://backscatter.io/). +- **features**: +>The module takes a source or destination IP address as input and displays the information known by backscatter.io. +> +> +- **input**: +>IP addresses. +- **output**: +>Text containing a history of the IP addresses especially on scanning based on backscatter.io information . +- **references**: +>https://pypi.org/project/backscatter/ +- **requirements**: +>backscatter python library + +----- + +#### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) + +Query BGP Ranking (https://bgpranking-ng.circl.lu/). +- **features**: +>The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking. +> +> +- **input**: +>Autonomous system number. +- **output**: +>Text containing a description of the ASN, its history, and the position in BGP Ranking. +- **references**: +>https://github.com/D4-project/BGP-Ranking/ +- **requirements**: +>pybgpranking python library + +----- + +#### [btc_scam_check](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_scam_check.py) + + + +An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused. +- **features**: +>The module queries a dns blacklist directly with the bitcoin address and get a response if the address has been abused. +- **input**: +>btc address attribute. +- **output**: +>Text to indicate if the BTC address has been abused. +- **references**: +>https://btcblack.it/ +- **requirements**: +>dnspython3: dns python library + +----- + +#### [btc_steroids](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_steroids.py) + + + +An expansion hover module to get a blockchain balance from a BTC address in MISP. +- **input**: +>btc address attribute. +- **output**: +>Text to describe the blockchain balance and the transactions related to the btc address in input. + +----- + +#### [censys_enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/censys_enrich.py) + +An expansion module to enrich attributes in MISP by quering the censys.io API +- **features**: +>This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API. +- **input**: +>IP, domain or certificate fingerprint (md5, sha1 or sha256) +- **output**: +>MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details +- **references**: +>https://www.censys.io +- **requirements**: +>API credentials to censys.io + +----- + +#### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) + + + +Module to access CIRCL Passive DNS. +- **features**: +>This module takes a hostname, domain or ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive DNS REST API to get the asssociated passive dns entries and return them as MISP objects. +> +>To make it work a username and a password are thus required to authenticate to the CIRCL Passive DNS API. +- **input**: +>Hostname, domain, or ip-address attribute. +- **ouput**: +>Passive DNS objects related to the input attribute. +- **references**: +>https://www.circl.lu/services/passive-dns/, https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/ +- **requirements**: +>pypdns: Passive DNS python library, A CIRCL passive DNS account with username & password + +----- + +#### [circl_passivessl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) + + + +Modules to access CIRCL Passive SSL. +- **features**: +>This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive SSL REST API to gather the related certificates and return the corresponding MISP objects. +> +>To make it work a username and a password are required to authenticate to the CIRCL Passive SSL API. +- **input**: +>IP address attribute. +- **output**: +>x509 certificate objects seen by the IP address(es). +- **references**: +>https://www.circl.lu/services/passive-ssl/ +- **requirements**: +>pypssl: Passive SSL python library, A CIRCL passive SSL account with username & password + +----- + +#### [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) + +Module to expand country codes. +- **features**: +>The module takes a domain or a hostname as input, and returns the country it belongs to. +> +>For non country domains, a list of the most common possible extensions is used. +- **input**: +>Hostname or domain attribute. +- **output**: +>Text with the country code the input belongs to. + +----- + +#### [crowdstrike_falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) + + + +Module to query Crowdstrike Falcon. +- **features**: +>This module takes a MISP attribute as input to query a CrowdStrike Falcon API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. +> +>Please note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported. +- **input**: +>A MISP attribute included in the following list: +>- domain +>- email-attachment +>- email-dst +>- email-reply-to +>- email-src +>- email-subject +>- filename +>- hostname +>- ip-src +>- ip-dst +>- md5 +>- mutex +>- regkey +>- sha1 +>- sha256 +>- uri +>- url +>- user-agent +>- whois-registrant-email +>- x509-fingerprint-md5 +- **output**: +>MISP attributes mapped after the CrowdStrike API has been queried, included in the following list: +>- hostname +>- email-src +>- email-subject +>- filename +>- md5 +>- sha1 +>- sha256 +>- ip-dst +>- ip-dst +>- mutex +>- regkey +>- url +>- user-agent +>- x509-fingerprint-md5 +- **references**: +>https://www.crowdstrike.com/products/crowdstrike-falcon-faq/ +- **requirements**: +>A CrowdStrike API access (API id & key) + +----- + +#### [cuckoo_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) + + + +An expansion module to submit files and URLs to Cuckoo Sandbox. +- **features**: +>The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox. +> The returned task id can be used to retrieve results when the analysis completed. +- **input**: +>A malware-sample or attachment for files. A url or domain for URLs. +- **output**: +>A text field containing 'Cuckoo task id: ' +- **references**: +>https://cuckoosandbox.org/, https://cuckoo.sh/docs/ +- **requirements**: +>Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key) + +----- + +#### [cve](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) + + + +An expansion hover module to expand information about CVE id. +- **features**: +>The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to get information about the vulnerability as it is described in the list of CVEs. +- **input**: +>Vulnerability attribute. +- **output**: +>Text giving information about the CVE related to the Vulnerability. +- **references**: +>https://cve.circl.lu/, https://cve.mitre.org/ + +----- + +#### [cytomic_orion](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cytomic_orion.py) + + + +An expansion module to enrich attributes in MISP by quering the Cytomic Orion API +- **features**: +>This module takes an MD5 hash and searches for occurrences of this hash in the Cytomic Orion database. Returns observed files and machines. +- **input**: +>MD5, hash of the sample / malware to search for. +- **output**: +>MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines. +- **references**: +>https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/, https://www.cytomicmodel.com/solutions/ +- **requirements**: +>Access (license) to Cytomic Orion + +----- + +#### [dbl_spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) + + + +Module to check Spamhaus DBL for a domain name. +- **features**: +>This modules takes a domain or a hostname in input and queries the Domain Block List provided by Spamhaus to determine what kind of domain it is. +> +>DBL then returns a response code corresponding to a certain classification of the domain we display. If the queried domain is not in the list, it is also mentionned. +> +>Please note that composite MISP attributes containing domain or hostname are supported as well. +- **input**: +>Domain or hostname attribute. +- **output**: +>Information about the nature of the input. +- **references**: +>https://www.spamhaus.org/faq/section/Spamhaus%20DBL +- **requirements**: +>dnspython3: DNS python3 library + +----- + +#### [dns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) + +A simple DNS expansion service to resolve IP address from domain MISP attributes. +- **features**: +>The module takes a domain of hostname attribute as input, and tries to resolve it. If no error is encountered, the IP address that resolves the domain is returned, otherwise the origin of the error is displayed. +> +>The address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8). +> +>Please note that composite MISP attributes containing domain or hostname are supported as well. +- **input**: +>Domain or hostname attribute. +- **output**: +>IP address resolving the input. +- **requirements**: +>dnspython3: DNS python3 library + +----- + +#### [docx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/docx-enrich.py) + + + +Module to extract freetext from a .docx document. +- **features**: +>The module reads the text contained in a .docx document. The result is passed to the freetext import parser so IoCs can be extracted out of it. +- **input**: +>Attachment attribute containing a .docx document. +- **output**: +>Text and freetext parsed from the document. +- **requirements**: +>docx python library + +----- + +#### [domaintools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) + + + +DomainTools MISP expansion module. +- **features**: +>This module takes a MISP attribute as input to query the Domaintools API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. +> +>Please note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported. +- **input**: +>A MISP attribute included in the following list: +>- domain +>- hostname +>- email-src +>- email-dst +>- target-email +>- whois-registrant-email +>- whois-registrant-name +>- whois-registrant-phone +>- ip-src +>- ip-dst +- **output**: +>MISP attributes mapped after the Domaintools API has been queried, included in the following list: +>- whois-registrant-email +>- whois-registrant-phone +>- whois-registrant-name +>- whois-registrar +>- whois-creation-date +>- text +>- domain +- **references**: +>https://www.domaintools.com/ +- **requirements**: +>Domaintools python library, A Domaintools API access (username & apikey) + +----- + +#### [eql](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eql.py) + + + +EQL query generation for a MISP attribute. +- **features**: +>This module adds a new attribute to a MISP event containing an EQL query for a network or file attribute. +- **input**: +>A filename or ip attribute. +- **output**: +>Attribute containing EQL for a network or file attribute. +- **references**: +>https://eql.readthedocs.io/en/latest/ + +----- + +#### [eupi](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) + + + +A module to query the Phishing Initiative service (https://phishing-initiative.lu). +- **features**: +>This module takes a domain, hostname or url MISP attribute as input to query the Phishing Initiative API. The API returns then the result of the query with some information about the value queried. +> +>Please note that composite attributes containing domain or hostname are also supported. +- **input**: +>A domain, hostname or url MISP attribute. +- **output**: +>Text containing information about the input, resulting from the query on Phishing Initiative. +- **references**: +>https://phishing-initiative.eu/?lang=en +- **requirements**: +>pyeupi: eupi python library, An access to the Phishing Initiative API (apikey & url) + +----- + +#### [farsight_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) + + + +Module to access Farsight DNSDB Passive DNS. +- **features**: +>This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. The API returns then the result of the query with some information about the value queried. +- **input**: +>A domain, hostname or IP address MISP attribute. +- **output**: +>Text containing information about the input, resulting from the query on the Farsight Passive DNS API. +- **references**: +>https://www.farsightsecurity.com/ +- **requirements**: +>An access to the Farsight Passive DNS API (apikey) + +----- + +#### [geoip_country](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) + + + +Module to query a local copy of Maxmind's Geolite database. +- **features**: +>This module takes an IP address MISP attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the location of this IP address. +> +>Please note that composite attributes domain|ip are also supported. +- **input**: +>An IP address MISP Attribute. +- **output**: +>Text containing information about the location of the IP address. +- **references**: +>https://www.maxmind.com/en/home +- **requirements**: +>A local copy of Maxmind's Geolite database + +----- + +#### [greynoise](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/greynoise.py) + + + +Module to access GreyNoise.io API +- **features**: +>The module takes an IP address as input and queries Greynoise for some additional information about it. The result is returned as text. +- **input**: +>An IP address. +- **output**: +>Additional information about the IP fetched from Greynoise API. +- **references**: +>https://greynoise.io/, https://github.com/GreyNoise-Intelligence/api.greynoise.io + +----- + +#### [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) + +A hover module to check hashes against hashdd.com including NSLR dataset. +- **features**: +>This module takes a hash attribute as input to check its known level, using the hashdd API. This information is then displayed. +- **input**: +>A hash MISP attribute (md5). +- **output**: +>Text describing the known level of the hash in the hashdd databases. +- **references**: +>https://hashdd.com/ + +----- + +#### [hibp](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hibp.py) + + + +Module to access haveibeenpwned.com API. +- **features**: +>The module takes an email address as input and queries haveibeenpwned.com API to find additional information about it. This additional information actually tells if any account using the email address has already been compromised in a data breach. +- **input**: +>An email address +- **output**: +>Additional information about the email address. +- **references**: +>https://haveibeenpwned.com/ + +----- + +#### [intelmq_eventdb](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intelmq_eventdb.py) + + + +Module to access intelmqs eventdb. +- **features**: +>/!\ EXPERIMENTAL MODULE, some features may not work /!\ +> +>This module takes a domain, hostname, IP address or Autonomous system MISP attribute as input to query the IntelMQ database. The result of the query gives then additional information about the input. +- **input**: +>A hostname, domain, IP address or AS attribute. +- **output**: +>Text giving information about the input using IntelMQ database. +- **references**: +>https://github.com/certtools/intelmq, https://intelmq.readthedocs.io/en/latest/Developers-Guide/ +- **requirements**: +>psycopg2: Python library to support PostgreSQL, An access to the IntelMQ database (username, password, hostname and database reference) + +----- + +#### [ipasn](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) + +Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History). +- **features**: +>This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object. +- **input**: +>An IP address MISP attribute. +- **output**: +>Asn object(s) objects related to the IP address used as input. +- **references**: +>https://github.com/D4-project/IPASN-History +- **requirements**: +>pyipasnhistory: Python library to access IPASN-history instance + +----- + +#### [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) + +Module to query IPRep data for IP addresses. +- **features**: +>This module takes an IP address attribute as input and queries the database from packetmail.net to get some information about the reputation of the IP. +- **input**: +>An IP address MISP attribute. +- **output**: +>Text describing additional information about the input after a query on the IPRep API. +- **references**: +>https://github.com/mahesh557/packetmail +- **requirements**: +>An access to the packetmail API (apikey) + +----- + +#### [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) + + + +Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and converted into MISP attributes and objects. + +This url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py). +- **features**: +>Module using the new format of modules able to return attributes and objects. +> +>The module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input. +> +>Even if the introspection will allow all kinds of links to call this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return results. +> +>To make it work you will need to fill the 'apikey' configuration with your Joe Sandbox API key and provide a valid link as input. +- **input**: +>Link of a Joe Sandbox sample or url submission. +- **output**: +>MISP attributes & objects parsed from the analysis report. +- **references**: +>https://www.joesecurity.org, https://www.joesandbox.com/ +- **requirements**: +>jbxapi: Joe Sandbox API python3 library + +----- + +#### [joesandbox_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py) + + + +A module to submit files or URLs to Joe Sandbox for an advanced analysis, and return the link of the submission. +- **features**: +>The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted analysis. +> +>It is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link. +- **input**: +>Sample, url (or domain) to submit to Joe Sandbox for an advanced analysis. +- **output**: +>Link of the report generated in Joe Sandbox. +- **references**: +>https://www.joesecurity.org, https://www.joesandbox.com/ +- **requirements**: +>jbxapi: Joe Sandbox API python3 library + +----- + +#### [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) + + + +Query Lastline with an analysis link and parse the report into MISP attributes and objects. +The analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module. +- **features**: +>The module requires a Lastline Portal `username` and `password`. +>The module uses the new format and it is able to return MISP attributes and objects. +>The module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module. +- **input**: +>Link to a Lastline analysis. +- **output**: +>MISP attributes and objects parsed from the analysis report. +- **references**: +>https://www.lastline.com + +----- + +#### [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) + + + +Module to submit a file or URL to Lastline. +- **features**: +>The module requires a Lastline Analysis `api_token` and `key`. +>When the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module. +- **input**: +>File or URL to submit to Lastline. +- **output**: +>Link to the report generated by Lastline. +- **references**: +>https://www.lastline.com + +----- + +#### [macaddress_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) + + + +MISP hover module for macaddress.io +- **features**: +>This module takes a MAC address attribute as input and queries macaddress.io for additional information. +> +>This information contains data about: +>- MAC address details +>- Vendor details +>- Block details +- **input**: +>MAC address MISP attribute. +- **output**: +>Text containing information on the MAC address fetched from a query on macaddress.io. +- **references**: +>https://macaddress.io/, https://github.com/CodeLineFi/maclookup-python +- **requirements**: +>maclookup: macaddress.io python library, An access to the macaddress.io API (apikey) + +----- + +#### [macvendors](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macvendors.py) + + + +Module to access Macvendors API. +- **features**: +>The module takes a MAC address as input and queries macvendors.com for some information about it. The API returns the name of the vendor related to the address. +- **input**: +>A MAC address. +- **output**: +>Additional information about the MAC address. +- **references**: +>https://macvendors.com/, https://macvendors.com/api + +----- + +#### [malwarebazaar](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/malwarebazaar.py) + +Query the MALWAREbazaar API to get additional information about the input hash attribute. +- **features**: +>The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it. The result, if the payload is known on the databases, is at least one file object describing the file the input hash is related to. +> +>The module is using the new format of modules able to return object since the result is one or multiple MISP object(s). +- **input**: +>A hash attribute (md5, sha1 or sha256). +- **output**: +>File object(s) related to the input attribute found on MALWAREbazaar databases. +- **references**: +>https://bazaar.abuse.ch/ + +----- + +#### [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py) + +Module to process some optical character recognition on pictures. +- **features**: +>The module takes an attachment attributes as input and process some optical character recognition on it. The text found is then passed to the Freetext importer to extract potential IoCs. +- **input**: +>A picture attachment. +- **output**: +>Text and freetext fetched from the input picture. +- **requirements**: +>cv2: The OpenCV python library. + +----- + +#### [ods-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ods-enrich.py) + + + +Module to extract freetext from a .ods document. +- **features**: +>The module reads the text contained in a .ods document. The result is passed to the freetext import parser so IoCs can be extracted out of it. +- **input**: +>Attachment attribute containing a .ods document. +- **output**: +>Text and freetext parsed from the document. +- **requirements**: +>ezodf: Python package to create/manipulate OpenDocumentFormat files., pandas_ods_reader: Python library to read in ODS files. + +----- + +#### [odt-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/odt-enrich.py) + + + +Module to extract freetext from a .odt document. +- **features**: +>The module reads the text contained in a .odt document. The result is passed to the freetext import parser so IoCs can be extracted out of it. +- **input**: +>Attachment attribute containing a .odt document. +- **output**: +>Text and freetext parsed from the document. +- **requirements**: +>ODT reader python library. + +----- + +#### [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) + + + +Module to process a query on Onyphe. +- **features**: +>This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted. +- **input**: +>A domain, hostname or IP address MISP attribute. +- **output**: +>MISP attributes fetched from the Onyphe query. +- **references**: +>https://www.onyphe.io/, https://github.com/sebdraven/pyonyphe +- **requirements**: +>onyphe python library, An access to the Onyphe API (apikey) + +----- + +#### [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) + + + +Module to process a full query on Onyphe. +- **features**: +>This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted. +> +>The parsing is here more advanced than the one on onyphe module, and is returning more attributes, since more fields of the query result are watched and parsed. +- **input**: +>A domain, hostname or IP address MISP attribute. +- **output**: +>MISP attributes fetched from the Onyphe query. +- **references**: +>https://www.onyphe.io/, https://github.com/sebdraven/pyonyphe +- **requirements**: +>onyphe python library, An access to the Onyphe API (apikey) + +----- + +#### [otx](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) + + + +Module to get information from AlienVault OTX. +- **features**: +>This module takes a MISP attribute as input to query the OTX Alienvault API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes. +- **input**: +>A MISP attribute included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- md5 +>- sha1 +>- sha256 +>- sha512 +- **output**: +>MISP attributes mapped from the result of the query on OTX, included in the following list: +>- domain +>- ip-src +>- ip-dst +>- text +>- md5 +>- sha1 +>- sha256 +>- sha512 +>- email +- **references**: +>https://www.alienvault.com/open-threat-exchange +- **requirements**: +>An access to the OTX API (apikey) + +----- + +#### [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) + + + + +- **features**: +>The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register +- **input**: +>A MISP attribute included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- x509-fingerprint-sha1 +>- email-src +>- email-dst +>- target-email +>- whois-registrant-email +>- whois-registrant-phone +>- text +>- whois-registrant-name +>- whois-registrar +>- whois-creation-date +- **output**: +>MISP attributes mapped from the result of the query on PassiveTotal, included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- x509-fingerprint-sha1 +>- email-src +>- email-dst +>- target-email +>- whois-registrant-email +>- whois-registrant-phone +>- text +>- whois-registrant-name +>- whois-registrar +>- whois-creation-date +>- md5 +>- sha1 +>- sha256 +>- link +- **references**: +>https://www.passivetotal.org/register +- **requirements**: +>Passivetotal python library, An access to the PassiveTotal API (apikey) + +----- + +#### [pdf-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pdf-enrich.py) + + + +Module to extract freetext from a PDF document. +- **features**: +>The module reads the text contained in a PDF document. The result is passed to the freetext import parser so IoCs can be extracted out of it. +- **input**: +>Attachment attribute containing a PDF document. +- **output**: +>Text and freetext parsed from the document. +- **requirements**: +>pdftotext: Python library to extract text from PDF. + +----- + +#### [pptx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pptx-enrich.py) + + + +Module to extract freetext from a .pptx document. +- **features**: +>The module reads the text contained in a .pptx document. The result is passed to the freetext import parser so IoCs can be extracted out of it. +- **input**: +>Attachment attribute containing a .pptx document. +- **output**: +>Text and freetext parsed from the document. +- **requirements**: +>pptx: Python library to read PowerPoint files. + +----- + +#### [qrcode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/qrcode.py) + +Module to decode QR codes. +- **features**: +>The module reads the QR code and returns the related address, which can be an URL or a bitcoin address. +- **input**: +>A QR code stored as attachment attribute. +- **output**: +>The URL or bitcoin address the QR code is pointing to. +- **requirements**: +>cv2: The OpenCV python library., pyzbar: Python library to read QR codes. + +----- + +#### [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) + +Module to check an IPv4 address against known RBLs. +- **features**: +>This module takes an IP address attribute as input and queries multiple know Real-time Blackhost Lists to check if they have already seen this IP address. +> +>We display then all the information we get from those different sources. +- **input**: +>IP address attribute. +- **output**: +>Text with additional data from Real-time Blackhost Lists about the IP address. +- **references**: +>[RBLs list](https://github.com/MISP/misp-modules/blob/8817de476572a10a9c9d03258ec81ca70f3d926d/misp_modules/modules/expansion/rbl.py#L20) +- **requirements**: +>dnspython3: DNS python3 library + +----- + +#### [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) + +Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. +- **features**: +>The module takes an IP address as input and tries to find the hostname this IP address is resolved into. +> +>The address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8). +> +>Please note that composite MISP attributes containing IP addresses are supported as well. +- **input**: +>An IP address attribute. +- **output**: +>Hostname attribute the input is resolved into. +- **requirements**: +>DNS python library + +----- + +#### [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) + + + +An expansion modules for SecurityTrails. +- **features**: +>The module takes a domain, hostname or IP address attribute as input and queries the SecurityTrails API with it. +> +>Multiple parsing operations are then processed on the result of the query to extract a much information as possible. +> +>From this data extracted are then mapped MISP attributes. +- **input**: +>A domain, hostname or IP address attribute. +- **output**: +>MISP attributes resulting from the query on SecurityTrails API, included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- dns-soa-email +>- whois-registrant-email +>- whois-registrant-phone +>- whois-registrant-name +>- whois-registrar +>- whois-creation-date +>- domain +- **references**: +>https://securitytrails.com/ +- **requirements**: +>dnstrails python library, An access to the SecurityTrails API (apikey) + +----- + +#### [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) + + + +Module to query on Shodan. +- **features**: +>The module takes an IP address as input and queries the Shodan API to get some additional data about it. +- **input**: +>An IP address MISP attribute. +- **output**: +>Text with additional data about the input, resulting from the query on Shodan. +- **references**: +>https://www.shodan.io/ +- **requirements**: +>shodan python library, An access to the Shodan API (apikey) + +----- + +#### [sigma_queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) + + + +An expansion hover module to display the result of sigma queries. +- **features**: +>This module takes a Sigma rule attribute as input and tries all the different queries available to convert it into different formats recognized by SIEMs. +- **input**: +>A Sigma attribute. +- **output**: +>Text displaying results of queries on the Sigma attribute. +- **references**: +>https://github.com/Neo23x0/sigma/wiki +- **requirements**: +>Sigma python library + +----- + +#### [sigma_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) + + + +An expansion hover module to perform a syntax check on sigma rules. +- **features**: +>This module takes a Sigma rule attribute as input and performs a syntax check on it. +> +>It displays then that the rule is valid if it is the case, and the error related to the rule otherwise. +- **input**: +>A Sigma attribute. +- **output**: +>Text describing the validity of the Sigma rule. +- **references**: +>https://github.com/Neo23x0/sigma/wiki +- **requirements**: +>Sigma python library, Yaml python library + +----- + +#### [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) + +Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page. +- **features**: +>This module takes a link or url attribute as input and caches the related web page. It returns then a link of the cached page. +- **input**: +>A link or url attribute. +- **output**: +>A malware-sample attribute describing the cached page. +- **references**: +>https://github.com/adulau/url_archiver +- **requirements**: +>urlarchiver: python library to fetch and archive URL on the file-system + +----- + +#### [stix2_pattern_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) + + + +An expansion hover module to perform a syntax check on stix2 patterns. +- **features**: +>This module takes a STIX2 pattern attribute as input and performs a syntax check on it. +> +>It displays then that the rule is valid if it is the case, and the error related to the rule otherwise. +- **input**: +>A STIX2 pattern attribute. +- **output**: +>Text describing the validity of the STIX2 pattern. +- **references**: +>[STIX2.0 patterning specifications](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html) +- **requirements**: +>stix2patterns python library + +----- + +#### [threatcrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) + + + +Module to get information from ThreatCrowd. +- **features**: +>This module takes a MISP attribute as input and queries ThreatCrowd with it. +> +>The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. +- **input**: +>A MISP attribute included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- md5 +>- sha1 +>- sha256 +>- sha512 +>- whois-registrant-email +- **output**: +>MISP attributes mapped from the result of the query on ThreatCrowd, included in the following list: +>- domain +>- ip-src +>- ip-dst +>- text +>- md5 +>- sha1 +>- sha256 +>- sha512 +>- hostname +>- whois-registrant-email +- **references**: +>https://www.threatcrowd.org/ + +----- + +#### [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) + + + +Module to get information from ThreatMiner. +- **features**: +>This module takes a MISP attribute as input and queries ThreatMiner with it. +> +>The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. +- **input**: +>A MISP attribute included in the following list: +>- hostname +>- domain +>- ip-src +>- ip-dst +>- md5 +>- sha1 +>- sha256 +>- sha512 +- **output**: +>MISP attributes mapped from the result of the query on ThreatMiner, included in the following list: +>- domain +>- ip-src +>- ip-dst +>- text +>- md5 +>- sha1 +>- sha256 +>- sha512 +>- ssdeep +>- authentihash +>- filename +>- whois-registrant-email +>- url +>- link +- **references**: +>https://www.threatminer.org/ + +----- + +#### [trustar_enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/trustar_enrich.py) + + + +Module to get enrich indicators with TruSTAR. +- **features**: +>This module enriches MISP attributes with scoring and metadata from TruSTAR. +> +>The TruSTAR indicator summary is appended to the attributes along with links to any associated reports. +- **input**: +>Any of the following MISP attributes: +>- btc +>- domain +>- email-src +>- filename +>- hostname +>- ip-src +>- ip-dst +>- md5 +>- sha1 +>- sha256 +>- url +- **output**: +>MISP attributes enriched with indicator summary data from the TruSTAR API. Data includes a severity level score and additional source and scoring info. +- **references**: +>https://docs.trustar.co/api/v13/indicators/get_indicator_summaries.html + +----- + +#### [urlhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlhaus.py) + + + +Query of the URLhaus API to get additional information about the input attribute. +- **features**: +>Module using the new format of modules able to return attributes and objects. +> +>The module takes one of the attribute type specified as input, and query the URLhaus API with it. If any result is returned by the API, attributes and objects are created accordingly. +- **input**: +>A domain, hostname, url, ip, md5 or sha256 attribute. +- **output**: +>MISP attributes & objects fetched from the result of the URLhaus API query. +- **references**: +>https://urlhaus.abuse.ch/ + +----- + +#### [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) + + + +An expansion module to query urlscan.io. +- **features**: +>This module takes a MISP attribute as input and queries urlscan.io with it. +> +>The result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute. +- **input**: +>A domain, hostname or url attribute. +- **output**: +>MISP attributes mapped from the result of the query on urlscan.io. +- **references**: +>https://urlscan.io/ +- **requirements**: +>An access to the urlscan.io API + +----- + +#### [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) + + + +Module to get advanced information from virustotal. +- **features**: +>New format of modules able to return attributes and objects. +> +>A module to take a MISP attribute as input and query the VirusTotal API to get additional data about it. +> +>Compared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request. +> +>Thus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them. +- **input**: +>A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute. +- **output**: +>MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute. +- **references**: +>https://www.virustotal.com/, https://developers.virustotal.com/reference +- **requirements**: +>An access to the VirusTotal API (apikey), with a high request rate limit. + +----- + +#### [virustotal_public](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal_public.py) + + + +Module to get information from VirusTotal. +- **features**: +>New format of modules able to return attributes and objects. +> +>A module to take a MISP attribute as input and query the VirusTotal API to get additional data about it. +> +>Compared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit. +> +>Thus, it only queries the API once and returns the results that is parsed into MISP attributes and objects. +- **input**: +>A domain, hostname, ip, url or hash (md5, sha1, sha256 or sha512) attribute. +- **output**: +>MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute. +- **references**: +>https://www.virustotal.com, https://developers.virustotal.com/reference +- **requirements**: +>An access to the VirusTotal API (apikey) + +----- + +#### [vmray_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) + + + +Module to submit a sample to VMRay. +- **features**: +>This module takes an attachment or malware-sample attribute as input to query the VMRay API. +> +>The sample contained within the attribute in then enriched with data from VMRay mapped into MISP attributes. +- **input**: +>An attachment or malware-sample attribute. +- **output**: +>MISP attributes mapped from the result of the query on VMRay API, included in the following list: +>- text +>- sha1 +>- sha256 +>- md5 +>- link +- **references**: +>https://www.vmray.com/ +- **requirements**: +>An access to the VMRay API (apikey & url) + +----- + +#### [vulndb](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) + + + +Module to query VulnDB (RiskBasedSecurity.com). +- **features**: +>This module takes a vulnerability attribute as input and queries VulnDB in order to get some additional data about it. +> +>The API gives the result of the query which can be displayed in the screen, and/or mapped into MISP attributes to add in the event. +- **input**: +>A vulnerability attribute. +- **output**: +>Additional data enriching the CVE input, fetched from VulnDB. +- **references**: +>https://vulndb.cyberriskanalytics.com/ +- **requirements**: +>An access to the VulnDB API (apikey, apisecret) + +----- + +#### [vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) + + + +An expansion hover module to expand information about CVE id using Vulners API. +- **features**: +>This module takes a vulnerability attribute as input and queries the Vulners API in order to get some additional data about it. +> +>The API then returns details about the vulnerability. +- **input**: +>A vulnerability attribute. +- **output**: +>Text giving additional information about the CVE in input. +- **references**: +>https://vulners.com/ +- **requirements**: +>Vulners python library, An access to the Vulners API + +----- + +#### [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) + +Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd). +- **features**: +>This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server). +- **input**: +>A domain or IP address attribute. +- **output**: +>Text describing the result of a whois request for the input value. +- **references**: +>https://github.com/rafiot/uwhoisd +- **requirements**: +>uwhois: A whois python library + +----- + +#### [wiki](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) + + + +An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis. +- **features**: +>This module takes a text attribute as input and queries the Wikidata API. If the text attribute is clear enough to define a specific term, the API returns a wikidata link in response. +- **input**: +>Text attribute. +- **output**: +>Text attribute. +- **references**: +>https://www.wikidata.org +- **requirements**: +>SPARQLWrapper python library + +----- + +#### [xforceexchange](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) + + + +An expansion module for IBM X-Force Exchange. +- **features**: +>This module takes a MISP attribute as input to query the X-Force API. The API returns then additional information known in their threats data, that is mapped into MISP attributes. +- **input**: +>A MISP attribute included in the following list: +>- ip-src +>- ip-dst +>- vulnerability +>- md5 +>- sha1 +>- sha256 +- **output**: +>MISP attributes mapped from the result of the query on X-Force Exchange. +- **references**: +>https://exchange.xforce.ibmcloud.com/ +- **requirements**: +>An access to the X-Force API (apikey) + +----- + +#### [xlsx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xlsx-enrich.py) + + + +Module to extract freetext from a .xlsx document. +- **features**: +>The module reads the text contained in a .xlsx document. The result is passed to the freetext import parser so IoCs can be extracted out of it. +- **input**: +>Attachment attribute containing a .xlsx document. +- **output**: +>Text and freetext parsed from the document. +- **requirements**: +>pandas: Python library to perform data analysis, time series and statistics. + +----- + +#### [yara_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) + + + +An expansion & hover module to translate any hash attribute into a yara rule. +- **features**: +>The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it. This YARA rule is also validated using the same method as in 'yara_syntax_validator' module. +>Both hover and expansion functionalities are supported with this module, where the hover part is displaying the resulting YARA rule and the expansion part allows you to add the rule as a new attribute, as usual with expansion modules. +- **input**: +>MISP Hash attribute (md5, sha1, sha256, imphash, or any of the composite attribute with filename and one of the previous hash type). +- **output**: +>YARA rule. +- **references**: +>https://virustotal.github.io/yara/, https://github.com/virustotal/yara-python +- **requirements**: +>yara-python python library + +----- + +#### [yara_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) + + + +An expansion hover module to perform a syntax check on if yara rules are valid or not. +- **features**: +>This modules simply takes a YARA rule as input, and checks its syntax. It returns then a confirmation if the syntax is valid, otherwise the syntax error is displayed. +- **input**: +>YARA rule attribute. +- **output**: +>Text to inform users if their rule is valid. +- **references**: +>http://virustotal.github.io/yara/ +- **requirements**: +>yara_python python library + +----- + +## Export Modules + +#### [cef_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) + +Module to export a MISP event in CEF format. +- **features**: +>The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in Common Event Format. +>Thus, there is no particular feature concerning MISP Events since any event can be exported. However, 4 configuration parameters recognized by CEF format are required and should be provided by users before exporting data: the device vendor, product and version, as well as the default severity of data. +- **input**: +>MISP Event attributes +- **output**: +>Common Event Format file +- **references**: +>https://community.softwaregrp.com/t5/ArcSight-Connectors/ArcSight-Common-Event-Format-CEF-Guide/ta-p/1589306?attachment-id=65537 + +----- + +#### [cisco_firesight_manager_ACL_rule_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) + + + +Module to export malicious network activity attributes to Cisco fireSIGHT manager block rules. +- **features**: +>The module goes through the attributes to find all the network activity ones in order to create block rules for the Cisco fireSIGHT manager. +- **input**: +>Network activity attributes (IPs, URLs). +- **output**: +>Cisco fireSIGHT manager block rules. +- **requirements**: +>Firesight manager console credentials + +----- + +#### [goamlexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) + + + +This module is used to export MISP events containing transaction objects into GoAML format. +- **features**: +>The module works as long as there is at least one transaction object in the Event. +> +>Then in order to have a valid GoAML document, please follow these guidelines: +>- For each transaction object, use either a bank-account, person, or legal-entity object to describe the origin of the transaction, and again one of them to describe the target of the transaction. +>- Create an object reference for both origin and target objects of the transaction. +>- A bank-account object needs a signatory, which is a person object, put as object reference of the bank-account. +>- A person can have an address, which is a geolocation object, put as object reference of the person. +> +>Supported relation types for object references that are recommended for each object are the folowing: +>- transaction: +> - 'from', 'from_my_client': Origin of the transaction - at least one of them is required. +> - 'to', 'to_my_client': Target of the transaction - at least one of them is required. +> - 'address': Location of the transaction - optional. +>- bank-account: +> - 'signatory': Signatory of a bank-account - the reference from bank-account to a signatory is required, but the relation-type is optional at the moment since this reference will always describe a signatory. +> - 'entity': Entity owning the bank account - optional. +>- person: +> - 'address': Address of a person - optional. +- **input**: +>MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target. +- **output**: +>GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities). +- **references**: +>http://goaml.unodc.org/ +- **requirements**: +>PyMISP, MISP objects + +----- + +#### [liteexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) + +Lite export of a MISP event. +- **features**: +>This module is simply producing a json MISP event format file, but exporting only Attributes from the Event. Thus, MISP Events exported with this module should have attributes that are not internal references, otherwise the resulting event would be empty. +- **input**: +>MISP Event attributes +- **output**: +>Lite MISP Event + +----- + +#### [mass_eql_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/mass_eql_export.py) + + + +Mass EQL query export for a MISP event. +- **features**: +>This module produces EQL queries for all relevant attributes in a MISP event. +- **input**: +>MISP Event attributes +- **output**: +>Text file containing one or more EQL queries +- **references**: +>https://eql.readthedocs.io/en/latest/ + +----- + +#### [nexthinkexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) + + + +Nexthink NXQL query export module +- **features**: +>This module export an event as Nexthink NXQL queries that can then be used in your own python3 tool or from wget/powershell +- **input**: +>MISP Event attributes +- **output**: +>Nexthink NXQL queries +- **references**: +>https://doc.nexthink.com/Documentation/Nexthink/latest/APIAndIntegrations/IntroducingtheWebAPIV2 + +----- + +#### [osqueryexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) + + + +OSQuery export of a MISP event. +- **features**: +>This module export an event as osquery queries that can be used in packs or in fleet management solution like Kolide. +- **input**: +>MISP Event attributes +- **output**: +>osquery SQL queries + +----- + +#### [pdfexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) + +Simple export of a MISP event to PDF. +- **features**: +>The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf. +> 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata +> 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event +> 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies. +> 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event ! +> 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation. +> 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option +- **input**: +>MISP Event +- **output**: +>MISP Event in a PDF file. +- **references**: +>https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html +- **requirements**: +>PyMISP, reportlab + +----- + +#### [testexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/testexport.py) + +Skeleton export module. + +----- + +#### [threatStream_misp_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) + + + +Module to export a structured CSV file for uploading to threatStream. +- **features**: +>The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatStream. +- **input**: +>MISP Event attributes +- **output**: +>ThreatStream CSV format file +- **references**: +>https://www.anomali.com/platform/threatstream, https://github.com/threatstream +- **requirements**: +>csv + +----- + +#### [threat_connect_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) + + + +Module to export a structured CSV file for uploading to ThreatConnect. +- **features**: +>The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatConnect. +>Users should then provide, as module configuration, the source of data they export, because it is required by the output format. +- **input**: +>MISP Event attributes +- **output**: +>ThreatConnect CSV format file +- **references**: +>https://www.threatconnect.com +- **requirements**: +>csv + +----- + +#### [vt_graph](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/vt_graph.py) + + + +This module is used to create a VirusTotal Graph from a MISP event. +- **features**: +>The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event. +> +>Once the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal. +- **input**: +>A MISP event. +- **output**: +>Link of the VirusTotal Graph created for the event. +- **references**: +>https://www.virustotal.com/gui/graph-overview +- **requirements**: +>vt_graph_api, the python library to query the VirusTotal graph API + +----- + +## Import Modules + +#### [csvimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) + +Module to import MISP attributes from a csv file. +- **features**: +>In order to parse data from a csv file, a header is required to let the module know which column is matching with known attribute fields / MISP types. +> +>This header either comes from the csv file itself or is part of the configuration of the module and should be filled out in MISP plugin settings, each field separated by COMMAS. Fields that do not match with any type known in MISP or are not MISP attribute fields should be ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment, '). +> +>If the csv file already contains a header that does not start by a '#', you should tick the checkbox 'has_header' to avoid importing it and have potential issues. You can also redefine the header even if it is already contained in the file, by following the rules for headers explained earlier. One reason why you would redefine a header is for instance when you want to skip some fields, or some fields are not valid types. +- **input**: +>CSV format file. +- **output**: +>MISP Event attributes +- **references**: +>https://tools.ietf.org/html/rfc4180, https://tools.ietf.org/html/rfc7111 +- **requirements**: +>PyMISP + +----- + +#### [cuckooimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) + + + +Module to import Cuckoo JSON. +- **features**: +>The module simply imports MISP Attributes from a Cuckoo JSON format file. There is thus no special feature to make it work. +- **input**: +>Cuckoo JSON file +- **output**: +>MISP Event attributes +- **references**: +>https://cuckoosandbox.org/, https://github.com/cuckoosandbox/cuckoo + +----- + +#### [email_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) + +Module to import emails in MISP. +- **features**: +>This module can be used to import e-mail text as well as attachments and urls. +>3 configuration parameters are then used to unzip attachments, guess zip attachment passwords, and extract urls: set each one of them to True or False to process or not the respective corresponding actions. +- **input**: +>E-mail file +- **output**: +>MISP Event attributes + +----- + +#### [goamlimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) + + + +Module to import MISP objects about financial transactions from GoAML files. +- **features**: +>Unlike the GoAML export module, there is here no special feature to import data from GoAML external files, since the module will import MISP Objects with their References on its own, as it is required for the export module to rebuild a valid GoAML document. +- **input**: +>GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities). +- **output**: +>MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target. +- **references**: +>http://goaml.unodc.org/ +- **requirements**: +>PyMISP + +----- + +#### [joe_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) + + + +A module to import data from a Joe Sandbox analysis json report. +- **features**: +>Module using the new format of modules able to return attributes and objects. +> +>The module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report. +> +> +- **input**: +>Json report of a Joe Sandbox analysis. +- **output**: +>MISP attributes & objects parsed from the analysis report. +- **references**: +>https://www.joesecurity.org, https://www.joesandbox.com/ + +----- + +#### [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) + + + +Module to import and parse reports from Lastline analysis links. +- **features**: +>The module requires a Lastline Portal `username` and `password`. +>The module uses the new format and it is able to return MISP attributes and objects. +>The module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module. +- **input**: +>Link to a Lastline analysis. +- **output**: +>MISP attributes and objects parsed from the analysis report. +- **references**: +>https://www.lastline.com + +----- + +#### [mispjson](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/mispjson.py) + +Module to import MISP JSON format for merging MISP events. +- **features**: +>The module simply imports MISP Attributes from an other MISP Event in order to merge events together. There is thus no special feature to make it work. +- **input**: +>MISP Event +- **output**: +>MISP Event attributes + +----- + +#### [ocr](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) + +Optical Character Recognition (OCR) module for MISP. +- **features**: +>The module tries to recognize some text from an image and import the result as a freetext attribute, there is then no special feature asked to users to make it work. +- **input**: +>Image +- **output**: +>freetext MISP attribute + +----- + +#### [openiocimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) + +Module to import OpenIOC packages. +- **features**: +>The module imports MISP Attributes from OpenIOC packages, there is then no special feature for users to make it work. +- **input**: +>OpenIOC packages +- **output**: +>MISP Event attributes +- **references**: +>https://www.fireeye.com/blog/threat-research/2013/10/openioc-basics.html +- **requirements**: +>PyMISP + +----- + +#### [threatanalyzer_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) + +Module to import ThreatAnalyzer archive.zip / analysis.json files. +- **features**: +>The module imports MISP Attributes from a ThreatAnalyzer format file. This file can be either ZIP, or JSON format. +>There is by the way no special feature for users to make the module work. +- **input**: +>ThreatAnalyzer format file +- **output**: +>MISP Event attributes +- **references**: +>https://www.threattrack.com/malware-analysis.aspx + +----- + +#### [vmray_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) + + + +Module to import VMRay (VTI) results. +- **features**: +>The module imports MISP Attributes from VMRay format, using the VMRay api. +>Users should then provide as the module configuration the API Key as well as the server url in order to fetch their data to import. +- **input**: +>VMRay format +- **output**: +>MISP Event attributes +- **references**: +>https://www.vmray.com/ +- **requirements**: +>vmray_rest_api + +----- diff --git a/doc/expansion/apiosintds.json b/doc/expansion/apiosintds.json new file mode 100644 index 0000000..81a1eec --- /dev/null +++ b/doc/expansion/apiosintds.json @@ -0,0 +1,8 @@ +{ + "description": "On demand query API for OSINT.digitalside.it project.", + "requirements": ["The apiosintDS python library to query the OSINT.digitalside.it API."], + "input": "A domain, ip, url or hash attribute.", + "output": "Hashes and urls resulting from the query to OSINT.digitalside.it", + "references": ["https://osint.digitalside.it/#About"], + "features": "The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute.\n\nThe result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls.\n\nFurthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it" +} diff --git a/doc/expansion/apivoid.json b/doc/expansion/apivoid.json new file mode 100644 index 0000000..2173d5b --- /dev/null +++ b/doc/expansion/apivoid.json @@ -0,0 +1,9 @@ +{ + "description": "Module to query APIVoid with some domain attributes.", + "logo": "logos/apivoid.png", + "requirements": ["A valid APIVoid API key with enough credits to proceed 2 queries"], + "input": "A domain attribute.", + "output": "DNS records and SSL certificates related to the domain.", + "features": "This module takes a domain name and queries API Void to get the related DNS records and the SSL certificates. It returns then those pieces of data as MISP objects that can be added to the event.\n\nTo make it work, a valid API key and enough credits to proceed 2 queries (0.06 + 0.07 credits) are required.", + "references": ["https://www.apivoid.com/"] +} diff --git a/doc/expansion/assemblyline_query.json b/doc/expansion/assemblyline_query.json new file mode 100644 index 0000000..700bde0 --- /dev/null +++ b/doc/expansion/assemblyline_query.json @@ -0,0 +1,9 @@ +{ + "description": "A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.", + "logo": "logos/assemblyline.png", + "requirements": ["assemblyline_client: Python library to query the AssemblyLine rest API."], + "input": "Link of an AssemblyLine submission report.", + "output": "MISP attributes & objects parsed from the AssemblyLine submission.", + "references": ["https://www.cyber.cg.ca/en/assemblyline"], + "features": "The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the used-ID and an API key or the password associated to the user-ID.\n\nThe submission ID extracted from the submission link is then used to query AssemblyLine and get the full submission report. This report is parsed to extract file objects and the associated IPs, domains or URLs the files are connecting to.\n\nSome more data may be parsed in the future." +} diff --git a/doc/expansion/assemblyline_submit.json b/doc/expansion/assemblyline_submit.json new file mode 100644 index 0000000..9fe9af6 --- /dev/null +++ b/doc/expansion/assemblyline_submit.json @@ -0,0 +1,9 @@ +{ + "description": "A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.", + "logo": "logos/assemblyline.png", + "requirements": ["assemblyline_client: Python library to query the AssemblyLine rest API."], + "input": "Sample, or url to submit to AssemblyLine.", + "output": "Link of the report generated in AssemblyLine.", + "references": ["https://www.cyber.gc.ca/en/assemblyline"], + "features": "The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the user-ID and an API key or the password associated to the user-ID.\n\nIf the sample or url is correctly submitted, you get then the link of the submission." +} diff --git a/doc/expansion/backscatter_io.json b/doc/expansion/backscatter_io.json new file mode 100644 index 0000000..a8475c5 --- /dev/null +++ b/doc/expansion/backscatter_io.json @@ -0,0 +1,9 @@ +{ + "description": "Query backscatter.io (https://backscatter.io/).", + "requirements": ["backscatter python library"], + "features": "The module takes a source or destination IP address as input and displays the information known by backscatter.io.\n\n", + "logo": "logos/backscatter_io.png", + "references": ["https://pypi.org/project/backscatter/"], + "input": "IP addresses.", + "output": "Text containing a history of the IP addresses especially on scanning based on backscatter.io information ." +} diff --git a/doc/expansion/bgpranking.json b/doc/expansion/bgpranking.json new file mode 100644 index 0000000..a98b780 --- /dev/null +++ b/doc/expansion/bgpranking.json @@ -0,0 +1,8 @@ +{ + "description": "Query BGP Ranking (https://bgpranking-ng.circl.lu/).", + "requirements": ["pybgpranking python library"], + "features": "The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking.\n\n", + "references": ["https://github.com/D4-project/BGP-Ranking/"], + "input": "Autonomous system number.", + "output": "Text containing a description of the ASN, its history, and the position in BGP Ranking." +} diff --git a/doc/expansion/btc_scam_check.json b/doc/expansion/btc_scam_check.json new file mode 100644 index 0000000..44fce03 --- /dev/null +++ b/doc/expansion/btc_scam_check.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.", + "requirements": ["dnspython3: dns python library"], + "features": "The module queries a dns blacklist directly with the bitcoin address and get a response if the address has been abused.", + "logo": "logos/bitcoin.png", + "input": "btc address attribute.", + "output" : "Text to indicate if the BTC address has been abused.", + "references": ["https://btcblack.it/"] +} diff --git a/doc/expansion/btc_steroids.json b/doc/expansion/btc_steroids.json new file mode 100644 index 0000000..fd264d8 --- /dev/null +++ b/doc/expansion/btc_steroids.json @@ -0,0 +1,6 @@ +{ + "description": "An expansion hover module to get a blockchain balance from a BTC address in MISP.", + "logo": "logos/bitcoin.png", + "input": "btc address attribute.", + "output": "Text to describe the blockchain balance and the transactions related to the btc address in input." +} diff --git a/doc/expansion/censys_enrich.json b/doc/expansion/censys_enrich.json new file mode 100644 index 0000000..83e6d5f --- /dev/null +++ b/doc/expansion/censys_enrich.json @@ -0,0 +1,8 @@ +{ + "description": "An expansion module to enrich attributes in MISP by quering the censys.io API", + "requirements": ["API credentials to censys.io"], + "input": "IP, domain or certificate fingerprint (md5, sha1 or sha256)", + "output": "MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details", + "references": ["https://www.censys.io"], + "features": "This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API." +} diff --git a/doc/expansion/circl_passivedns.json b/doc/expansion/circl_passivedns.json new file mode 100644 index 0000000..024437c --- /dev/null +++ b/doc/expansion/circl_passivedns.json @@ -0,0 +1,9 @@ +{ + "description": "Module to access CIRCL Passive DNS.", + "logo": "logos/passivedns.png", + "requirements": ["pypdns: Passive DNS python library", "A CIRCL passive DNS account with username & password"], + "input": "Hostname, domain, or ip-address attribute.", + "ouput": "Passive DNS objects related to the input attribute.", + "features": "This module takes a hostname, domain or ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive DNS REST API to get the asssociated passive dns entries and return them as MISP objects.\n\nTo make it work a username and a password are thus required to authenticate to the CIRCL Passive DNS API.", + "references": ["https://www.circl.lu/services/passive-dns/", "https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/"] +} diff --git a/doc/expansion/circl_passivessl.json b/doc/expansion/circl_passivessl.json new file mode 100644 index 0000000..f9792e1 --- /dev/null +++ b/doc/expansion/circl_passivessl.json @@ -0,0 +1,9 @@ +{ + "description": "Modules to access CIRCL Passive SSL.", + "logo": "logos/passivessl.png", + "requirements": ["pypssl: Passive SSL python library", "A CIRCL passive SSL account with username & password"], + "input": "IP address attribute.", + "output": "x509 certificate objects seen by the IP address(es).", + "features": "This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive SSL REST API to gather the related certificates and return the corresponding MISP objects.\n\nTo make it work a username and a password are required to authenticate to the CIRCL Passive SSL API.", + "references": ["https://www.circl.lu/services/passive-ssl/"] +} diff --git a/doc/expansion/countrycode.json b/doc/expansion/countrycode.json new file mode 100644 index 0000000..c6214e5 --- /dev/null +++ b/doc/expansion/countrycode.json @@ -0,0 +1,6 @@ +{ + "description": "Module to expand country codes.", + "input": "Hostname or domain attribute.", + "output": "Text with the country code the input belongs to.", + "features": "The module takes a domain or a hostname as input, and returns the country it belongs to.\n\nFor non country domains, a list of the most common possible extensions is used." +} diff --git a/doc/expansion/crowdstrike_falcon.json b/doc/expansion/crowdstrike_falcon.json new file mode 100644 index 0000000..07e9dbd --- /dev/null +++ b/doc/expansion/crowdstrike_falcon.json @@ -0,0 +1,9 @@ +{ + "description": "Module to query Crowdstrike Falcon.", + "logo": "logos/crowdstrike.png", + "requirements": ["A CrowdStrike API access (API id & key)"], + "input": "A MISP attribute included in the following list:\n- domain\n- email-attachment\n- email-dst\n- email-reply-to\n- email-src\n- email-subject\n- filename\n- hostname\n- ip-src\n- ip-dst\n- md5\n- mutex\n- regkey\n- sha1\n- sha256\n- uri\n- url\n- user-agent\n- whois-registrant-email\n- x509-fingerprint-md5", + "output": "MISP attributes mapped after the CrowdStrike API has been queried, included in the following list:\n- hostname\n- email-src\n- email-subject\n- filename\n- md5\n- sha1\n- sha256\n- ip-dst\n- ip-dst\n- mutex\n- regkey\n- url\n- user-agent\n- x509-fingerprint-md5", + "references": ["https://www.crowdstrike.com/products/crowdstrike-falcon-faq/"], + "features": "This module takes a MISP attribute as input to query a CrowdStrike Falcon API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes.\n\nPlease note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported." +} diff --git a/doc/expansion/cuckoo_submit.json b/doc/expansion/cuckoo_submit.json new file mode 100644 index 0000000..7fe8067 --- /dev/null +++ b/doc/expansion/cuckoo_submit.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion module to submit files and URLs to Cuckoo Sandbox.", + "logo": "logos/cuckoo.png", + "requirements": ["Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)"], + "input": "A malware-sample or attachment for files. A url or domain for URLs.", + "output": "A text field containing 'Cuckoo task id: '", + "references": ["https://cuckoosandbox.org/", "https://cuckoo.sh/docs/"], + "features": "The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox.\n The returned task id can be used to retrieve results when the analysis completed." +} diff --git a/doc/expansion/cve.json b/doc/expansion/cve.json new file mode 100644 index 0000000..04f131f --- /dev/null +++ b/doc/expansion/cve.json @@ -0,0 +1,8 @@ +{ + "description": "An expansion hover module to expand information about CVE id.", + "logo": "logos/cve.png", + "input": "Vulnerability attribute.", + "output": "Text giving information about the CVE related to the Vulnerability.", + "references": ["https://cve.circl.lu/", "https://cve.mitre.org/"], + "features": "The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to get information about the vulnerability as it is described in the list of CVEs." +} diff --git a/doc/expansion/cytomic_orion.json b/doc/expansion/cytomic_orion.json new file mode 100644 index 0000000..6f87657 --- /dev/null +++ b/doc/expansion/cytomic_orion.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion module to enrich attributes in MISP by quering the Cytomic Orion API", + "logo": "logos/cytomic_orion.png", + "requirements": ["Access (license) to Cytomic Orion"], + "input": "MD5, hash of the sample / malware to search for.", + "output": "MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines.", + "references": ["https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/", "https://www.cytomicmodel.com/solutions/"], + "features": "This module takes an MD5 hash and searches for occurrences of this hash in the Cytomic Orion database. Returns observed files and machines." +} diff --git a/doc/expansion/dbl_spamhaus.json b/doc/expansion/dbl_spamhaus.json new file mode 100644 index 0000000..ea73dcb --- /dev/null +++ b/doc/expansion/dbl_spamhaus.json @@ -0,0 +1,9 @@ +{ + "description": "Module to check Spamhaus DBL for a domain name.", + "logo": "logos/spamhaus.jpg", + "requirements": ["dnspython3: DNS python3 library"], + "input": "Domain or hostname attribute.", + "output": "Information about the nature of the input.", + "references": ["https://www.spamhaus.org/faq/section/Spamhaus%20DBL"], + "features": "This modules takes a domain or a hostname in input and queries the Domain Block List provided by Spamhaus to determine what kind of domain it is.\n\nDBL then returns a response code corresponding to a certain classification of the domain we display. If the queried domain is not in the list, it is also mentionned.\n\nPlease note that composite MISP attributes containing domain or hostname are supported as well." +} diff --git a/doc/expansion/dns.json b/doc/expansion/dns.json new file mode 100644 index 0000000..dc43b64 --- /dev/null +++ b/doc/expansion/dns.json @@ -0,0 +1,7 @@ +{ + "description": "A simple DNS expansion service to resolve IP address from domain MISP attributes.", + "requirements": ["dnspython3: DNS python3 library"], + "input": "Domain or hostname attribute.", + "output": "IP address resolving the input.", + "features": "The module takes a domain of hostname attribute as input, and tries to resolve it. If no error is encountered, the IP address that resolves the domain is returned, otherwise the origin of the error is displayed.\n\nThe address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8).\n\nPlease note that composite MISP attributes containing domain or hostname are supported as well." +} diff --git a/doc/expansion/docx-enrich.json b/doc/expansion/docx-enrich.json new file mode 100644 index 0000000..fccba57 --- /dev/null +++ b/doc/expansion/docx-enrich.json @@ -0,0 +1,9 @@ +{ + "description": "Module to extract freetext from a .docx document.", + "logo": "logos/docx.png", + "requirements": ["docx python library"], + "input": "Attachment attribute containing a .docx document.", + "output": "Text and freetext parsed from the document.", + "references": [], + "features": "The module reads the text contained in a .docx document. The result is passed to the freetext import parser so IoCs can be extracted out of it." +} diff --git a/doc/expansion/domaintools.json b/doc/expansion/domaintools.json new file mode 100644 index 0000000..849028c --- /dev/null +++ b/doc/expansion/domaintools.json @@ -0,0 +1,9 @@ +{ + "description": "DomainTools MISP expansion module.", + "logo": "logos/domaintools.png", + "requirements": ["Domaintools python library", "A Domaintools API access (username & apikey)"], + "input": "A MISP attribute included in the following list:\n- domain\n- hostname\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-name\n- whois-registrant-phone\n- ip-src\n- ip-dst", + "output": "MISP attributes mapped after the Domaintools API has been queried, included in the following list:\n- whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- text\n- domain", + "references": ["https://www.domaintools.com/"], + "features": "This module takes a MISP attribute as input to query the Domaintools API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes.\n\nPlease note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported." +} diff --git a/doc/expansion/eql.json b/doc/expansion/eql.json new file mode 100644 index 0000000..1a32adf --- /dev/null +++ b/doc/expansion/eql.json @@ -0,0 +1,9 @@ +{ + "description": "EQL query generation for a MISP attribute.", + "logo": "logos/eql.png", + "requirements": [], + "input": "A filename or ip attribute.", + "output": "Attribute containing EQL for a network or file attribute.", + "references": ["https://eql.readthedocs.io/en/latest/"], + "features": "This module adds a new attribute to a MISP event containing an EQL query for a network or file attribute." +} diff --git a/doc/expansion/eupi.json b/doc/expansion/eupi.json new file mode 100644 index 0000000..02a16fb --- /dev/null +++ b/doc/expansion/eupi.json @@ -0,0 +1,9 @@ +{ + "description": "A module to query the Phishing Initiative service (https://phishing-initiative.lu).", + "logo": "logos/eupi.png", + "requirements": ["pyeupi: eupi python library", "An access to the Phishing Initiative API (apikey & url)"], + "input": "A domain, hostname or url MISP attribute.", + "output": "Text containing information about the input, resulting from the query on Phishing Initiative.", + "references": ["https://phishing-initiative.eu/?lang=en"], + "features": "This module takes a domain, hostname or url MISP attribute as input to query the Phishing Initiative API. The API returns then the result of the query with some information about the value queried.\n\nPlease note that composite attributes containing domain or hostname are also supported." +} diff --git a/doc/expansion/farsight_passivedns.json b/doc/expansion/farsight_passivedns.json new file mode 100644 index 0000000..2c1bf05 --- /dev/null +++ b/doc/expansion/farsight_passivedns.json @@ -0,0 +1,9 @@ +{ + "description": "Module to access Farsight DNSDB Passive DNS.", + "logo": "logos/farsight.png", + "requirements": ["An access to the Farsight Passive DNS API (apikey)"], + "input": "A domain, hostname or IP address MISP attribute.", + "output": "Text containing information about the input, resulting from the query on the Farsight Passive DNS API.", + "references": ["https://www.farsightsecurity.com/"], + "features": "This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. The API returns then the result of the query with some information about the value queried." +} diff --git a/doc/expansion/geoip_country.json b/doc/expansion/geoip_country.json new file mode 100644 index 0000000..9db49a2 --- /dev/null +++ b/doc/expansion/geoip_country.json @@ -0,0 +1,9 @@ +{ + "description": "Module to query a local copy of Maxmind's Geolite database.", + "logo": "logos/maxmind.png", + "requirements": ["A local copy of Maxmind's Geolite database"], + "input": "An IP address MISP Attribute.", + "output": "Text containing information about the location of the IP address.", + "references": ["https://www.maxmind.com/en/home"], + "features": "This module takes an IP address MISP attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the location of this IP address.\n\nPlease note that composite attributes domain|ip are also supported." +} diff --git a/doc/expansion/greynoise.json b/doc/expansion/greynoise.json new file mode 100644 index 0000000..f1f1003 --- /dev/null +++ b/doc/expansion/greynoise.json @@ -0,0 +1,9 @@ +{ + "description": "Module to access GreyNoise.io API", + "logo": "logos/greynoise.png", + "requirements": [], + "input": "An IP address.", + "output": "Additional information about the IP fetched from Greynoise API.", + "references": ["https://greynoise.io/", "https://github.com/GreyNoise-Intelligence/api.greynoise.io"], + "features": "The module takes an IP address as input and queries Greynoise for some additional information about it. The result is returned as text." +} diff --git a/doc/expansion/hashdd.json b/doc/expansion/hashdd.json new file mode 100644 index 0000000..d963820 --- /dev/null +++ b/doc/expansion/hashdd.json @@ -0,0 +1,7 @@ +{ + "description": "A hover module to check hashes against hashdd.com including NSLR dataset.", + "input": "A hash MISP attribute (md5).", + "output": "Text describing the known level of the hash in the hashdd databases.", + "references": ["https://hashdd.com/"], + "features": "This module takes a hash attribute as input to check its known level, using the hashdd API. This information is then displayed." +} diff --git a/doc/expansion/hibp.json b/doc/expansion/hibp.json new file mode 100644 index 0000000..3c3ee54 --- /dev/null +++ b/doc/expansion/hibp.json @@ -0,0 +1,9 @@ +{ + "description": "Module to access haveibeenpwned.com API.", + "logo": "logos/hibp.png", + "requirements": [], + "input": "An email address", + "output": "Additional information about the email address.", + "references": ["https://haveibeenpwned.com/"], + "features": "The module takes an email address as input and queries haveibeenpwned.com API to find additional information about it. This additional information actually tells if any account using the email address has already been compromised in a data breach." +} diff --git a/doc/expansion/intelmq_eventdb.json b/doc/expansion/intelmq_eventdb.json new file mode 100644 index 0000000..bc48414 --- /dev/null +++ b/doc/expansion/intelmq_eventdb.json @@ -0,0 +1,9 @@ +{ + "description": "Module to access intelmqs eventdb.", + "logo": "logos/intelmq.png", + "requirements": ["psycopg2: Python library to support PostgreSQL", "An access to the IntelMQ database (username, password, hostname and database reference)"], + "input": "A hostname, domain, IP address or AS attribute.", + "output": "Text giving information about the input using IntelMQ database.", + "references": ["https://github.com/certtools/intelmq", "https://intelmq.readthedocs.io/en/latest/Developers-Guide/"], + "features": "/!\\ EXPERIMENTAL MODULE, some features may not work /!\\\n\nThis module takes a domain, hostname, IP address or Autonomous system MISP attribute as input to query the IntelMQ database. The result of the query gives then additional information about the input." +} diff --git a/doc/expansion/ipasn.json b/doc/expansion/ipasn.json new file mode 100644 index 0000000..8caed92 --- /dev/null +++ b/doc/expansion/ipasn.json @@ -0,0 +1,8 @@ +{ + "description": "Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).", + "requirements": ["pyipasnhistory: Python library to access IPASN-history instance"], + "input": "An IP address MISP attribute.", + "output": "Asn object(s) objects related to the IP address used as input.", + "references": ["https://github.com/D4-project/IPASN-History"], + "features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object." +} diff --git a/doc/expansion/iprep.json b/doc/expansion/iprep.json new file mode 100644 index 0000000..95250e0 --- /dev/null +++ b/doc/expansion/iprep.json @@ -0,0 +1,8 @@ +{ + "description": "Module to query IPRep data for IP addresses.", + "requirements": ["An access to the packetmail API (apikey)"], + "input": "An IP address MISP attribute.", + "output": "Text describing additional information about the input after a query on the IPRep API.", + "references": ["https://github.com/mahesh557/packetmail"], + "features": "This module takes an IP address attribute as input and queries the database from packetmail.net to get some information about the reputation of the IP." +} diff --git a/doc/expansion/joesandbox_query.json b/doc/expansion/joesandbox_query.json new file mode 100644 index 0000000..1a94edb --- /dev/null +++ b/doc/expansion/joesandbox_query.json @@ -0,0 +1,9 @@ +{ + "description": "Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and converted into MISP attributes and objects.\n\nThis url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py).", + "logo": "logos/joesandbox.png", + "requirements": ["jbxapi: Joe Sandbox API python3 library"], + "input": "Link of a Joe Sandbox sample or url submission.", + "output": "MISP attributes & objects parsed from the analysis report.", + "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], + "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.\n\nEven if the introspection will allow all kinds of links to call this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return results.\n\nTo make it work you will need to fill the 'apikey' configuration with your Joe Sandbox API key and provide a valid link as input." +} diff --git a/doc/expansion/joesandbox_submit.json b/doc/expansion/joesandbox_submit.json new file mode 100644 index 0000000..ad59239 --- /dev/null +++ b/doc/expansion/joesandbox_submit.json @@ -0,0 +1,9 @@ +{ + "description": "A module to submit files or URLs to Joe Sandbox for an advanced analysis, and return the link of the submission.", + "logo": "logos/joesandbox.png", + "requirements": ["jbxapi: Joe Sandbox API python3 library"], + "input": "Sample, url (or domain) to submit to Joe Sandbox for an advanced analysis.", + "output": "Link of the report generated in Joe Sandbox.", + "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], + "features": "The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted analysis.\n\nIt is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link." +} diff --git a/doc/expansion/lastline_query.json b/doc/expansion/lastline_query.json new file mode 100644 index 0000000..6165890 --- /dev/null +++ b/doc/expansion/lastline_query.json @@ -0,0 +1,9 @@ +{ + "description": "Query Lastline with an analysis link and parse the report into MISP attributes and objects.\nThe analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module.", + "logo": "logos/lastline.png", + "requirements": [], + "input": "Link to a Lastline analysis.", + "output": "MISP attributes and objects parsed from the analysis report.", + "references": ["https://www.lastline.com"], + "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." +} diff --git a/doc/expansion/lastline_submit.json b/doc/expansion/lastline_submit.json new file mode 100644 index 0000000..d053f55 --- /dev/null +++ b/doc/expansion/lastline_submit.json @@ -0,0 +1,9 @@ +{ + "description": "Module to submit a file or URL to Lastline.", + "logo": "logos/lastline.png", + "requirements": [], + "input": "File or URL to submit to Lastline.", + "output": "Link to the report generated by Lastline.", + "references": ["https://www.lastline.com"], + "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." +} diff --git a/doc/expansion/macaddress_io.json b/doc/expansion/macaddress_io.json new file mode 100644 index 0000000..6bd2658 --- /dev/null +++ b/doc/expansion/macaddress_io.json @@ -0,0 +1,9 @@ +{ + "description": "MISP hover module for macaddress.io", + "logo": "logos/macaddress_io.png", + "requirements": ["maclookup: macaddress.io python library", "An access to the macaddress.io API (apikey)"], + "input": "MAC address MISP attribute.", + "output": "Text containing information on the MAC address fetched from a query on macaddress.io.", + "references": ["https://macaddress.io/", "https://github.com/CodeLineFi/maclookup-python"], + "features": "This module takes a MAC address attribute as input and queries macaddress.io for additional information.\n\nThis information contains data about:\n- MAC address details\n- Vendor details\n- Block details" +} diff --git a/doc/expansion/macvendors.json b/doc/expansion/macvendors.json new file mode 100644 index 0000000..cc10475 --- /dev/null +++ b/doc/expansion/macvendors.json @@ -0,0 +1,9 @@ +{ + "description": "Module to access Macvendors API.", + "logo": "logos/macvendors.png", + "requirements": [], + "input": "A MAC address.", + "output": "Additional information about the MAC address.", + "references": ["https://macvendors.com/", "https://macvendors.com/api"], + "features": "The module takes a MAC address as input and queries macvendors.com for some information about it. The API returns the name of the vendor related to the address." +} diff --git a/doc/expansion/malwarebazaar.json b/doc/expansion/malwarebazaar.json new file mode 100644 index 0000000..2db6ad5 --- /dev/null +++ b/doc/expansion/malwarebazaar.json @@ -0,0 +1,8 @@ +{ + "description": "Query the MALWAREbazaar API to get additional information about the input hash attribute.", + "requirements": [], + "input": "A hash attribute (md5, sha1 or sha256).", + "output": "File object(s) related to the input attribute found on MALWAREbazaar databases.", + "references": ["https://bazaar.abuse.ch/"], + "features": "The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it. The result, if the payload is known on the databases, is at least one file object describing the file the input hash is related to.\n\nThe module is using the new format of modules able to return object since the result is one or multiple MISP object(s)." +} diff --git a/doc/expansion/ocr-enrich.json b/doc/expansion/ocr-enrich.json new file mode 100644 index 0000000..8765b22 --- /dev/null +++ b/doc/expansion/ocr-enrich.json @@ -0,0 +1,8 @@ +{ + "description": "Module to process some optical character recognition on pictures.", + "requirements": ["cv2: The OpenCV python library."], + "input": "A picture attachment.", + "output": "Text and freetext fetched from the input picture.", + "references": [], + "features": "The module takes an attachment attributes as input and process some optical character recognition on it. The text found is then passed to the Freetext importer to extract potential IoCs." +} diff --git a/doc/expansion/ods-enrich.json b/doc/expansion/ods-enrich.json new file mode 100644 index 0000000..dda4281 --- /dev/null +++ b/doc/expansion/ods-enrich.json @@ -0,0 +1,10 @@ +{ + "description": "Module to extract freetext from a .ods document.", + "logo": "logos/ods.png", + "requirements": ["ezodf: Python package to create/manipulate OpenDocumentFormat files.", + "pandas_ods_reader: Python library to read in ODS files."], + "input": "Attachment attribute containing a .ods document.", + "output": "Text and freetext parsed from the document.", + "references": [], + "features": "The module reads the text contained in a .ods document. The result is passed to the freetext import parser so IoCs can be extracted out of it." +} diff --git a/doc/expansion/odt-enrich.json b/doc/expansion/odt-enrich.json new file mode 100644 index 0000000..e201c77 --- /dev/null +++ b/doc/expansion/odt-enrich.json @@ -0,0 +1,9 @@ +{ + "description": "Module to extract freetext from a .odt document.", + "logo": "logos/odt.png", + "requirements": ["ODT reader python library."], + "input": "Attachment attribute containing a .odt document.", + "output": "Text and freetext parsed from the document.", + "references": [], + "features": "The module reads the text contained in a .odt document. The result is passed to the freetext import parser so IoCs can be extracted out of it." +} diff --git a/doc/expansion/onyphe.json b/doc/expansion/onyphe.json new file mode 100644 index 0000000..04ebdd3 --- /dev/null +++ b/doc/expansion/onyphe.json @@ -0,0 +1,9 @@ +{ + "description": "Module to process a query on Onyphe.", + "logo": "logos/onyphe.jpg", + "requirements": ["onyphe python library", "An access to the Onyphe API (apikey)"], + "input": "A domain, hostname or IP address MISP attribute.", + "output": "MISP attributes fetched from the Onyphe query.", + "references": ["https://www.onyphe.io/", "https://github.com/sebdraven/pyonyphe"], + "features": "This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted." +} diff --git a/doc/expansion/onyphe_full.json b/doc/expansion/onyphe_full.json new file mode 100644 index 0000000..4b722fa --- /dev/null +++ b/doc/expansion/onyphe_full.json @@ -0,0 +1,9 @@ +{ + "description": "Module to process a full query on Onyphe.", + "logo": "logos/onyphe.jpg", + "requirements": ["onyphe python library", "An access to the Onyphe API (apikey)"], + "input": "A domain, hostname or IP address MISP attribute.", + "output": "MISP attributes fetched from the Onyphe query.", + "references": ["https://www.onyphe.io/", "https://github.com/sebdraven/pyonyphe"], + "features": "This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted.\n\nThe parsing is here more advanced than the one on onyphe module, and is returning more attributes, since more fields of the query result are watched and parsed." +} diff --git a/doc/expansion/otx.json b/doc/expansion/otx.json new file mode 100644 index 0000000..c6032cc --- /dev/null +++ b/doc/expansion/otx.json @@ -0,0 +1,9 @@ +{ + "description": "Module to get information from AlienVault OTX.", + "logo": "logos/otx.png", + "requirements": ["An access to the OTX API (apikey)"], + "input": "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512", + "output": "MISP attributes mapped from the result of the query on OTX, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- email", + "references": ["https://www.alienvault.com/open-threat-exchange"], + "features": "This module takes a MISP attribute as input to query the OTX Alienvault API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes." +} diff --git a/doc/expansion/passivetotal.json b/doc/expansion/passivetotal.json new file mode 100644 index 0000000..ef8b044 --- /dev/null +++ b/doc/expansion/passivetotal.json @@ -0,0 +1,9 @@ +{ + "description": "", + "logo": "logos/passivetotal.png", + "requirements": ["Passivetotal python library", "An access to the PassiveTotal API (apikey)"], + "input": "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- x509-fingerprint-sha1\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-phone\n- text\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date", + "output": "MISP attributes mapped from the result of the query on PassiveTotal, included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- x509-fingerprint-sha1\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-phone\n- text\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- md5\n- sha1\n- sha256\n- link", + "references": ["https://www.passivetotal.org/register"], + "features": "The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register" +} diff --git a/doc/expansion/pdf-enrich.json b/doc/expansion/pdf-enrich.json new file mode 100644 index 0000000..5b3f0a8 --- /dev/null +++ b/doc/expansion/pdf-enrich.json @@ -0,0 +1,9 @@ +{ + "description": "Module to extract freetext from a PDF document.", + "logo": "logos/pdf.jpg", + "requirements": ["pdftotext: Python library to extract text from PDF."], + "input": "Attachment attribute containing a PDF document.", + "output": "Text and freetext parsed from the document.", + "references": [], + "features": "The module reads the text contained in a PDF document. The result is passed to the freetext import parser so IoCs can be extracted out of it." +} diff --git a/doc/expansion/pptx-enrich.json b/doc/expansion/pptx-enrich.json new file mode 100644 index 0000000..aff0d8d --- /dev/null +++ b/doc/expansion/pptx-enrich.json @@ -0,0 +1,9 @@ +{ + "description": "Module to extract freetext from a .pptx document.", + "logo": "logos/pptx.png", + "requirements": ["pptx: Python library to read PowerPoint files."], + "input": "Attachment attribute containing a .pptx document.", + "output": "Text and freetext parsed from the document.", + "references": [], + "features": "The module reads the text contained in a .pptx document. The result is passed to the freetext import parser so IoCs can be extracted out of it." +} diff --git a/doc/expansion/qrcode.json b/doc/expansion/qrcode.json new file mode 100644 index 0000000..38ed77c --- /dev/null +++ b/doc/expansion/qrcode.json @@ -0,0 +1,9 @@ +{ + "description": "Module to decode QR codes.", + "requirements": ["cv2: The OpenCV python library.", + "pyzbar: Python library to read QR codes."], + "input": "A QR code stored as attachment attribute.", + "output": "The URL or bitcoin address the QR code is pointing to.", + "references": [], + "features": "The module reads the QR code and returns the related address, which can be an URL or a bitcoin address." +} diff --git a/doc/expansion/rbl.json b/doc/expansion/rbl.json new file mode 100644 index 0000000..9700eca --- /dev/null +++ b/doc/expansion/rbl.json @@ -0,0 +1,8 @@ +{ + "description": "Module to check an IPv4 address against known RBLs.", + "requirements": ["dnspython3: DNS python3 library"], + "input": "IP address attribute.", + "output": "Text with additional data from Real-time Blackhost Lists about the IP address.", + "references": ["[RBLs list](https://github.com/MISP/misp-modules/blob/8817de476572a10a9c9d03258ec81ca70f3d926d/misp_modules/modules/expansion/rbl.py#L20)"], + "features": "This module takes an IP address attribute as input and queries multiple know Real-time Blackhost Lists to check if they have already seen this IP address.\n\nWe display then all the information we get from those different sources." +} diff --git a/doc/expansion/reversedns.json b/doc/expansion/reversedns.json new file mode 100644 index 0000000..6934462 --- /dev/null +++ b/doc/expansion/reversedns.json @@ -0,0 +1,7 @@ +{ + "description": "Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.", + "requirements": ["DNS python library"], + "input": "An IP address attribute.", + "output": "Hostname attribute the input is resolved into.", + "features": "The module takes an IP address as input and tries to find the hostname this IP address is resolved into.\n\nThe address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8).\n\nPlease note that composite MISP attributes containing IP addresses are supported as well." +} diff --git a/doc/expansion/securitytrails.json b/doc/expansion/securitytrails.json new file mode 100644 index 0000000..8541e4e --- /dev/null +++ b/doc/expansion/securitytrails.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion modules for SecurityTrails.", + "logo": "logos/securitytrails.png", + "requirements": ["dnstrails python library", "An access to the SecurityTrails API (apikey)"], + "input": "A domain, hostname or IP address attribute.", + "output": "MISP attributes resulting from the query on SecurityTrails API, included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- dns-soa-email\n- whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- domain", + "references": ["https://securitytrails.com/"], + "features": "The module takes a domain, hostname or IP address attribute as input and queries the SecurityTrails API with it.\n\nMultiple parsing operations are then processed on the result of the query to extract a much information as possible.\n\nFrom this data extracted are then mapped MISP attributes." +} diff --git a/doc/expansion/shodan.json b/doc/expansion/shodan.json new file mode 100644 index 0000000..57241f0 --- /dev/null +++ b/doc/expansion/shodan.json @@ -0,0 +1,9 @@ +{ + "description": "Module to query on Shodan.", + "logo": "logos/shodan.png", + "requirements": ["shodan python library", "An access to the Shodan API (apikey)"], + "input": "An IP address MISP attribute.", + "output": "Text with additional data about the input, resulting from the query on Shodan.", + "references": ["https://www.shodan.io/"], + "features": "The module takes an IP address as input and queries the Shodan API to get some additional data about it." +} diff --git a/doc/expansion/sigma_queries.json b/doc/expansion/sigma_queries.json new file mode 100644 index 0000000..f127ba4 --- /dev/null +++ b/doc/expansion/sigma_queries.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion hover module to display the result of sigma queries.", + "logo": "logos/sigma.png", + "requirements": ["Sigma python library"], + "input": "A Sigma attribute.", + "output": "Text displaying results of queries on the Sigma attribute.", + "references": ["https://github.com/Neo23x0/sigma/wiki"], + "features": "This module takes a Sigma rule attribute as input and tries all the different queries available to convert it into different formats recognized by SIEMs." +} diff --git a/doc/expansion/sigma_syntax_validator.json b/doc/expansion/sigma_syntax_validator.json new file mode 100644 index 0000000..8e17ae0 --- /dev/null +++ b/doc/expansion/sigma_syntax_validator.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion hover module to perform a syntax check on sigma rules.", + "logo": "logos/sigma.png", + "requirements": ["Sigma python library", "Yaml python library"], + "input": "A Sigma attribute.", + "output": "Text describing the validity of the Sigma rule.", + "references": ["https://github.com/Neo23x0/sigma/wiki"], + "features": "This module takes a Sigma rule attribute as input and performs a syntax check on it.\n\nIt displays then that the rule is valid if it is the case, and the error related to the rule otherwise." +} diff --git a/doc/expansion/sourcecache.json b/doc/expansion/sourcecache.json new file mode 100644 index 0000000..ab4669c --- /dev/null +++ b/doc/expansion/sourcecache.json @@ -0,0 +1,8 @@ +{ + "description": "Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page.", + "requirements": ["urlarchiver: python library to fetch and archive URL on the file-system"], + "input": "A link or url attribute.", + "output": "A malware-sample attribute describing the cached page.", + "references": ["https://github.com/adulau/url_archiver"], + "features": "This module takes a link or url attribute as input and caches the related web page. It returns then a link of the cached page." +} diff --git a/doc/expansion/stix2_pattern_syntax_validator.json b/doc/expansion/stix2_pattern_syntax_validator.json new file mode 100644 index 0000000..2ea43b5 --- /dev/null +++ b/doc/expansion/stix2_pattern_syntax_validator.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion hover module to perform a syntax check on stix2 patterns.", + "logo": "logos/stix.png", + "requirements": ["stix2patterns python library"], + "input": "A STIX2 pattern attribute.", + "output": "Text describing the validity of the STIX2 pattern.", + "references": ["[STIX2.0 patterning specifications](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html)"], + "features": "This module takes a STIX2 pattern attribute as input and performs a syntax check on it.\n\nIt displays then that the rule is valid if it is the case, and the error related to the rule otherwise." +} diff --git a/doc/expansion/threatcrowd.json b/doc/expansion/threatcrowd.json new file mode 100644 index 0000000..99725b8 --- /dev/null +++ b/doc/expansion/threatcrowd.json @@ -0,0 +1,8 @@ +{ + "description": "Module to get information from ThreatCrowd.", + "logo": "logos/threatcrowd.png", + "input": "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512\n- whois-registrant-email", + "output": "MISP attributes mapped from the result of the query on ThreatCrowd, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- hostname\n- whois-registrant-email", + "references": ["https://www.threatcrowd.org/"], + "features": "This module takes a MISP attribute as input and queries ThreatCrowd with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." +} diff --git a/doc/expansion/threatminer.json b/doc/expansion/threatminer.json new file mode 100644 index 0000000..d2f26bd --- /dev/null +++ b/doc/expansion/threatminer.json @@ -0,0 +1,8 @@ +{ + "description": "Module to get information from ThreatMiner.", + "logo": "logos/threatminer.png", + "input": "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512", + "output": "MISP attributes mapped from the result of the query on ThreatMiner, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- ssdeep\n- authentihash\n- filename\n- whois-registrant-email\n- url\n- link", + "references": ["https://www.threatminer.org/"], + "features": "This module takes a MISP attribute as input and queries ThreatMiner with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." +} diff --git a/doc/expansion/trustar_enrich.json b/doc/expansion/trustar_enrich.json new file mode 100644 index 0000000..294419d --- /dev/null +++ b/doc/expansion/trustar_enrich.json @@ -0,0 +1,8 @@ +{ + "description": "Module to get enrich indicators with TruSTAR.", + "logo": "logos/trustar.png", + "input": "Any of the following MISP attributes:\n- btc\n- domain\n- email-src\n- filename\n- hostname\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- url", + "output": "MISP attributes enriched with indicator summary data from the TruSTAR API. Data includes a severity level score and additional source and scoring info.", + "references": ["https://docs.trustar.co/api/v13/indicators/get_indicator_summaries.html"], + "features": "This module enriches MISP attributes with scoring and metadata from TruSTAR.\n\nThe TruSTAR indicator summary is appended to the attributes along with links to any associated reports." +} diff --git a/doc/expansion/urlhaus.json b/doc/expansion/urlhaus.json new file mode 100644 index 0000000..8e5cef3 --- /dev/null +++ b/doc/expansion/urlhaus.json @@ -0,0 +1,9 @@ +{ + "description": "Query of the URLhaus API to get additional information about the input attribute.", + "logo": "logos/urlhaus.png", + "requirements": [], + "input": "A domain, hostname, url, ip, md5 or sha256 attribute.", + "output": "MISP attributes & objects fetched from the result of the URLhaus API query.", + "references": ["https://urlhaus.abuse.ch/"], + "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module takes one of the attribute type specified as input, and query the URLhaus API with it. If any result is returned by the API, attributes and objects are created accordingly." +} diff --git a/doc/expansion/urlscan.json b/doc/expansion/urlscan.json new file mode 100644 index 0000000..d847761 --- /dev/null +++ b/doc/expansion/urlscan.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion module to query urlscan.io.", + "logo": "logos/urlscan.jpg", + "requirements": ["An access to the urlscan.io API"], + "input": "A domain, hostname or url attribute.", + "output": "MISP attributes mapped from the result of the query on urlscan.io.", + "references": ["https://urlscan.io/"], + "features": "This module takes a MISP attribute as input and queries urlscan.io with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." +} diff --git a/doc/expansion/virustotal.json b/doc/expansion/virustotal.json new file mode 100644 index 0000000..31fd6ac --- /dev/null +++ b/doc/expansion/virustotal.json @@ -0,0 +1,9 @@ +{ + "description": "Module to get advanced information from virustotal.", + "logo": "logos/virustotal.png", + "requirements": ["An access to the VirusTotal API (apikey), with a high request rate limit."], + "input": "A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute.", + "output": "MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute.", + "references": ["https://www.virustotal.com/", "https://developers.virustotal.com/reference"], + "features": "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.\n\nThus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them." +} diff --git a/doc/expansion/virustotal_public.json b/doc/expansion/virustotal_public.json new file mode 100644 index 0000000..242c734 --- /dev/null +++ b/doc/expansion/virustotal_public.json @@ -0,0 +1,9 @@ +{ + "description": "Module to get information from VirusTotal.", + "logo": "logos/virustotal.png", + "requirements": ["An access to the VirusTotal API (apikey)"], + "input": "A domain, hostname, ip, url or hash (md5, sha1, sha256 or sha512) attribute.", + "output": "MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute.", + "references": ["https://www.virustotal.com", "https://developers.virustotal.com/reference"], + "features": "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.\n\nThus, it only queries the API once and returns the results that is parsed into MISP attributes and objects." +} diff --git a/doc/expansion/vmray_submit.json b/doc/expansion/vmray_submit.json new file mode 100644 index 0000000..ea6cf3f --- /dev/null +++ b/doc/expansion/vmray_submit.json @@ -0,0 +1,9 @@ +{ + "description": "Module to submit a sample to VMRay.", + "logo": "logos/vmray.png", + "requirements": ["An access to the VMRay API (apikey & url)"], + "input": "An attachment or malware-sample attribute.", + "output": "MISP attributes mapped from the result of the query on VMRay API, included in the following list:\n- text\n- sha1\n- sha256\n- md5\n- link", + "references": ["https://www.vmray.com/"], + "features": "This module takes an attachment or malware-sample attribute as input to query the VMRay API.\n\nThe sample contained within the attribute in then enriched with data from VMRay mapped into MISP attributes." +} diff --git a/doc/expansion/vulndb.json b/doc/expansion/vulndb.json new file mode 100644 index 0000000..330a3eb --- /dev/null +++ b/doc/expansion/vulndb.json @@ -0,0 +1,9 @@ +{ + "description": "Module to query VulnDB (RiskBasedSecurity.com).", + "logo": "logos/vulndb.png", + "requirements": ["An access to the VulnDB API (apikey, apisecret)"], + "input": "A vulnerability attribute.", + "output": "Additional data enriching the CVE input, fetched from VulnDB.", + "references": ["https://vulndb.cyberriskanalytics.com/"], + "features": "This module takes a vulnerability attribute as input and queries VulnDB in order to get some additional data about it.\n\nThe API gives the result of the query which can be displayed in the screen, and/or mapped into MISP attributes to add in the event." +} diff --git a/doc/expansion/vulners.json b/doc/expansion/vulners.json new file mode 100644 index 0000000..f3f3026 --- /dev/null +++ b/doc/expansion/vulners.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion hover module to expand information about CVE id using Vulners API.", + "logo": "logos/vulners.png", + "requirements": ["Vulners python library", "An access to the Vulners API"], + "input": "A vulnerability attribute.", + "output": "Text giving additional information about the CVE in input.", + "references": ["https://vulners.com/"], + "features": "This module takes a vulnerability attribute as input and queries the Vulners API in order to get some additional data about it.\n\nThe API then returns details about the vulnerability." +} diff --git a/doc/expansion/whois.json b/doc/expansion/whois.json new file mode 100644 index 0000000..938bad5 --- /dev/null +++ b/doc/expansion/whois.json @@ -0,0 +1,8 @@ +{ + "description": "Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).", + "requirements": ["uwhois: A whois python library"], + "input": "A domain or IP address attribute.", + "output": "Text describing the result of a whois request for the input value.", + "references": ["https://github.com/rafiot/uwhoisd"], + "features": "This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server)." +} diff --git a/doc/expansion/wiki.json b/doc/expansion/wiki.json new file mode 100644 index 0000000..d6de62b --- /dev/null +++ b/doc/expansion/wiki.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.", + "logo": "logos/wikidata.png", + "requirements": ["SPARQLWrapper python library"], + "input": "Text attribute.", + "output": "Text attribute.", + "references": ["https://www.wikidata.org"], + "features": "This module takes a text attribute as input and queries the Wikidata API. If the text attribute is clear enough to define a specific term, the API returns a wikidata link in response." +} diff --git a/doc/expansion/xforceexchange.json b/doc/expansion/xforceexchange.json new file mode 100644 index 0000000..bbe3c86 --- /dev/null +++ b/doc/expansion/xforceexchange.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion module for IBM X-Force Exchange.", + "logo": "logos/xforce.png", + "requirements": ["An access to the X-Force API (apikey)"], + "input": "A MISP attribute included in the following list:\n- ip-src\n- ip-dst\n- vulnerability\n- md5\n- sha1\n- sha256", + "output": "MISP attributes mapped from the result of the query on X-Force Exchange.", + "references": ["https://exchange.xforce.ibmcloud.com/"], + "features": "This module takes a MISP attribute as input to query the X-Force API. The API returns then additional information known in their threats data, that is mapped into MISP attributes." +} diff --git a/doc/expansion/xlsx-enrich.json b/doc/expansion/xlsx-enrich.json new file mode 100644 index 0000000..c41f17c --- /dev/null +++ b/doc/expansion/xlsx-enrich.json @@ -0,0 +1,9 @@ +{ + "description": "Module to extract freetext from a .xlsx document.", + "logo": "logos/xlsx.png", + "requirements": ["pandas: Python library to perform data analysis, time series and statistics."], + "input": "Attachment attribute containing a .xlsx document.", + "output": "Text and freetext parsed from the document.", + "references": [], + "features": "The module reads the text contained in a .xlsx document. The result is passed to the freetext import parser so IoCs can be extracted out of it." +} diff --git a/doc/expansion/yara_query.json b/doc/expansion/yara_query.json new file mode 100644 index 0000000..408353d --- /dev/null +++ b/doc/expansion/yara_query.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion & hover module to translate any hash attribute into a yara rule.", + "logo": "logos/yara.png", + "requirements": ["yara-python python library"], + "features": "The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it. This YARA rule is also validated using the same method as in 'yara_syntax_validator' module.\nBoth hover and expansion functionalities are supported with this module, where the hover part is displaying the resulting YARA rule and the expansion part allows you to add the rule as a new attribute, as usual with expansion modules.", + "input": "MISP Hash attribute (md5, sha1, sha256, imphash, or any of the composite attribute with filename and one of the previous hash type).", + "output": "YARA rule.", + "references": ["https://virustotal.github.io/yara/", "https://github.com/virustotal/yara-python"] +} diff --git a/doc/expansion/yara_syntax_validator.json b/doc/expansion/yara_syntax_validator.json new file mode 100644 index 0000000..93a96ee --- /dev/null +++ b/doc/expansion/yara_syntax_validator.json @@ -0,0 +1,9 @@ +{ + "description": "An expansion hover module to perform a syntax check on if yara rules are valid or not.", + "logo": "logos/yara.png", + "requirements": ["yara_python python library"], + "input": "YARA rule attribute.", + "output": "Text to inform users if their rule is valid.", + "references": ["http://virustotal.github.io/yara/"], + "features": "This modules simply takes a YARA rule as input, and checks its syntax. It returns then a confirmation if the syntax is valid, otherwise the syntax error is displayed." +} diff --git a/doc/export_mod/cef_export.json b/doc/export_mod/cef_export.json new file mode 100644 index 0000000..84bba8e --- /dev/null +++ b/doc/export_mod/cef_export.json @@ -0,0 +1,8 @@ +{ + "description": "Module to export a MISP event in CEF format.", + "requirements": [], + "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in Common Event Format.\nThus, there is no particular feature concerning MISP Events since any event can be exported. However, 4 configuration parameters recognized by CEF format are required and should be provided by users before exporting data: the device vendor, product and version, as well as the default severity of data.", + "references": ["https://community.softwaregrp.com/t5/ArcSight-Connectors/ArcSight-Common-Event-Format-CEF-Guide/ta-p/1589306?attachment-id=65537"], + "input": "MISP Event attributes", + "output": "Common Event Format file" +} diff --git a/doc/export_mod/cisco_firesight_manager_ACL_rule_export.json b/doc/export_mod/cisco_firesight_manager_ACL_rule_export.json new file mode 100644 index 0000000..6d1d0dd --- /dev/null +++ b/doc/export_mod/cisco_firesight_manager_ACL_rule_export.json @@ -0,0 +1,9 @@ +{ + "description": "Module to export malicious network activity attributes to Cisco fireSIGHT manager block rules.", + "logo": "logos/cisco.png", + "requirements": ["Firesight manager console credentials"], + "input": "Network activity attributes (IPs, URLs).", + "output": "Cisco fireSIGHT manager block rules.", + "references": [], + "features": "The module goes through the attributes to find all the network activity ones in order to create block rules for the Cisco fireSIGHT manager." +} diff --git a/doc/export_mod/goamlexport.json b/doc/export_mod/goamlexport.json new file mode 100644 index 0000000..57a1587 --- /dev/null +++ b/doc/export_mod/goamlexport.json @@ -0,0 +1,9 @@ +{ + "description": "This module is used to export MISP events containing transaction objects into GoAML format.", + "logo": "logos/goAML.jpg", + "requirements": ["PyMISP","MISP objects"], + "features": "The module works as long as there is at least one transaction object in the Event.\n\nThen in order to have a valid GoAML document, please follow these guidelines:\n- For each transaction object, use either a bank-account, person, or legal-entity object to describe the origin of the transaction, and again one of them to describe the target of the transaction.\n- Create an object reference for both origin and target objects of the transaction.\n- A bank-account object needs a signatory, which is a person object, put as object reference of the bank-account.\n- A person can have an address, which is a geolocation object, put as object reference of the person.\n\nSupported relation types for object references that are recommended for each object are the folowing:\n- transaction:\n\t- 'from', 'from_my_client': Origin of the transaction - at least one of them is required.\n\t- 'to', 'to_my_client': Target of the transaction - at least one of them is required.\n\t- 'address': Location of the transaction - optional.\n- bank-account:\n\t- 'signatory': Signatory of a bank-account - the reference from bank-account to a signatory is required, but the relation-type is optional at the moment since this reference will always describe a signatory.\n\t- 'entity': Entity owning the bank account - optional.\n- person:\n\t- 'address': Address of a person - optional.", + "references": ["http://goaml.unodc.org/"], + "input": "MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target.", + "output": "GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities)." +} diff --git a/doc/export_mod/liteexport.json b/doc/export_mod/liteexport.json new file mode 100644 index 0000000..110577c --- /dev/null +++ b/doc/export_mod/liteexport.json @@ -0,0 +1,8 @@ +{ + "description": "Lite export of a MISP event.", + "requirements": [], + "features": "This module is simply producing a json MISP event format file, but exporting only Attributes from the Event. Thus, MISP Events exported with this module should have attributes that are not internal references, otherwise the resulting event would be empty.", + "references": [], + "input": "MISP Event attributes", + "output": "Lite MISP Event" +} diff --git a/doc/export_mod/mass_eql_export.json b/doc/export_mod/mass_eql_export.json new file mode 100644 index 0000000..5eadd23 --- /dev/null +++ b/doc/export_mod/mass_eql_export.json @@ -0,0 +1,9 @@ +{ + "description": "Mass EQL query export for a MISP event.", + "logo": "logos/eql.png", + "requirements": [], + "features": "This module produces EQL queries for all relevant attributes in a MISP event.", + "references": ["https://eql.readthedocs.io/en/latest/"], + "input": "MISP Event attributes", + "output": "Text file containing one or more EQL queries" + } diff --git a/doc/export_mod/nexthinkexport.json b/doc/export_mod/nexthinkexport.json new file mode 100644 index 0000000..182448c --- /dev/null +++ b/doc/export_mod/nexthinkexport.json @@ -0,0 +1,9 @@ +{ + "description": "Nexthink NXQL query export module", + "requirements": [], + "features": "This module export an event as Nexthink NXQL queries that can then be used in your own python3 tool or from wget/powershell", + "references": ["https://doc.nexthink.com/Documentation/Nexthink/latest/APIAndIntegrations/IntroducingtheWebAPIV2"], + "input": "MISP Event attributes", + "output": "Nexthink NXQL queries", + "logo": "logos/nexthink.svg" +} diff --git a/doc/export_mod/osqueryexport.json b/doc/export_mod/osqueryexport.json new file mode 100644 index 0000000..6543cb1 --- /dev/null +++ b/doc/export_mod/osqueryexport.json @@ -0,0 +1,9 @@ +{ + "description": "OSQuery export of a MISP event.", + "requirements": [], + "features": "This module export an event as osquery queries that can be used in packs or in fleet management solution like Kolide.", + "references": [], + "input": "MISP Event attributes", + "output": "osquery SQL queries", + "logo": "logos/osquery.png" +} diff --git a/doc/export_mod/pdfexport.json b/doc/export_mod/pdfexport.json new file mode 100644 index 0000000..f1654dc --- /dev/null +++ b/doc/export_mod/pdfexport.json @@ -0,0 +1,8 @@ +{ + "description": "Simple export of a MISP event to PDF.", + "requirements": ["PyMISP", "reportlab"], + "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option ", + "references": ["https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html"], + "input": "MISP Event", + "output": "MISP Event in a PDF file." +} diff --git a/doc/export_mod/testexport.json b/doc/export_mod/testexport.json new file mode 100644 index 0000000..213ea92 --- /dev/null +++ b/doc/export_mod/testexport.json @@ -0,0 +1,3 @@ +{ + "description": "Skeleton export module." +} diff --git a/doc/export_mod/threatStream_misp_export.json b/doc/export_mod/threatStream_misp_export.json new file mode 100644 index 0000000..3fdc50a --- /dev/null +++ b/doc/export_mod/threatStream_misp_export.json @@ -0,0 +1,9 @@ +{ + "description": "Module to export a structured CSV file for uploading to threatStream.", + "logo": "logos/threatstream.png", + "requirements": ["csv"], + "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatStream.", + "references": ["https://www.anomali.com/platform/threatstream", "https://github.com/threatstream"], + "input": "MISP Event attributes", + "output": "ThreatStream CSV format file" +} diff --git a/doc/export_mod/threat_connect_export.json b/doc/export_mod/threat_connect_export.json new file mode 100644 index 0000000..8d19572 --- /dev/null +++ b/doc/export_mod/threat_connect_export.json @@ -0,0 +1,9 @@ +{ + "description": "Module to export a structured CSV file for uploading to ThreatConnect.", + "logo": "logos/threatconnect.png", + "requirements": ["csv"], + "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatConnect.\nUsers should then provide, as module configuration, the source of data they export, because it is required by the output format.", + "references": ["https://www.threatconnect.com"], + "input": "MISP Event attributes", + "output": "ThreatConnect CSV format file" +} diff --git a/doc/export_mod/vt_graph.json b/doc/export_mod/vt_graph.json new file mode 100644 index 0000000..e317730 --- /dev/null +++ b/doc/export_mod/vt_graph.json @@ -0,0 +1,9 @@ +{ + "description": "This module is used to create a VirusTotal Graph from a MISP event.", + "logo": "logos/virustotal.png", + "requirements": ["vt_graph_api, the python library to query the VirusTotal graph API"], + "features": "The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event.\n\nOnce the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal.", + "references": ["https://www.virustotal.com/gui/graph-overview"], + "input": "A MISP event.", + "output": "Link of the VirusTotal Graph created for the event." +} diff --git a/doc/generate_documentation.py b/doc/generate_documentation.py new file mode 100644 index 0000000..f86b5a7 --- /dev/null +++ b/doc/generate_documentation.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +import os +import json + +module_types = ['expansion', 'export_mod', 'import_mod'] +titles = ['Expansion Modules', 'Export Modules', 'Import Modules'] +markdown = ["# MISP modules documentation\n"] +githublink = 'https://github.com/MISP/misp-modules/tree/master/misp_modules/modules' + + +def generate_doc(root_path): + for _path, title in zip(module_types, titles): + markdown.append('\n## {}\n'.format(title)) + current_path = os.path.join(root_path, _path) + files = sorted(os.listdir(current_path)) + githubpath = '{}/{}'.format(githublink, _path) + for _file in files: + modulename = _file.split('.json')[0] + githubref = '{}/{}.py'.format(githubpath, modulename) + markdown.append('\n#### [{}]({})\n'.format(modulename, githubref)) + filename = os.path.join(current_path, _file) + with open(filename, 'rt') as f: + definition = json.loads(f.read()) + if 'logo' in definition: + markdown.append('\n\n'.format(definition.pop('logo'))) + if 'description' in definition: + markdown.append('\n{}\n'.format(definition.pop('description'))) + for field, value in sorted(definition.items()): + if value: + value = ', '.join(value) if isinstance(value, list) else '{}'.format(value.replace('\n', '\n>')) + markdown.append('- **{}**:\n>{}\n'.format(field, value)) + markdown.append('\n-----\n') + with open('README.md', 'w') as w: + w.write(''.join(markdown)) + +def generate_docs_for_mkdocs(root_path): + for _path, title in zip(module_types, titles): + markdown = [] + #markdown.append('## {}\n'.format(title)) + current_path = os.path.join(root_path, _path) + files = sorted(os.listdir(current_path)) + githubpath = '{}/{}'.format(githublink, _path) + for _file in files: + modulename = _file.split('.json')[0] + githubref = '{}/{}.py'.format(githubpath, modulename) + markdown.append('\n#### [{}]({})\n'.format(modulename, githubref)) + filename = os.path.join(current_path, _file) + with open(filename, 'rt') as f: + definition = json.loads(f.read()) + if 'logo' in definition: + markdown.append('\n\n'.format(definition.pop('logo'))) + if 'description' in definition: + markdown.append('\n{}\n'.format(definition.pop('description'))) + for field, value in sorted(definition.items()): + if value: + value = ', '.join(value) if isinstance(value, list) else '{}'.format(value.replace('\n', '\n>')) + markdown.append('- **{}**:\n>{}\n'.format(field, value)) + markdown.append('\n-----\n') + with open(root_path+"/../"+"/docs/"+_path+".md", 'w') as w: + w.write(''.join(markdown)) + +if __name__ == '__main__': + root_path = os.path.dirname(os.path.realpath(__file__)) + generate_doc(root_path) + generate_docs_for_mkdocs(root_path) diff --git a/doc/import_mod/csvimport.json b/doc/import_mod/csvimport.json new file mode 100644 index 0000000..66a10fd --- /dev/null +++ b/doc/import_mod/csvimport.json @@ -0,0 +1,8 @@ +{ + "description": "Module to import MISP attributes from a csv file.", + "requirements": ["PyMISP"], + "features": "In order to parse data from a csv file, a header is required to let the module know which column is matching with known attribute fields / MISP types.\n\nThis header either comes from the csv file itself or is part of the configuration of the module and should be filled out in MISP plugin settings, each field separated by COMMAS. Fields that do not match with any type known in MISP or are not MISP attribute fields should be ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment, ').\n\nIf the csv file already contains a header that does not start by a '#', you should tick the checkbox 'has_header' to avoid importing it and have potential issues. You can also redefine the header even if it is already contained in the file, by following the rules for headers explained earlier. One reason why you would redefine a header is for instance when you want to skip some fields, or some fields are not valid types.", + "references": ["https://tools.ietf.org/html/rfc4180", "https://tools.ietf.org/html/rfc7111"], + "input": "CSV format file.", + "output": "MISP Event attributes" +} diff --git a/doc/import_mod/cuckooimport.json b/doc/import_mod/cuckooimport.json new file mode 100644 index 0000000..8091d07 --- /dev/null +++ b/doc/import_mod/cuckooimport.json @@ -0,0 +1,9 @@ +{ + "description": "Module to import Cuckoo JSON.", + "logo": "logos/cuckoo.png", + "requirements": [], + "features": "The module simply imports MISP Attributes from a Cuckoo JSON format file. There is thus no special feature to make it work.", + "references": ["https://cuckoosandbox.org/", "https://github.com/cuckoosandbox/cuckoo"], + "input": "Cuckoo JSON file", + "output": "MISP Event attributes" +} diff --git a/doc/import_mod/email_import.json b/doc/import_mod/email_import.json new file mode 100644 index 0000000..1f53852 --- /dev/null +++ b/doc/import_mod/email_import.json @@ -0,0 +1,8 @@ +{ + "description": "Module to import emails in MISP.", + "requirements": [], + "features": "This module can be used to import e-mail text as well as attachments and urls.\n3 configuration parameters are then used to unzip attachments, guess zip attachment passwords, and extract urls: set each one of them to True or False to process or not the respective corresponding actions.", + "references": [], + "input": "E-mail file", + "output": "MISP Event attributes" +} diff --git a/doc/import_mod/goamlimport.json b/doc/import_mod/goamlimport.json new file mode 100644 index 0000000..f2a1ec2 --- /dev/null +++ b/doc/import_mod/goamlimport.json @@ -0,0 +1,9 @@ +{ + "description": "Module to import MISP objects about financial transactions from GoAML files.", + "logo": "logos/goAML.jpg", + "requirements": ["PyMISP"], + "features": "Unlike the GoAML export module, there is here no special feature to import data from GoAML external files, since the module will import MISP Objects with their References on its own, as it is required for the export module to rebuild a valid GoAML document.", + "references": "http://goaml.unodc.org/", + "input": "GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities).", + "output": "MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target." +} diff --git a/doc/import_mod/joe_import.json b/doc/import_mod/joe_import.json new file mode 100644 index 0000000..ceba4ab --- /dev/null +++ b/doc/import_mod/joe_import.json @@ -0,0 +1,9 @@ +{ + "description": "A module to import data from a Joe Sandbox analysis json report.", + "logo": "logos/joesandbox.png", + "requirements": [], + "input": "Json report of a Joe Sandbox analysis.", + "output": "MISP attributes & objects parsed from the analysis report.", + "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], + "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report.\n\n" +} diff --git a/doc/import_mod/lastline_import.json b/doc/import_mod/lastline_import.json new file mode 100644 index 0000000..99414e0 --- /dev/null +++ b/doc/import_mod/lastline_import.json @@ -0,0 +1,9 @@ +{ + "description": "Module to import and parse reports from Lastline analysis links.", + "logo": "logos/lastline.png", + "requirements": [], + "input": "Link to a Lastline analysis.", + "output": "MISP attributes and objects parsed from the analysis report.", + "references": ["https://www.lastline.com"], + "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." +} diff --git a/doc/import_mod/mispjson.json b/doc/import_mod/mispjson.json new file mode 100644 index 0000000..dd11405 --- /dev/null +++ b/doc/import_mod/mispjson.json @@ -0,0 +1,8 @@ +{ + "description": "Module to import MISP JSON format for merging MISP events.", + "requirements": [], + "features": "The module simply imports MISP Attributes from an other MISP Event in order to merge events together. There is thus no special feature to make it work.", + "references": [], + "input": "MISP Event", + "output": "MISP Event attributes" +} diff --git a/doc/import_mod/ocr.json b/doc/import_mod/ocr.json new file mode 100644 index 0000000..14bbf0b --- /dev/null +++ b/doc/import_mod/ocr.json @@ -0,0 +1,8 @@ +{ + "description": "Optical Character Recognition (OCR) module for MISP.", + "requirements": [], + "features": "The module tries to recognize some text from an image and import the result as a freetext attribute, there is then no special feature asked to users to make it work.", + "references": [], + "input": "Image", + "output": "freetext MISP attribute" +} diff --git a/doc/import_mod/openiocimport.json b/doc/import_mod/openiocimport.json new file mode 100644 index 0000000..e173392 --- /dev/null +++ b/doc/import_mod/openiocimport.json @@ -0,0 +1,8 @@ +{ + "description": "Module to import OpenIOC packages.", + "requirements": ["PyMISP"], + "features": "The module imports MISP Attributes from OpenIOC packages, there is then no special feature for users to make it work.", + "references": ["https://www.fireeye.com/blog/threat-research/2013/10/openioc-basics.html"], + "input": "OpenIOC packages", + "output": "MISP Event attributes" +} diff --git a/doc/import_mod/threatanalyzer_import.json b/doc/import_mod/threatanalyzer_import.json new file mode 100644 index 0000000..40e4436 --- /dev/null +++ b/doc/import_mod/threatanalyzer_import.json @@ -0,0 +1,8 @@ +{ + "description": "Module to import ThreatAnalyzer archive.zip / analysis.json files.", + "requirements": [], + "features": "The module imports MISP Attributes from a ThreatAnalyzer format file. This file can be either ZIP, or JSON format.\nThere is by the way no special feature for users to make the module work.", + "references": ["https://www.threattrack.com/malware-analysis.aspx"], + "input": "ThreatAnalyzer format file", + "output": "MISP Event attributes" +} diff --git a/doc/import_mod/vmray_import.json b/doc/import_mod/vmray_import.json new file mode 100644 index 0000000..b7c0dad --- /dev/null +++ b/doc/import_mod/vmray_import.json @@ -0,0 +1,9 @@ +{ + "description": "Module to import VMRay (VTI) results.", + "logo": "logos/vmray.png", + "requirements": ["vmray_rest_api"], + "features": "The module imports MISP Attributes from VMRay format, using the VMRay api.\nUsers should then provide as the module configuration the API Key as well as the server url in order to fetch their data to import.", + "references": ["https://www.vmray.com/"], + "input": "VMRay format", + "output": "MISP Event attributes" +} diff --git a/doc/logos/apivoid.png b/doc/logos/apivoid.png new file mode 100644 index 0000000..e4f84a7 Binary files /dev/null and b/doc/logos/apivoid.png differ diff --git a/doc/logos/assemblyline.png b/doc/logos/assemblyline.png new file mode 100644 index 0000000..bda4518 Binary files /dev/null and b/doc/logos/assemblyline.png differ diff --git a/doc/logos/backscatter_io.png b/doc/logos/backscatter_io.png new file mode 100644 index 0000000..0973112 Binary files /dev/null and b/doc/logos/backscatter_io.png differ diff --git a/doc/logos/bitcoin.png b/doc/logos/bitcoin.png new file mode 100644 index 0000000..e80ad6d Binary files /dev/null and b/doc/logos/bitcoin.png differ diff --git a/doc/logos/cisco.png b/doc/logos/cisco.png new file mode 100644 index 0000000..87b863b Binary files /dev/null and b/doc/logos/cisco.png differ diff --git a/doc/logos/crowdstrike.png b/doc/logos/crowdstrike.png new file mode 100644 index 0000000..359cb01 Binary files /dev/null and b/doc/logos/crowdstrike.png differ diff --git a/doc/logos/cuckoo.png b/doc/logos/cuckoo.png new file mode 100644 index 0000000..57cf35a Binary files /dev/null and b/doc/logos/cuckoo.png differ diff --git a/doc/logos/cve.png b/doc/logos/cve.png new file mode 100644 index 0000000..315ccd8 Binary files /dev/null and b/doc/logos/cve.png differ diff --git a/doc/logos/cytomic_orion.png b/doc/logos/cytomic_orion.png new file mode 100644 index 0000000..45704e9 Binary files /dev/null and b/doc/logos/cytomic_orion.png differ diff --git a/doc/logos/docx.png b/doc/logos/docx.png new file mode 100644 index 0000000..018d2c1 Binary files /dev/null and b/doc/logos/docx.png differ diff --git a/doc/logos/domaintools.png b/doc/logos/domaintools.png new file mode 100644 index 0000000..69965e1 Binary files /dev/null and b/doc/logos/domaintools.png differ diff --git a/doc/logos/eql.png b/doc/logos/eql.png new file mode 100644 index 0000000..4cddb91 Binary files /dev/null and b/doc/logos/eql.png differ diff --git a/doc/logos/eupi.png b/doc/logos/eupi.png new file mode 100644 index 0000000..1800657 Binary files /dev/null and b/doc/logos/eupi.png differ diff --git a/doc/logos/farsight.png b/doc/logos/farsight.png new file mode 100644 index 0000000..31a73c1 Binary files /dev/null and b/doc/logos/farsight.png differ diff --git a/doc/logos/goAML.jpg b/doc/logos/goAML.jpg new file mode 100644 index 0000000..4e938ee Binary files /dev/null and b/doc/logos/goAML.jpg differ diff --git a/doc/logos/greynoise.png b/doc/logos/greynoise.png new file mode 100644 index 0000000..b4d4f91 Binary files /dev/null and b/doc/logos/greynoise.png differ diff --git a/doc/logos/hibp.png b/doc/logos/hibp.png new file mode 100644 index 0000000..849ccf2 Binary files /dev/null and b/doc/logos/hibp.png differ diff --git a/doc/logos/intelmq.png b/doc/logos/intelmq.png new file mode 100644 index 0000000..fad627c Binary files /dev/null and b/doc/logos/intelmq.png differ diff --git a/doc/logos/joesandbox.png b/doc/logos/joesandbox.png new file mode 100644 index 0000000..8072f6e Binary files /dev/null and b/doc/logos/joesandbox.png differ diff --git a/doc/logos/lastline.png b/doc/logos/lastline.png new file mode 100644 index 0000000..6bffe77 Binary files /dev/null and b/doc/logos/lastline.png differ diff --git a/doc/logos/macaddress_io.png b/doc/logos/macaddress_io.png new file mode 100644 index 0000000..e77f455 Binary files /dev/null and b/doc/logos/macaddress_io.png differ diff --git a/doc/logos/macvendors.png b/doc/logos/macvendors.png new file mode 100644 index 0000000..3316ea3 Binary files /dev/null and b/doc/logos/macvendors.png differ diff --git a/doc/logos/maxmind.png b/doc/logos/maxmind.png new file mode 100644 index 0000000..8f8a6c6 Binary files /dev/null and b/doc/logos/maxmind.png differ diff --git a/doc/logos/nexthink.svg b/doc/logos/nexthink.svg new file mode 100644 index 0000000..f18ba8f --- /dev/null +++ b/doc/logos/nexthink.svg @@ -0,0 +1,22 @@ + + + + nexthink + Created with Sketch. + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/logos/ods.png b/doc/logos/ods.png new file mode 100644 index 0000000..19b42f1 Binary files /dev/null and b/doc/logos/ods.png differ diff --git a/doc/logos/odt.png b/doc/logos/odt.png new file mode 100644 index 0000000..d177a21 Binary files /dev/null and b/doc/logos/odt.png differ diff --git a/doc/logos/onyphe.jpg b/doc/logos/onyphe.jpg new file mode 100644 index 0000000..cd16f76 Binary files /dev/null and b/doc/logos/onyphe.jpg differ diff --git a/doc/logos/osquery.png b/doc/logos/osquery.png new file mode 100644 index 0000000..2e4320e Binary files /dev/null and b/doc/logos/osquery.png differ diff --git a/doc/logos/otx.png b/doc/logos/otx.png new file mode 100644 index 0000000..eae32c1 Binary files /dev/null and b/doc/logos/otx.png differ diff --git a/doc/logos/passivedns.png b/doc/logos/passivedns.png new file mode 100644 index 0000000..4959a84 Binary files /dev/null and b/doc/logos/passivedns.png differ diff --git a/doc/logos/passivessl.png b/doc/logos/passivessl.png new file mode 100644 index 0000000..e92c87d Binary files /dev/null and b/doc/logos/passivessl.png differ diff --git a/doc/logos/passivetotal.png b/doc/logos/passivetotal.png new file mode 100644 index 0000000..87cef69 Binary files /dev/null and b/doc/logos/passivetotal.png differ diff --git a/doc/logos/pdf.jpg b/doc/logos/pdf.jpg new file mode 100644 index 0000000..74f4297 Binary files /dev/null and b/doc/logos/pdf.jpg differ diff --git a/doc/logos/pptx.png b/doc/logos/pptx.png new file mode 100644 index 0000000..11b2133 Binary files /dev/null and b/doc/logos/pptx.png differ diff --git a/doc/logos/securitytrails.png b/doc/logos/securitytrails.png new file mode 100644 index 0000000..072dac5 Binary files /dev/null and b/doc/logos/securitytrails.png differ diff --git a/doc/logos/shodan.png b/doc/logos/shodan.png new file mode 100644 index 0000000..7de068e Binary files /dev/null and b/doc/logos/shodan.png differ diff --git a/doc/logos/sigma.png b/doc/logos/sigma.png new file mode 100644 index 0000000..0bd0db1 Binary files /dev/null and b/doc/logos/sigma.png differ diff --git a/doc/logos/spamhaus.jpg b/doc/logos/spamhaus.jpg new file mode 100644 index 0000000..4c868e4 Binary files /dev/null and b/doc/logos/spamhaus.jpg differ diff --git a/doc/logos/stix.png b/doc/logos/stix.png new file mode 100644 index 0000000..e8b8241 Binary files /dev/null and b/doc/logos/stix.png differ diff --git a/doc/logos/threatconnect.png b/doc/logos/threatconnect.png new file mode 100644 index 0000000..4c8a5b1 Binary files /dev/null and b/doc/logos/threatconnect.png differ diff --git a/doc/logos/threatcrowd.png b/doc/logos/threatcrowd.png new file mode 100644 index 0000000..94eacfc Binary files /dev/null and b/doc/logos/threatcrowd.png differ diff --git a/doc/logos/threatminer.png b/doc/logos/threatminer.png new file mode 100644 index 0000000..d7ac96e Binary files /dev/null and b/doc/logos/threatminer.png differ diff --git a/doc/logos/threatstream.png b/doc/logos/threatstream.png new file mode 100644 index 0000000..eb3837e Binary files /dev/null and b/doc/logos/threatstream.png differ diff --git a/doc/logos/trustar.png b/doc/logos/trustar.png new file mode 100644 index 0000000..d4ac521 Binary files /dev/null and b/doc/logos/trustar.png differ diff --git a/doc/logos/urlhaus.png b/doc/logos/urlhaus.png new file mode 100644 index 0000000..3460d81 Binary files /dev/null and b/doc/logos/urlhaus.png differ diff --git a/doc/logos/urlscan.jpg b/doc/logos/urlscan.jpg new file mode 100644 index 0000000..52e24e2 Binary files /dev/null and b/doc/logos/urlscan.jpg differ diff --git a/doc/logos/virustotal.png b/doc/logos/virustotal.png new file mode 100644 index 0000000..935c5cc Binary files /dev/null and b/doc/logos/virustotal.png differ diff --git a/doc/logos/vmray.png b/doc/logos/vmray.png new file mode 100644 index 0000000..e2e9fa1 Binary files /dev/null and b/doc/logos/vmray.png differ diff --git a/doc/logos/vulndb.png b/doc/logos/vulndb.png new file mode 100644 index 0000000..bfaf40f Binary files /dev/null and b/doc/logos/vulndb.png differ diff --git a/doc/logos/vulners.png b/doc/logos/vulners.png new file mode 100644 index 0000000..ef9bab4 Binary files /dev/null and b/doc/logos/vulners.png differ diff --git a/doc/logos/wikidata.png b/doc/logos/wikidata.png new file mode 100644 index 0000000..0ffb4b1 Binary files /dev/null and b/doc/logos/wikidata.png differ diff --git a/doc/logos/xforce.png b/doc/logos/xforce.png new file mode 100644 index 0000000..96db659 Binary files /dev/null and b/doc/logos/xforce.png differ diff --git a/doc/logos/xlsx.png b/doc/logos/xlsx.png new file mode 100644 index 0000000..fbe6e13 Binary files /dev/null and b/doc/logos/xlsx.png differ diff --git a/doc/logos/yara.png b/doc/logos/yara.png new file mode 100644 index 0000000..c74c314 Binary files /dev/null and b/doc/logos/yara.png differ diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..8ac6d9f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,130 @@ +FROM python:3.7-buster AS build + +ENV DEBIAN_FRONTEND noninteractive +ENV WORKDIR="/usr/local/src/misp_modules" +ENV VENV_DIR="/misp_modules" + +# Install Packages for build +RUN set -eu \ + ;mkdir -p ${WORKDIR} ${VENV_DIR} \ + ;apt-get update \ + ;apt-get install -y \ + git \ + libpq5 \ + libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick \ + virtualenv \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + libzbar-dev \ + libfuzzy-dev \ + ;apt-get -y autoremove \ + ;apt-get -y clean \ + ;rm -rf /var/lib/apt/lists/* \ + ; + +# Create MISP Modules +RUN set -eu \ + ;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \ + ;virtualenv -p python3 ${VENV_DIR}/venv \ + ;cd ${WORKDIR} \ + ;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \ + ;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \ + ; + +######################################### +# Start Final Docker Image +# +FROM python:3.7-slim-buster AS final + +ENV DEBIAN_FRONTEND noninteractive +ENV VENV_DIR="/misp_modules" + +# Copy all builded files from build stage +COPY --from=build ${VENV_DIR} ${VENV_DIR} + +# Install Packages to run it +RUN set -eu \ + ;apt-get update \ + ;apt-get install -y \ + curl \ + libpq5 \ + # libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick \ + # virtualenv \ + # libopencv-dev \ + zbar-tools \ + libzbar0 \ + # libzbar-dev \ + # libfuzzy-dev \ + ;apt-get -y autoremove \ + ;apt-get -y clean \ + ;rm -rf /var/lib/apt/lists/* \ + ;chown -R nobody ${VENV_DIR} \ + ; + +# Entrypoint + COPY files/entrypoint.sh /entrypoint.sh + ENTRYPOINT [ "/entrypoint.sh" ] + +# Add Healthcheck Config + COPY files/healthcheck.sh /healthcheck.sh + HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"] + +# Change Workdir + WORKDIR ${VENV_DIR} + +# Change from root to www-data + USER nobody + +# Expose Port + EXPOSE 6666 + +# Shortterm ARG Variables: + ARG VENDOR="MISP" + ARG COMPONENT="misp-modules" + ARG BUILD_DATE + ARG GIT_REPO="https://github.com/MISP/misp-modules" + ARG VCS_REF + ARG RELEASE_DATE + ARG NAME="MISP-dockerized-misp-modules" + ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container." + ARG DOCUMENTATION="https://misp.github.io/misp-modules/" + ARG AUTHOR="MISP" + ARG LICENSE="BSD-3-Clause" + +# Longterm Environment Variables +ENV \ + BUILD_DATE=${BUILD_DATE} \ + NAME=${NAME} \ + PATH=$PATH:${VENV_DIR}/venv/bin + +# Labels +LABEL org.label-schema.build-date="${BUILD_DATE}" \ + org.label-schema.name="${NAME}" \ + org.label-schema.description="${DESCRIPTION}" \ + org.label-schema.vcs-ref="${VCS_REF}" \ + org.label-schema.vcs-url="${GIT_REPO}" \ + org.label-schema.url="${GIT_REPO}" \ + org.label-schema.vendor="${VENDOR}" \ + org.label-schema.version="${VERSION}" \ + org.label-schema.usage="${DOCUMENTATION}" \ + org.label-schema.schema-version="1.0.0-rc1" + +LABEL org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.url="${GIT_REPO}" \ + org.opencontainers.image.source="${GIT_REPO}" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.vendor="${VENDOR}" \ + org.opencontainers.image.title="${NAME}" \ + org.opencontainers.image.description="${DESCRIPTION}" \ + org.opencontainers.image.documentation="${DOCUMENTATION}" \ + org.opencontainers.image.authors="${AUTHOR}" \ + org.opencontainers.image.licenses="${LICENSE}" + diff --git a/docker/files/entrypoint.sh b/docker/files/entrypoint.sh new file mode 100755 index 0000000..73d8f39 --- /dev/null +++ b/docker/files/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu + +# Variables +NC='\033[0m' # No Color +Light_Green='\033[1;32m' +STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}" +VENV_DIR=${VENV_DIR:-"/misp-modules"} +MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules" +DEBUG="" + +# Functions +echo (){ + command echo "$STARTMSG $*" +} + +# Environment Variables +MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"} + +# +# MAIN +# + + +# Check if debugging mode should be enabled +[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d" + +# check if a command parameter exists and start misp-modules +if [ $# = 0 ] +then + # If no cmd parameter is set + echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr +else + # If cmd parameter is set + echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr & + exec "$@" +fi diff --git a/docker/files/healthcheck.sh b/docker/files/healthcheck.sh new file mode 100755 index 0000000..d6a1f91 --- /dev/null +++ b/docker/files/healthcheck.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# If no contain is there or curl get an error back: exit 1. Docker restart then the container. +curl -fk http://0.0.0.0:6666/modules || exit 1 \ No newline at end of file diff --git a/docs/REQUIREMENTS.txt b/docs/REQUIREMENTS.txt new file mode 100644 index 0000000..ad07dd1 --- /dev/null +++ b/docs/REQUIREMENTS.txt @@ -0,0 +1,3 @@ +mkdocs +mkdocs-material +markdown_include \ No newline at end of file diff --git a/docs/contribute.md b/docs/contribute.md new file mode 100644 index 0000000..ef312f6 --- /dev/null +++ b/docs/contribute.md @@ -0,0 +1,374 @@ +## How to add your own MISP modules? + +Create your module in [misp_modules/modules/expansion/](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/), [misp_modules/modules/export_mod/](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/), or [misp_modules/modules/import_mod/](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/). The module should have at minimum three functions: + +* **introspection** function that returns a dict of the supported attributes (input and output) by your expansion module. +* **handler** function which accepts a JSON document to expand the values and return a dictionary of the expanded values. +* **version** function that returns a dict with the version and the associated meta-data including potential configurations required of the module. + +Don't forget to return an error key and value if an error is raised to propagate it to the MISP user-interface. + +Your module's script name should also be added in the `__all__` list of `/__init__.py` in order for it to be loaded. + +~~~python +... + # Checking for required value + if not request.get('ip-src'): + # Return an error message + return {'error': "A source IP is required"} +... +~~~ + + +### introspection + +The function that returns a dict of the supported attributes (input and output) by your expansion module. + +~~~python +mispattributes = {'input': ['link', 'url'], + 'output': ['attachment', 'malware-sample']} + +def introspection(): + return mispattributes +~~~ + +### version + +The function that returns a dict with the version and the associated meta-data including potential configurations required of the module. + + +### Additional Configuration Values + +If your module requires additional configuration (to be exposed via the MISP user-interface), you can define those in the moduleconfig value returned by the version function. + +~~~python +# config fields that your code expects from the site admin +moduleconfig = ["apikey", "event_limit"] + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo +~~~ + + +When you do this a config array is added to the meta-data output containing all the potential configuration values: + +~~~ +"meta": { + "description": "PassiveTotal expansion service to expand values with multiple Passive DNS sources", + "config": [ + "username", + "password" + ], + "module-type": [ + "expansion", + "hover" + ], + +... +~~~ + + +If you want to use the configuration values set in the web interface they are stored in the key `config` in the JSON object passed to the handler. + +~~~ +def handler(q=False): + + # Check if we were given a configuration + config = q.get("config", {}) + + # Find out if there is a username field + username = config.get("username", None) +~~~ + + +### handler + +The function which accepts a JSON document to expand the values and return a dictionary of the expanded values. + +~~~python +def handler(q=False): + "Fully functional rot-13 encoder" + if q is False: + return False + request = json.loads(q) + src = request.get('ip-src') + if src is None: + # Return an error message + return {'error': "A source IP is required"} + else: + return {'results': + codecs.encode(src, "rot-13")} +~~~ + +#### export module + +For an export module, the `request["data"]` object corresponds to a list of events (dictionaries) to handle. + +Iterating over events attributes is performed using their `Attribute` key. + +~~~python +... +for event in request["data"]: + for attribute in event["Attribute"]: + # do stuff w/ attribute['type'], attribute['value'], ... +... + +### Returning Binary Data + +If you want to return a file or other data you need to add a data attribute. + +~~~python +{"results": {"values": "filename.txt", + "types": "attachment", + "data" : base64.b64encode() # base64 encode your data first + "comment": "This is an attachment"}} +~~~ + +If the binary file is malware you can use 'malware-sample' as the type. If you do this the malware sample will be automatically zipped and password protected ('infected') after being uploaded. + + +~~~python +{"results": {"values": "filename.txt", + "types": "malware-sample", + "data" : base64.b64encode() # base64 encode your data first + "comment": "This is an attachment"}} +~~~ + +[To learn more about how data attributes are processed you can read the processing code here.](https://github.com/MISP/PyMISP/blob/4f230c9299ad9d2d1c851148c629b61a94f3f117/pymisp/mispevent.py#L185-L200) + + +### Module type + +A MISP module can be of four types: + +- **expansion** - service related to an attribute that can be used to extend and update an existing event. +- **hover** - service related to an attribute to provide additional information to the users without updating the event. +- **import** - service related to importing and parsing an external object that can be used to extend an existing event. +- **export** - service related to exporting an object, event, or data. + +module-type is an array where the list of supported types can be added. + +## Testing your modules? + +MISP uses the **modules** function to discover the available MISP modules and their supported MISP attributes: + +~~~ +% curl -s http://127.0.0.1:6666/modules | jq . +[ + { + "name": "passivetotal", + "type": "expansion", + "mispattributes": { + "input": [ + "hostname", + "domain", + "ip-src", + "ip-dst" + ], + "output": [ + "ip-src", + "ip-dst", + "hostname", + "domain" + ] + }, + "meta": { + "description": "PassiveTotal expansion service to expand values with multiple Passive DNS sources", + "config": [ + "username", + "password" + ], + "author": "Alexandre Dulaunoy", + "version": "0.1" + } + }, + { + "name": "sourcecache", + "type": "expansion", + "mispattributes": { + "input": [ + "link" + ], + "output": [ + "link" + ] + }, + "meta": { + "description": "Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page.", + "author": "Alexandre Dulaunoy", + "version": "0.1" + } + }, + { + "name": "dns", + "type": "expansion", + "mispattributes": { + "input": [ + "hostname", + "domain" + ], + "output": [ + "ip-src", + "ip-dst" + ] + }, + "meta": { + "description": "Simple DNS expansion service to resolve IP address from MISP attributes", + "author": "Alexandre Dulaunoy", + "version": "0.1" + } + } +] + +~~~ + +The MISP module service returns the available modules in a JSON array containing each module name along with their supported input attributes. + +Based on this information, a query can be built in a JSON format and saved as body.json: + +~~~json +{ + "hostname": "www.foo.be", + "module": "dns" +} +~~~ + +Then you can POST this JSON format query towards the MISP object server: + +~~~bash +curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @body.json -X POST +~~~ + +The module should output the following JSON: + +~~~json +{ + "results": [ + { + "types": [ + "ip-src", + "ip-dst" + ], + "values": [ + "188.65.217.78" + ] + } + ] +} +~~~ + +It is also possible to restrict the category options of the resolved attributes by passing a list of categories along (optional): + +~~~json +{ + "results": [ + { + "types": [ + "ip-src", + "ip-dst" + ], + "values": [ + "188.65.217.78" + ], + "categories": [ + "Network activity", + "Payload delivery" + ] + } + ] +} +~~~ + +For both the type and the category lists, the first item in the list will be the default setting on the interface. + +### Enable your module in the web interface + +For a module to be activated in the MISP web interface it must be enabled in the "Plugin Settings. + +Go to "Administration > Server Settings" in the top menu +- Go to "Plugin Settings" in the top "tab menu bar" +- Click on the name of the type of module you have created to expand the list of plugins to show your module. +- Find the name of your plugin's "enabled" value in the Setting Column. +"Plugin.[MODULE NAME]_enabled" +- Double click on its "Value" column + +~~~ +Priority Setting Value Description Error Message +Recommended Plugin.Import_ocr_enabled false Enable or disable the ocr module. Value not set. +~~~ + +- Use the drop-down to set the enabled value to 'true' + +~~~ +Priority Setting Value Description Error Message +Recommended Plugin.Import_ocr_enabled true Enable or disable the ocr module. Value not set. +~~~ + +### Set any other required settings for your module + +In this same menu set any other plugin settings that are required for testing. + + + +## Documentation + +In order to provide documentation about some modules that require specific input / output / configuration, the [doc](https://github.com/MISP/misp-modules/tree/master/doc) directory contains detailed information about the general purpose, requirements, features, input and output of each of these modules: + +- ***description** - quick description of the general purpose of the module, as the one given by the moduleinfo +- **requirements** - special libraries needed to make the module work +- **features** - description of the way to use the module, with the required MISP features to make the module give the intended result +- **references** - link(s) giving additional information about the format concerned in the module +- **input** - description of the format of data used in input +- **output** - description of the format given as the result of the module execution + +In addition to the module documentation please add your module to [docs/index.md](https://github.com/MISP/misp-modules/tree/master/docs/index.md). + +There are also [complementary slides](https://www.misp-project.org/misp-training/3.1-misp-modules.pdf) for the creation of MISP modules. + + +## Tips for developers creating modules + +Download a pre-built virtual image from the [MISP training materials](https://www.circl.lu/services/misp-training-materials/). + +- Create a Host-Only adapter in VirtualBox +- Set your Misp OVA to that Host-Only adapter +- Start the virtual machine +- Get the IP address of the virutal machine +- SSH into the machine (Login info on training page) +- Go into the misp-modules directory + +~~~bash +cd /usr/local/src/misp-modules +~~~ + +Set the git repo to your fork and checkout your development branch. If you SSH'ed in as the misp user you will have to use sudo. + +~~~bash +sudo git remote set-url origin https://github.com/YourRepo/misp-modules.git +sudo git pull +sudo git checkout MyModBranch +~~~ + +Remove the contents of the build directory and re-install misp-modules. + +~~~python +sudo rm -fr build/* +sudo pip3 install --upgrade . +~~~ + +SSH in with a different terminal and run `misp-modules` with debugging enabled. + +~~~python +sudo killall misp-modules +misp-modules -d +~~~ + + +In your original terminal you can now run your tests manually and see any errors that arrive + +~~~bash +cd tests/ +curl -s http://127.0.0.1:6666/query -H "Content-Type: application/json" --data @MY_TEST_FILE.json -X POST +cd ../ +~~~ diff --git a/docs/img/favicon.ico b/docs/img/favicon.ico new file mode 100644 index 0000000..dca12d1 Binary files /dev/null and b/docs/img/favicon.ico differ diff --git a/docs/img/misp.png b/docs/img/misp.png new file mode 100644 index 0000000..5f2d4dd Binary files /dev/null and b/docs/img/misp.png differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..1297a3b --- /dev/null +++ b/docs/index.md @@ -0,0 +1,120 @@ +# Home + +[![Build Status](https://travis-ci.org/MISP/misp-modules.svg?branch=master)](https://travis-ci.org/MISP/misp-modules) +[![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=master)](https://coveralls.io/github/MISP/misp-modules?branch=master) +[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/master/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules) +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_shield) + +MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP). + +The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities +without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. + +MISP modules support is included in MISP starting from version `2.4.28`. + +For more information: [Extending MISP with Python modules](https://www.circl.lu/assets/files/misp-training/switch2016/2-misp-modules.pdf) slides from MISP training. + + +## Existing MISP modules + +### Expansion modules + +* [Backscatter.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations. +* [BGP Ranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. +* [BTC scam check](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused. +* [BTC transactions](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. +* [CIRCL Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. +* [CIRCL Passive SSL](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. +* [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. +* [CrowdStrike Falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. +* [CVE](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). +* [CVE advanced](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). +* [Cuckoo submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. +* [DBL Spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. +* [DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. +* [docx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). +* [DomainTools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. +* [EUPI](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). +* [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate event query language (EQL) from an attribute. [Event Query Language](https://eql.readthedocs.io/en/latest/) +* [Farsight DNSDB Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. +* [GeoIP](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [Greynoise](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. +* [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. +* [hibp](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? +* [intel471](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). +* [IPASN](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. +* [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. +* [Joe Sandbox submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox. +* [Joe Sandbox query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data. +* [macaddress.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). +* [macvendors](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. +* [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP. +* [ods-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). +* [odt-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). +* [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. +* [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. +* [OTX](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). +* [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [pdf-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser). +* [pptx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser). +* [qrcode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values. +* [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. +* [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. +* [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/). +* [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. +* [Sigma queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. +* [Sigma syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. +* [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. +* [STIX2 pattern syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. +* [ThreatCrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). +* [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). +* [urlhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url. +* [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io). +* [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference)) +* [virustotal_public](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal_public.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a public key and a low request rate limit. (More details about the API: [here](https://developers.virustotal.com/reference)) +* [VMray](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray. +* [VulnDB](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/). +* [Vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API. +* [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd). +* [wikidata](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. +* [xforce](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. +* [xlsx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser). +* [YARA query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes. +* [YARA syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator. + +### Export modules + +* [CEF](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). +* [Cisco FireSight Manager ACL rule](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL. +* [GoAML export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). +* [Lite Export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) module to export a lite event. +* [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) module to export applicable attributes from an event to a mass EQL query. +* [PDF export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. +* [Nexthink query format](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format. +* [osquery](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format. +* [ThreatConnect](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. +* [ThreatStream](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format. + +### Import modules + +* [CSV import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module. +* [Cuckoo JSON](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. +* [Email Import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. +* [GoAML import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. +* [Joe Sandbox import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report. +* [OCR](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. +* [OpenIOC](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. +* [ThreatAnalyzer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. +* [VMRay](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. + + +## How to contribute your own module? + +Fork the project, add your module, test it and make a pull-request. Modules can be also private as you can add a module in your own MISP installation. +For further information please see [Contribute](contribute/). + + +## Licenses +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_large) + +For further Information see also the [license file](license/). \ No newline at end of file diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..662e675 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,191 @@ +## How to install and start MISP modules (in a Python virtualenv)? + +~~~~bash +SUDO_WWW="sudo -u www-data" + +sudo apt-get install -y \ + git \ + libpq5 \ + libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick virtualenv \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + libzbar-dev \ + libfuzzy-dev + +# BEGIN with virtualenv: +$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv +# END with virtualenv + +cd /usr/local/src/ +# Ideally you add your user to the staff group and make /usr/local/src group writeable, below follows an example with user misp +sudo adduser misp staff +sudo chmod 2775 /usr/local/src +sudo chown root:staff /usr/local/src +git clone https://github.com/MISP/misp-modules.git +git clone git://github.com/stricaud/faup.git faup +git clone git://github.com/stricaud/gtcaca.git gtcaca + +# Install gtcaca/faup +cd gtcaca +mkdir -p build +cd build +cmake .. && make +sudo make install +cd ../../faup +mkdir -p build +cd build +cmake .. && make +sudo make install +sudo ldconfig + +cd ../../misp-modules + +# BEGIN with virtualenv: +$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS +$SUDO_WWW /var/www/MISP/venv/bin/pip install . +# END with virtualenv + +# BEGIN without virtualenv: +sudo pip install -I -r REQUIREMENTS +sudo pip install . +# END without virtualenv + +# Start misp-modules as a service +sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now misp-modules +/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules +~~~~ + +## How to install and start MISP modules on RHEL-based distributions ? + +As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository. + +~~~~bash +SUDO_WWW="sudo -u apache" +sudo yum install \ + rh-ruby22 \ + openjpeg-devel \ + rubygem-rouge \ + rubygem-asciidoctor \ + zbar-devel \ + opencv-devel \ + gcc-c++ \ + pkgconfig \ + poppler-cpp-devel \ + python-devel \ + redhat-rpm-config +cd /usr/local/src/ +sudo git clone https://github.com/MISP/misp-modules.git +cd misp-modules +$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" +$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS +$SUDO_WWW /var/www/MISP/venv/bin/pip install -U . +~~~~ + +Create the service file /etc/systemd/system/misp-modules.service : + +~~~~bash +echo "[Unit] +Description=MISP's modules +After=misp-workers.service + +[Service] +Type=simple +User=apache +Group=apache +ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s' +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service +~~~~ + +The After=misp-workers.service must be changed or removed if you have not created a misp-workers service. Then, enable the misp-modules service and start it: + +~~~~bash +systemctl daemon-reload +systemctl enable --now misp-modules +~~~~ + +## How to use an MISP modules Docker container + +### Docker build + +~~~~bash +docker build -t misp-modules \ + --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \ + docker/ +~~~~ + +### Docker run + +~~~~bash +# Start Redis +docker run --rm -d --name=misp-redis redis:alpine +# Start MISP-modules +docker run \ + --rm -d --name=misp-modules \ + -e REDIS_BACKEND=misp-redis \ + -e REDIS_PORT="6379" \ + -e REDIS_PW="" \ + -e REDIS_DATABASE="245" \ + -e MISP_MODULES_DEBUG="false" \ + dcso/misp-dockerized-misp-modules +~~~~ + +### Docker-compose + +~~~~yml +services: + misp-modules: + # https://hub.docker.com/r/dcso/misp-dockerized-misp-modules + image: dcso/misp-dockerized-misp-modules:3 + + # Local image: + #image: misp-modules + #build: + # context: docker/ + + environment: + # Redis + REDIS_BACKEND: misp-redis + REDIS_PORT: "6379" + REDIS_DATABASE: "245" + # System PROXY (OPTIONAL) + http_proxy: + https_proxy: + no_proxy: 0.0.0.0 + # Timezone (OPTIONAL) + TZ: Europe/Berlin + # MISP-Modules (OPTIONAL) + MISP_MODULES_DEBUG: "false" + # Logging options (OPTIONAL) + LOG_SYSLOG_ENABLED: "no" + misp-redis: + # https://hub.docker.com/_/redis or alternative https://hub.docker.com/r/dcso/misp-dockerized-redis/ + image: redis:alpine +~~~~ + +## Install misp-module on an offline instance. +First, you need to grab all necessary packages for example like this : + +Use pip wheel to create an archive +~~~ +mkdir misp-modules-offline +pip3 wheel -r REQUIREMENTS shodan --wheel-dir=./misp-modules-offline +tar -cjvf misp-module-bundeled.tar.bz2 ./misp-modules-offline/* +~~~ +On offline machine : +~~~ +mkdir misp-modules-bundle +tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle +cd misp-modules-bundle +ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done +~~~ +Next you can follow standard install procedure. diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..dbbe355 --- /dev/null +++ b/docs/license.md @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/etc/systemd/system/misp-modules.service b/etc/systemd/system/misp-modules.service new file mode 100644 index 0000000..99cd102 --- /dev/null +++ b/etc/systemd/system/misp-modules.service @@ -0,0 +1,14 @@ +[Unit] +Description=System-wide instance of the MISP Modules +After=network.target + +[Service] +User=www-data +Group=www-data +WorkingDirectory=/usr/local/src/misp-modules +Environment="PATH=/var/www/MISP/venv/bin" +ExecStart=/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s + +[Install] +WantedBy=multi-user.target + diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 1c1713b..440ad3f 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -29,6 +29,7 @@ import fnmatch import argparse import re import datetime +import psutil import tornado.web import tornado.process @@ -37,14 +38,14 @@ from tornado.concurrent import run_on_executor from concurrent.futures import ThreadPoolExecutor try: - from .modules import * + from .modules import * # noqa HAS_PACKAGE_MODULES = True except Exception as e: print(e) HAS_PACKAGE_MODULES = False try: - from .helpers import * + from .helpers import * # noqa HAS_PACKAGE_HELPERS = True except Exception as e: print(e) @@ -54,7 +55,7 @@ log = logging.getLogger('misp-modules') def handle_signal(sig, frame): - IOLoop.instance().add_callback(IOLoop.instance().stop) + IOLoop.instance().add_callback_from_signal(IOLoop.instance().stop) def init_logger(level=False): @@ -147,7 +148,7 @@ def load_package_modules(): mhandlers = {} modules = [] for path, module in sys.modules.items(): - r = re.findall("misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path) + r = re.findall(r"misp_modules[.]modules[.](\w+)[.]([^_]\w+)", path) if r and len(r[0]) == 2: moduletype, modulename = r[0] mhandlers[modulename] = module @@ -158,6 +159,9 @@ def load_package_modules(): class ListModules(tornado.web.RequestHandler): + global loaded_modules + global mhandlers + def get(self): ret = [] for module in loaded_modules: @@ -193,7 +197,7 @@ class QueryModule(tornado.web.RequestHandler): if dict_payload.get('timeout'): timeout = datetime.timedelta(seconds=int(dict_payload.get('timeout'))) else: - timeout = datetime.timedelta(seconds=30) + timeout = datetime.timedelta(seconds=300) response = yield tornado.gen.with_timeout(timeout, self.run_request(jsonpayload)) self.write(response) except tornado.gen.TimeoutError: @@ -206,48 +210,89 @@ class QueryModule(tornado.web.RequestHandler): self.finish() +def _launch_from_current_dir(): + log.info('Launch MISP modules server from current directory.') + os.chdir(os.path.dirname(__file__)) + modulesdir = 'modules' + helpersdir = 'helpers' + load_helpers(helpersdir=helpersdir) + return load_modules(modulesdir) + + def main(): global mhandlers global loaded_modules signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) - argParser = argparse.ArgumentParser(description='misp-modules server') + argParser = argparse.ArgumentParser(description='misp-modules server', formatter_class=argparse.RawTextHelpFormatter) argParser.add_argument('-t', default=False, action='store_true', help='Test mode') argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') argParser.add_argument('-d', default=False, action='store_true', help='Enable debugging') argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)') argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)') argParser.add_argument('-m', default=[], action='append', help='Register a custom module') + argParser.add_argument('--devel', default=False, action='store_true', help='''Start in development mode, enable debug, start only the module(s) listed in -m.\nExample: -m misp_modules.modules.expansion.bgpranking''') args = argParser.parse_args() port = args.p listen = args.l - log = init_logger(level=args.d) - if args.s: - log.info('Launch MISP modules server from package.') - load_package_helpers() - mhandlers, loaded_modules = load_package_modules() + if args.devel: + log = init_logger(level=True) + log.info('Launch MISP modules server in developement mode. Enable debug, load a list of modules is -m is used.') + if args.m: + mhandlers = {} + modules = [] + for module in args.m: + splitted = module.split(".") + modulename = splitted[-1] + moduletype = splitted[2] + mhandlers[modulename] = importlib.import_module(module) + mhandlers['type:' + modulename] = moduletype + modules.append(modulename) + log.info('MISP modules {0} imported'.format(modulename)) + else: + mhandlers, loaded_modules = _launch_from_current_dir() else: - log.info('Launch MISP modules server from current directory.') - os.chdir(os.path.dirname(__file__)) - modulesdir = 'modules' - helpersdir = 'helpers' - load_helpers(helpersdir=helpersdir) - mhandlers, loaded_modules = load_modules(modulesdir) + log = init_logger(level=args.d) + if args.s: + log.info('Launch MISP modules server from package.') + load_package_helpers() + mhandlers, loaded_modules = load_package_modules() + else: + mhandlers, loaded_modules = _launch_from_current_dir() + + for module in args.m: + mispmod = importlib.import_module(module) + mispmod.register(mhandlers, loaded_modules) - for module in args.m: - mispmod = importlib.import_module(module) - mispmod.register(mhandlers, loaded_modules) - service = [(r'/modules', ListModules), (r'/query', QueryModule)] application = tornado.web.Application(service) - application.listen(port, address=listen) + try: + application.listen(port, address=listen) + except Exception as e: + if e.errno == 98: + pids = psutil.pids() + for pid in pids: + p = psutil.Process(pid) + if p.name() == "misp-modules": + print("\n\n\n") + print(e) + print("\nmisp-modules is still running as PID: {}\n".format(pid)) + print("Please kill accordingly:") + print("sudo kill {}".format(pid)) + sys.exit(-1) + print(e) + print("misp-modules might still be running.") + log.info('MISP modules server started on {0} port {1}'.format(listen, port)) if args.t: log.info('MISP modules started in test-mode, quitting immediately.') sys.exit() - IOLoop.instance().start() - IOLoop.instance().stop() + try: + IOLoop.instance().start() + finally: + IOLoop.instance().stop() + return 0 diff --git a/misp_modules/helpers/cache.py b/misp_modules/helpers/cache.py index 7d70159..f833b8e 100644 --- a/misp_modules/helpers/cache.py +++ b/misp_modules/helpers/cache.py @@ -19,32 +19,34 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import os + import redis import hashlib -port = 6379 -hostname = '127.0.0.1' -db = 5 +port = int(os.getenv("REDIS_PORT")) if os.getenv("REDIS_PORT") else 6379 +hostname = os.getenv("REDIS_BACKEND") or '127.0.0.1' +db = int(os.getenv("REDIS_DATABASE")) if os.getenv("REDIS_DATABASE") else 0 def selftest(enable=True): if not enable: return False - r = redis.StrictRedis(host=hostname, port=port, db=db) + r = redis.Redis(host=hostname, port=port, db=db) try: r.ping() - except: + except Exception: return 'Redis not running or not installed. Helper will be disabled.' def get(modulename=None, query=None, value=None, debug=False): if (modulename is None or query is None): return False - r = redis.StrictRedis(host=hostname, port=port, db=db) + r = redis.Redis(host=hostname, port=port, db=db, decode_responses=True) h = hashlib.sha1() h.update(query.encode('UTF-8')) hv = h.hexdigest() - key = "m:" + modulename + ":" + hv + key = "m:{}:{}".format(modulename, hv) if not r.exists(key): if debug: @@ -58,10 +60,11 @@ def get(modulename=None, query=None, value=None, debug=False): def flush(): - r = redis.StrictRedis(host=hostname, port=port, db=db) + r = redis.StrictRedis(host=hostname, port=port, db=db, decode_responses=True) returncode = r.flushdb() return returncode + if __name__ == "__main__": import sys if selftest() is not None: @@ -69,7 +72,7 @@ if __name__ == "__main__": else: print("Selftest ok") v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True) - if v == b'barfoo': + if v == 'barfoo': print("Cache ok") v = get(modulename="testmodule", query="abcdef") print(v) diff --git a/misp_modules/lib/__init__.py b/misp_modules/lib/__init__.py new file mode 100644 index 0000000..c078cf7 --- /dev/null +++ b/misp_modules/lib/__init__.py @@ -0,0 +1,3 @@ +from .vt_graph_parser import * # noqa + +all = ['joe_parser', 'lastline_api'] diff --git a/misp_modules/lib/joe_parser.py b/misp_modules/lib/joe_parser.py new file mode 100644 index 0000000..22a4918 --- /dev/null +++ b/misp_modules/lib/joe_parser.py @@ -0,0 +1,434 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +from datetime import datetime +from pymisp import MISPAttribute, MISPEvent, MISPObject +import json + + +arch_type_mapping = {'ANDROID': 'parse_apk', 'LINUX': 'parse_elf', 'WINDOWS': 'parse_pe'} +domain_object_mapping = {'@ip': ('ip-dst', 'ip'), '@name': ('domain', 'domain')} +dropped_file_mapping = {'@entropy': ('float', 'entropy'), + '@file': ('filename', 'filename'), + '@size': ('size-in-bytes', 'size-in-bytes'), + '@type': ('mime-type', 'mimetype')} +dropped_hash_mapping = {'MD5': 'md5', 'SHA': 'sha1', 'SHA-256': 'sha256', 'SHA-512': 'sha512'} +elf_object_mapping = {'epaddr': 'entrypoint-address', 'machine': 'arch', 'osabi': 'os_abi'} +elf_section_flags_mapping = {'A': 'ALLOC', 'I': 'INFO_LINK', 'M': 'MERGE', + 'S': 'STRINGS', 'T': 'TLS', 'W': 'WRITE', + 'X': 'EXECINSTR'} +file_object_fields = ['filename', 'md5', 'sha1', 'sha256', 'sha512', 'ssdeep'] +file_object_mapping = {'entropy': ('float', 'entropy'), + 'filesize': ('size-in-bytes', 'size-in-bytes'), + 'filetype': ('mime-type', 'mimetype')} +file_references_mapping = {'fileCreated': 'creates', 'fileDeleted': 'deletes', + 'fileMoved': 'moves', 'fileRead': 'reads', 'fileWritten': 'writes'} +network_behavior_fields = ('srcip', 'dstip', 'srcport', 'dstport') +network_connection_object_mapping = {'srcip': ('ip-src', 'ip-src'), 'dstip': ('ip-dst', 'ip-dst'), + 'srcport': ('port', 'src-port'), 'dstport': ('port', 'dst-port')} +pe_object_fields = {'entrypoint': ('text', 'entrypoint-address'), + 'imphash': ('imphash', 'imphash')} +pe_object_mapping = {'CompanyName': 'company-name', 'FileDescription': 'file-description', + 'FileVersion': 'file-version', 'InternalName': 'internal-filename', + 'LegalCopyright': 'legal-copyright', 'OriginalFilename': 'original-filename', + 'ProductName': 'product-filename', 'ProductVersion': 'product-version', + 'Translation': 'lang-id'} +pe_section_object_mapping = {'characteristics': ('text', 'characteristic'), + 'entropy': ('float', 'entropy'), + 'name': ('text', 'name'), 'rawaddr': ('hex', 'offset'), + 'rawsize': ('size-in-bytes', 'size-in-bytes'), + 'virtaddr': ('hex', 'virtual_address'), + 'virtsize': ('size-in-bytes', 'virtual_size')} +process_object_fields = {'cmdline': 'command-line', 'name': 'name', + 'parentpid': 'parent-pid', 'pid': 'pid', + 'path': 'current-directory'} +protocols = {'tcp': 4, 'udp': 4, 'icmp': 3, + 'http': 7, 'https': 7, 'ftp': 7} +registry_references_mapping = {'keyValueCreated': 'creates', 'keyValueModified': 'modifies'} +regkey_object_mapping = {'name': ('text', 'name'), 'newdata': ('text', 'data'), + 'path': ('regkey', 'key')} +signerinfo_object_mapping = {'sigissuer': ('text', 'issuer'), + 'version': ('text', 'version')} + + +class JoeParser(): + def __init__(self, config): + self.misp_event = MISPEvent() + self.references = defaultdict(list) + self.attributes = defaultdict(lambda: defaultdict(set)) + self.process_references = {} + + self.import_pe = config["import_pe"] + self.create_mitre_attack = config["mitre_attack"] + + def parse_data(self, data): + self.data = data + if self.analysis_type() == "file": + self.parse_fileinfo() + else: + self.parse_url_analysis() + + self.parse_system_behavior() + self.parse_network_behavior() + self.parse_screenshot() + self.parse_network_interactions() + self.parse_dropped_files() + + if self.attributes: + self.handle_attributes() + + if self.create_mitre_attack: + self.parse_mitre_attack() + + def build_references(self): + for misp_object in self.misp_event.objects: + object_uuid = misp_object.uuid + if object_uuid in self.references: + for reference in self.references[object_uuid]: + misp_object.add_reference(**reference) + + def handle_attributes(self): + for attribute_type, attribute in self.attributes.items(): + for attribute_value, references in attribute.items(): + attribute_uuid = self.create_attribute(attribute_type, attribute_value) + for reference in references: + source_uuid, relationship = reference + self.references[source_uuid].append(dict(referenced_uuid=attribute_uuid, + relationship_type=relationship)) + + def parse_dropped_files(self): + droppedinfo = self.data['droppedinfo'] + if droppedinfo: + for droppedfile in droppedinfo['hash']: + file_object = MISPObject('file') + for key, mapping in dropped_file_mapping.items(): + attribute_type, object_relation = mapping + file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': droppedfile[key], 'to_ids': False}) + if droppedfile['@malicious'] == 'true': + file_object.add_attribute('state', **{'type': 'text', 'value': 'Malicious', 'to_ids': False}) + for h in droppedfile['value']: + hash_type = dropped_hash_mapping[h['@algo']] + file_object.add_attribute(hash_type, **{'type': hash_type, 'value': h['$'], 'to_ids': False}) + self.misp_event.add_object(**file_object) + self.references[self.process_references[(int(droppedfile['@targetid']), droppedfile['@process'])]].append({ + 'referenced_uuid': file_object.uuid, + 'relationship_type': 'drops' + }) + + def parse_mitre_attack(self): + mitreattack = self.data['mitreattack'] + if mitreattack: + for tactic in mitreattack['tactic']: + if tactic.get('technique'): + for technique in tactic['technique']: + self.misp_event.add_tag('misp-galaxy:mitre-attack-pattern="{} - {}"'.format(technique['name'], technique['id'])) + + def parse_network_behavior(self): + network = self.data['behavior']['network'] + connections = defaultdict(lambda: defaultdict(set)) + for protocol, layer in protocols.items(): + if network.get(protocol): + for packet in network[protocol]['packet']: + timestamp = datetime.strptime(self.parse_timestamp(packet['timestamp']), '%b %d, %Y %H:%M:%S.%f') + connections[tuple(packet[field] for field in network_behavior_fields)][protocol].add(timestamp) + for connection, data in connections.items(): + attributes = self.prefetch_attributes_data(connection) + if len(data.keys()) == len(set(protocols[protocol] for protocol in data.keys())): + network_connection_object = MISPObject('network-connection') + for object_relation, attribute in attributes.items(): + network_connection_object.add_attribute(object_relation, **attribute) + network_connection_object.add_attribute('first-packet-seen', + **{'type': 'datetime', + 'value': min(tuple(min(timestamp) for timestamp in data.values())), + 'to_ids': False}) + for protocol in data.keys(): + network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), + **{'type': 'text', 'value': protocol, 'to_ids': False}) + self.misp_event.add_object(**network_connection_object) + self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, + relationship_type='initiates')) + else: + for protocol, timestamps in data.items(): + network_connection_object = MISPObject('network-connection') + for object_relation, attribute in attributes.items(): + network_connection_object.add_attribute(object_relation, **attribute) + network_connection_object.add_attribute('first-packet-seen', **{'type': 'datetime', 'value': min(timestamps), 'to_ids': False}) + network_connection_object.add_attribute('layer{}-protocol'.format(protocols[protocol]), **{'type': 'text', 'value': protocol, 'to_ids': False}) + self.misp_event.add_object(**network_connection_object) + self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=network_connection_object.uuid, + relationship_type='initiates')) + + def parse_screenshot(self): + screenshotdata = self.data['behavior']['screenshotdata'] + if screenshotdata: + screenshotdata = screenshotdata['interesting']['$'] + attribute = {'type': 'attachment', 'value': 'screenshot.jpg', + 'data': screenshotdata, 'disable_correlation': True, + 'to_ids': False} + self.misp_event.add_attribute(**attribute) + + def parse_system_behavior(self): + system = self.data['behavior']['system'] + if system.get('processes'): + process_activities = {'fileactivities': self.parse_fileactivities, + 'registryactivities': self.parse_registryactivities} + for process in system['processes']['process']: + general = process['general'] + process_object = MISPObject('process') + for feature, relation in process_object_fields.items(): + process_object.add_attribute(relation, **{'type': 'text', 'value': general[feature], 'to_ids': False}) + start_time = datetime.strptime('{} {}'.format(general['date'], general['time']), '%d/%m/%Y %H:%M:%S') + process_object.add_attribute('start-time', **{'type': 'datetime', 'value': start_time, 'to_ids': False}) + self.misp_event.add_object(**process_object) + for field, to_call in process_activities.items(): + if process.get(field): + to_call(process_object.uuid, process[field]) + self.references[self.analysisinfo_uuid].append(dict(referenced_uuid=process_object.uuid, + relationship_type='calls')) + self.process_references[(general['targetid'], general['path'])] = process_object.uuid + + def parse_fileactivities(self, process_uuid, fileactivities): + for feature, files in fileactivities.items(): + # ignore unknown features + if feature not in file_references_mapping: + continue + + if files: + for call in files['call']: + self.attributes['filename'][call['path']].add((process_uuid, file_references_mapping[feature])) + + def analysis_type(self): + generalinfo = self.data['generalinfo'] + + if generalinfo['target']['sample']: + return "file" + elif generalinfo['target']['url']: + return "url" + else: + raise Exception("Unknown analysis type") + + def parse_url_analysis(self): + generalinfo = self.data["generalinfo"] + + url_object = MISPObject("url") + self.analysisinfo_uuid = url_object.uuid + + url_object.add_attribute("url", generalinfo["target"]["url"], to_ids=False) + self.misp_event.add_object(**url_object) + + def parse_fileinfo(self): + fileinfo = self.data['fileinfo'] + + file_object = MISPObject('file') + self.analysisinfo_uuid = file_object.uuid + + for field in file_object_fields: + file_object.add_attribute(field, **{'type': field, 'value': fileinfo[field], 'to_ids': False}) + for field, mapping in file_object_mapping.items(): + attribute_type, object_relation = mapping + file_object.add_attribute(object_relation, **{'type': attribute_type, 'value': fileinfo[field], 'to_ids': False}) + arch = self.data['generalinfo']['arch'] + if arch in arch_type_mapping: + to_call = arch_type_mapping[arch] + getattr(self, to_call)(fileinfo, file_object) + else: + self.misp_event.add_object(**file_object) + + def parse_apk(self, fileinfo, file_object): + apkinfo = fileinfo['apk'] + self.misp_event.add_object(**file_object) + permission_lists = defaultdict(list) + for permission in apkinfo['requiredpermissions']['permission']: + permission = permission['@name'].split('.') + permission_lists[' '.join(permission[:-1])].append(permission[-1]) + attribute_type = 'text' + for comment, permissions in permission_lists.items(): + permission_object = MISPObject('android-permission') + permission_object.add_attribute('comment', **dict(type=attribute_type, value=comment, to_ids=False)) + for permission in permissions: + permission_object.add_attribute('permission', **dict(type=attribute_type, value=permission, to_ids=False)) + self.misp_event.add_object(**permission_object) + self.references[file_object.uuid].append(dict(referenced_uuid=permission_object.uuid, + relationship_type='grants')) + + def parse_elf(self, fileinfo, file_object): + elfinfo = fileinfo['elf'] + self.misp_event.add_object(**file_object) + attribute_type = 'text' + relationship = 'includes' + size = 'size-in-bytes' + for fileinfo in elfinfo['file']: + elf_object = MISPObject('elf') + self.references[file_object.uuid].append(dict(referenced_uuid=elf_object.uuid, + relationship_type=relationship)) + elf = fileinfo['main'][0]['header'][0] + if elf.get('type'): + # Haven't seen anything but EXEC yet in the files I tested + attribute_value = "EXECUTABLE" if elf['type'] == "EXEC (Executable file)" else elf['type'] + elf_object.add_attribute('type', **dict(type=attribute_type, value=attribute_value, to_ids=False)) + for feature, relation in elf_object_mapping.items(): + if elf.get(feature): + elf_object.add_attribute(relation, **dict(type=attribute_type, value=elf[feature], to_ids=False)) + sections_number = len(fileinfo['sections']['section']) + elf_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) + self.misp_event.add_object(**elf_object) + for section in fileinfo['sections']['section']: + section_object = MISPObject('elf-section') + for feature in ('name', 'type'): + if section.get(feature): + section_object.add_attribute(feature, **dict(type=attribute_type, value=section[feature], to_ids=False)) + if section.get('size'): + section_object.add_attribute(size, **dict(type=size, value=int(section['size'], 16), to_ids=False)) + for flag in section['flagsdesc']: + try: + attribute_value = elf_section_flags_mapping[flag] + section_object.add_attribute('flag', **dict(type=attribute_type, value=attribute_value, to_ids=False)) + except KeyError: + print(f'Unknown elf section flag: {flag}') + continue + self.misp_event.add_object(**section_object) + self.references[elf_object.uuid].append(dict(referenced_uuid=section_object.uuid, + relationship_type=relationship)) + + def parse_pe(self, fileinfo, file_object): + if not self.import_pe: + return + try: + peinfo = fileinfo['pe'] + except KeyError: + self.misp_event.add_object(**file_object) + return + pe_object = MISPObject('pe') + relationship = 'includes' + file_object.add_reference(pe_object.uuid, relationship) + self.misp_event.add_object(**file_object) + for field, mapping in pe_object_fields.items(): + attribute_type, object_relation = mapping + pe_object.add_attribute(object_relation, **{'type': attribute_type, 'value': peinfo[field], 'to_ids': False}) + pe_object.add_attribute('compilation-timestamp', **{'type': 'datetime', 'value': int(peinfo['timestamp'].split()[0], 16), 'to_ids': False}) + program_name = fileinfo['filename'] + if peinfo['versions']: + for feature in peinfo['versions']['version']: + name = feature['name'] + if name == 'InternalName': + program_name = feature['value'] + if name in pe_object_mapping: + pe_object.add_attribute(pe_object_mapping[name], **{'type': 'text', 'value': feature['value'], 'to_ids': False}) + sections_number = len(peinfo['sections']['section']) + pe_object.add_attribute('number-sections', **{'type': 'counter', 'value': sections_number, 'to_ids': False}) + signatureinfo = peinfo['signature'] + if signatureinfo['signed']: + signerinfo_object = MISPObject('authenticode-signerinfo') + pe_object.add_reference(signerinfo_object.uuid, 'signed-by') + self.misp_event.add_object(**pe_object) + signerinfo_object.add_attribute('program-name', **{'type': 'text', 'value': program_name, 'to_ids': False}) + for feature, mapping in signerinfo_object_mapping.items(): + attribute_type, object_relation = mapping + signerinfo_object.add_attribute(object_relation, **{'type': attribute_type, 'value': signatureinfo[feature], 'to_ids': False}) + self.misp_event.add_object(**signerinfo_object) + else: + self.misp_event.add_object(**pe_object) + for section in peinfo['sections']['section']: + section_object = self.parse_pe_section(section) + self.references[pe_object.uuid].append(dict(referenced_uuid=section_object.uuid, + relationship_type=relationship)) + self.misp_event.add_object(**section_object) + + def parse_pe_section(self, section): + section_object = MISPObject('pe-section') + for feature, mapping in pe_section_object_mapping.items(): + if section.get(feature): + attribute_type, object_relation = mapping + section_object.add_attribute(object_relation, **{'type': attribute_type, 'value': section[feature], 'to_ids': False}) + return section_object + + def parse_network_interactions(self): + domaininfo = self.data['domaininfo'] + if domaininfo: + for domain in domaininfo['domain']: + if domain['@ip'] != 'unknown': + domain_object = MISPObject('domain-ip') + for key, mapping in domain_object_mapping.items(): + attribute_type, object_relation = mapping + domain_object.add_attribute(object_relation, + **{'type': attribute_type, 'value': domain[key], 'to_ids': False}) + self.misp_event.add_object(**domain_object) + reference = dict(referenced_uuid=domain_object.uuid, relationship_type='contacts') + self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) + else: + attribute = MISPAttribute() + attribute.from_dict(**{'type': 'domain', 'value': domain['@name'], 'to_ids': False}) + self.misp_event.add_attribute(**attribute) + reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts') + self.add_process_reference(domain['@targetid'], domain['@currentpath'], reference) + ipinfo = self.data['ipinfo'] + if ipinfo: + for ip in ipinfo['ip']: + attribute = MISPAttribute() + attribute.from_dict(**{'type': 'ip-dst', 'value': ip['@ip'], 'to_ids': False}) + self.misp_event.add_attribute(**attribute) + reference = dict(referenced_uuid=attribute.uuid, relationship_type='contacts') + self.add_process_reference(ip['@targetid'], ip['@currentpath'], reference) + urlinfo = self.data['urlinfo'] + if urlinfo: + for url in urlinfo['url']: + target_id = int(url['@targetid']) + current_path = url['@currentpath'] + attribute = MISPAttribute() + attribute_dict = {'type': 'url', 'value': url['@name'], 'to_ids': False} + if target_id != -1 and current_path != 'unknown': + self.references[self.process_references[(target_id, current_path)]].append({ + 'referenced_uuid': attribute.uuid, + 'relationship_type': 'contacts' + }) + else: + attribute_dict['comment'] = 'From Memory - Enriched via the joe_import module' + attribute.from_dict(**attribute_dict) + self.misp_event.add_attribute(**attribute) + + def parse_registryactivities(self, process_uuid, registryactivities): + if registryactivities['keyCreated']: + for call in registryactivities['keyCreated']['call']: + self.attributes['regkey'][call['path']].add((process_uuid, 'creates')) + for feature, relationship in registry_references_mapping.items(): + if registryactivities[feature]: + for call in registryactivities[feature]['call']: + registry_key = MISPObject('registry-key') + for field, mapping in regkey_object_mapping.items(): + attribute_type, object_relation = mapping + registry_key.add_attribute(object_relation, **{'type': attribute_type, 'value': call[field], 'to_ids': False}) + registry_key.add_attribute('data-type', **{'type': 'text', 'value': 'REG_{}'.format(call['type'].upper()), 'to_ids': False}) + self.misp_event.add_object(**registry_key) + self.references[process_uuid].append(dict(referenced_uuid=registry_key.uuid, + relationship_type=relationship)) + + def add_process_reference(self, target, currentpath, reference): + try: + self.references[self.process_references[(int(target), currentpath)]].append(reference) + except KeyError: + self.references[self.analysisinfo_uuid].append(reference) + + def create_attribute(self, attribute_type, attribute_value): + attribute = MISPAttribute() + attribute.from_dict(**{'type': attribute_type, 'value': attribute_value, 'to_ids': False}) + self.misp_event.add_attribute(**attribute) + return attribute.uuid + + def finalize_results(self): + if self.references: + self.build_references() + event = json.loads(self.misp_event.to_json()) + self.results = {key: event[key] for key in ('Attribute', 'Object', 'Tag') if (key in event and event[key])} + + @staticmethod + def parse_timestamp(timestamp): + timestamp = timestamp.split(':') + timestamp[-1] = str(round(float(timestamp[-1].split(' ')[0]), 6)) + return ':'.join(timestamp) + + @staticmethod + def prefetch_attributes_data(connection): + attributes = {} + for field, value in zip(network_behavior_fields, connection): + attribute_type, object_relation = network_connection_object_mapping[field] + attributes[object_relation] = {'type': attribute_type, 'value': value, 'to_ids': False} + return attributes diff --git a/misp_modules/lib/lastline_api.py b/misp_modules/lib/lastline_api.py new file mode 100644 index 0000000..83726ad --- /dev/null +++ b/misp_modules/lib/lastline_api.py @@ -0,0 +1,841 @@ +""" +Lastline Community API Client and Utilities. + +:Copyright: + Copyright 2019 Lastline, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Copyright (c) 2010-2012 by Internet Systems Consortium, Inc. ("ISC") + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +""" +import abc +import logging +import io +import ipaddress +import pymisp +import re +import requests +from urllib import parse + + +DEFAULT_LL_PORTAL_API_URL = "https://user.lastline.com/papi" + +DEFAULT_LL_ANALYSIS_API_URL = "https://analysis.lastline.com" + +LL_HOSTED_DOMAINS = frozenset([ + "user.lastline.com", + "user.emea.lastline.com", +]) + + +def purge_none(d): + """Purge None entries from a dictionary.""" + return {k: v for k, v in d.items() if v is not None} + + +def get_task_link(uuid, analysis_url=None, portal_url=None): + """ + Get the task link given the task uuid and at least one API url. + + :param str uuid: the task uuid + :param str|None analysis_url: the URL to the analysis API endpoint + :param str|None portal_url: the URL to the portal API endpoint + :rtype: str + :return: the task link + :raises ValueError: if not enough parameters have been provided + """ + if not analysis_url and not portal_url: + raise ValueError("Neither analysis URL or portal URL have been specified") + if analysis_url: + portal_url = "{}/papi".format(analysis_url.replace("analysis.", "user.")) + portal_url_path = "../portal#/analyst/task/{}/overview".format(uuid) + return parse.urljoin(portal_url, portal_url_path) + + +def get_portal_url_from_task_link(task_link): + """ + Return the portal API url related to the provided task link. + + :param str task_link: a link + :rtype: str + :return: the portal API url + """ + parsed_uri = parse.urlparse(task_link) + return "{uri.scheme}://{uri.netloc}/papi".format(uri=parsed_uri) + + +def get_uuid_from_task_link(task_link): + """ + Return the uuid from a task link. + + :param str task_link: a link + :rtype: str + :return: the uuid + :raises ValueError: if the link contains not task uuid + """ + try: + return re.findall("[a-fA-F0-9]{32}", task_link)[0] + except IndexError: + raise ValueError("Link does not contain a valid task uuid") + + +def is_task_hosted(task_link): + """ + Return whether the portal link is pointing to a hosted submission. + + :param str task_link: a link + :rtype: boolean + :return: whether the link points to a hosted analysis + """ + for domain in LL_HOSTED_DOMAINS: + if domain in task_link: + return True + return False + + +class InvalidArgument(Exception): + """Error raised invalid.""" + + +class CommunicationError(Exception): + """Exception raised in case of timeouts or other network problem.""" + + +class Error(Exception): + """Generic server error.""" + + +class ApiError(Error): + """Server error with a message and an error code.""" + def __init__(self, error_msg, error_code=None): + super(ApiError, self).__init__(error_msg, error_code) + self.error_msg = error_msg + self.error_code = error_code + + def __str__(self): + if self.error_code is None: + error_code = "" + else: + error_code = " ({})".format(self.error_code) + return "{}{}".format(self.error_msg, error_code) + + +class LastlineAbstractClient(abc.ABC): + """"A very basic HTTP client providing basic functionality.""" + + __metaclass__ = abc.ABCMeta + + SUB_APIS = ('analysis', 'authentication', 'knowledgebase', 'login') + FORMATS = ["json", "xml"] + + @classmethod + def sanitize_login_params(cls, api_key, api_token, username, password): + """ + Return a dictionary with either API or USER credentials. + + :param str|None api_key: the API key + :param str|None api_token: the API token + :param str|None username: the username + :param str|None password: the password + :rtype: dict[str, str] + :return: the dictionary + :raises InvalidArgument: if too many values are invalid + """ + if api_key and api_token: + return { + "key": api_key, + "api_token": api_token, + } + elif username and password: + return { + "username": username, + "password": password, + } + else: + raise InvalidArgument("Arguments provided do not contain valid data") + + @classmethod + def get_login_params_from_dict(cls, d): + """ + Get the module configuration from a ConfigParser object. + + :param dict[str, str] d: the dictionary + :rtype: dict[str, str] + :return: the parsed configuration + """ + api_key = d.get("key") + api_token = d.get("api_token") + username = d.get("username") + password = d.get("password") + return cls.sanitize_login_params(api_key, api_token, username, password) + + @classmethod + def get_login_params_from_conf(cls, conf, section_name): + """ + Get the module configuration from a ConfigParser object. + + :param ConfigParser conf: the conf object + :param str section_name: the section name + :rtype: dict[str, str] + :return: the parsed configuration + """ + api_key = conf.get(section_name, "key", fallback=None) + api_token = conf.get(section_name, "api_token", fallback=None) + username = conf.get(section_name, "username", fallback=None) + password = conf.get(section_name, "password", fallback=None) + return cls.sanitize_login_params(api_key, api_token, username, password) + + @classmethod + def load_from_conf(cls, conf, section_name): + """ + Load client from a ConfigParser object. + + :param ConfigParser conf: the conf object + :param str section_name: the section name + :rtype: T <- LastlineAbstractClient + :return: the loaded client + """ + url = conf.get(section_name, "url") + return cls(url, cls.get_login_params_from_conf(conf, section_name)) + + def __init__(self, api_url, login_params, timeout=60, verify_ssl=True): + """ + Instantiate a Lastline mini client. + + :param str api_url: the URL of the API + :param dict[str, str]: the login parameters + :param int timeout: the timeout + :param boolean verify_ssl: whether to verify the SSL certificate + """ + self._url = api_url + self._login_params = login_params + self._timeout = timeout + self._verify_ssl = verify_ssl + self._session = None + self._logger = logging.getLogger(__name__) + + @abc.abstractmethod + def _login(self): + """Login using account-based or key-based methods.""" + + def _is_logged_in(self): + """Return whether we have an active session.""" + return self._session is not None + + @staticmethod + def _parse_response(response): + """ + Parse the response. + + :param requests.Response response: the response + :rtype: tuple(str|None, Error|ApiError) + :return: a tuple with mutually exclusive fields (either the response or the error) + """ + try: + ret = response.json() + if "success" not in ret: + return None, Error("no success field in response") + + if not ret["success"]: + error_msg = ret.get("error", "") + error_code = ret.get("error_code", None) + return None, ApiError(error_msg, error_code) + + if "data" not in ret: + return None, Error("no data field in response") + + return ret["data"], None + except ValueError as e: + return None, Error("Response not json {}".format(e)) + + def _handle_response(self, response, raw=False): + """ + Check a response for issues and parse the return. + + :param requests.Response response: the response + :param boolean raw: whether the raw body should be returned + :rtype: str + :return: if raw, return the response content; if not raw, the data field + :raises: CommunicationError, ApiError, Error + """ + # Check for HTTP errors, and re-raise in case + try: + response.raise_for_status() + except requests.RequestException as e: + _, err = self._parse_response(response) + if isinstance(err, ApiError): + err_msg = "{}: {}".format(e, err.error_msg) + else: + err_msg = "{}".format(e) + raise CommunicationError(err_msg) + + # Otherwise return the data (either parsed or not) but reraise if we have an API error + if raw: + return response.content + data, err = self._parse_response(response) + if err: + raise err + return data + + def _build_url(self, sub_api, parts, requested_format="json"): + if sub_api not in self.SUB_APIS: + raise InvalidArgument(sub_api) + if requested_format not in self.FORMATS: + raise InvalidArgument(requested_format) + num_parts = 2 + len(parts) + pattern = "/".join(["%s"] * num_parts) + ".%s" + params = [self._url, sub_api] + parts + [requested_format] + return pattern % tuple(params) + + def post(self, module, function, params=None, data=None, files=None, fmt="json"): + if isinstance(function, list): + functions = function + else: + functions = [function] if function else [] + url = self._build_url(module, functions, requested_format=fmt) + return self.do_request( + url=url, + method="POST", + params=params, + data=data, + files=files, + fmt=fmt, + ) + + def get(self, module, function, params=None, fmt="json"): + if isinstance(function, list): + functions = function + else: + functions = [function] if function else [] + url = self._build_url(module, functions, requested_format=fmt) + return self.do_request( + url=url, + method="GET", + params=params, + fmt=fmt, + ) + + def do_request( + self, + method, + url, + params=None, + data=None, + files=None, + fmt="json", + raw=False, + raw_response=False, + headers=None, + stream_response=False + ): + if raw_response: + raw = True + + if fmt: + fmt = fmt.lower().strip() + if fmt not in self.FORMATS: + raise InvalidArgument("Only json, xml, html and pdf supported") + elif not raw: + raise InvalidArgument("Unformatted response requires raw=True") + + if fmt != "json" and not raw: + raise InvalidArgument("Non-json format requires raw=True") + + if method not in ["POST", "GET"]: + raise InvalidArgument("Only POST and GET supported") + + if not self._is_logged_in(): + self._login() + + try: + try: + response = self._session.request( + method=method, + url=url, + data=data, + params=params, + files=files, + verify=self._verify_ssl, + timeout=self._timeout, + stream=stream_response, + headers=headers, + ) + except requests.RequestException as e: + raise CommunicationError(e) + + if raw_response: + return response + return self._handle_response(response, raw) + + except Error as e: + raise e + + except CommunicationError as e: + raise e + + +class AnalysisClient(LastlineAbstractClient): + + def _login(self): + """ + Creates auth session for malscape-service. + + Credentials are 'key' and 'api_token'. + """ + if self._session is None: + self._session = requests.session() + url = self._build_url("authentication", ["login"]) + self.do_request("POST", url, params=purge_none(self._login_params)) + + def get_progress(self, uuid): + """ + Get the completion progress of a given task. + :param str uuid: the unique identifier of the submitted task + :rtype: dict[str, int] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + url = self._build_url('analysis', ['get_progress']) + params = {'uuid': uuid} + return self.do_request("POST", url, params=params) + + def get_result(self, uuid): + """ + Get report results for a given task. + + :param str uuid: the unique identifier of the submitted task + :rtype: dict[str, any] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + # better: use 'get_results()' but that would break + # backwards-compatibility + url = self._build_url('analysis', ['get']) + params = {'uuid': uuid} + return self.do_request("GET", url, params=params) + + def submit_file( + self, + file_data, + file_name=None, + password=None, + analysis_env=None, + allow_network_traffic=True, + analysis_timeout=None, + bypass_cache=False, + ): + """ + Upload a file to be analyzed. + + :param bytes file_data: the data as a byte sequence + :param str|None file_name: if set, represents the name of the file to submit + :param str|None password: if set, use it to extract the sample + :param str|None analysis_env: if set, e.g windowsxp + :param boolean allow_network_traffic: if set to False, deny network connections + :param int|None analysis_timeout: if set limit the duration of the analysis + :param boolean bypass_cache: whether to re-process a file (requires special permissions) + :rtype: dict[str, any] + :return: a dictionary in the following form if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + file_stream = io.BytesIO(file_data) + api_url = self._build_url("analysis", ["submit", "file"]) + params = purge_none({ + "bypass_cache": bypass_cache and 1 or None, + "analysis_timeout": analysis_timeout, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic and 1 or None, + "filename": file_name, + "password": password, + "full_report_score": -1, + }) + + files = purge_none({ + # If an explicit filename was provided, we can pass it down to + # python-requests to use it in the multipart/form-data. This avoids + # having python-requests trying to guess the filename based on stream + # attributes. + # + # The problem with this is that, if the filename is not ASCII, then + # this triggers a bug in flask/werkzeug which means the file is + # thrown away. Thus, we just force an ASCII name + "file": ('dummy-ascii-name-for-file-param', file_stream), + }) + + return self.do_request("POST", api_url, params=params, files=files) + + def submit_url( + self, + url, + referer=None, + user_agent=None, + bypass_cache=False, + ): + """ + Upload an URL to be analyzed. + + :param str url: the url to analyze + :param str|None referer: the referer + :param str|None user_agent: the user agent + :param boolean bypass_cache: bypass_cache + :rtype: dict[str, any] + :return: a dictionary like the following if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + api_url = self._build_url("analysis", ["submit", "url"]) + params = purge_none({ + "url": url, + "referer": referer, + "bypass_cache": bypass_cache and 1 or None, + "user_agent": user_agent or None, + }) + return self.do_request("POST", api_url, params=params) + + +class PortalClient(LastlineAbstractClient): + + def _login(self): + """ + Login using account-based or key-based methods. + + Credentials are 'username' and 'password' + """ + if self._session is None: + self._session = requests.session() + self.post("login", function=None, data=self._login_params) + + def get_progress(self, uuid, analysis_instance=None): + """ + Get the completion progress of a given task. + + :param str uuid: the unique identifier of the submitted task + :param str analysis_instance: if set, defines the analysis instance to query + :rtype: dict[str, int] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + params = purge_none({"uuid": uuid, "analysis_instance": analysis_instance}) + return self.get("analysis", "get_progress", params=params) + + def get_result(self, uuid, analysis_instance=None): + """ + Get report results for a given task. + + :param str uuid: the unique identifier of the submitted task + :param str analysis_instance: if set, defines the analysis instance to query + :rtype: dict[str, any] + :return: a dictionary like the the following: + { + "completed": 1, + "progress": 100 + } + """ + params = purge_none( + { + "uuid": uuid, + "analysis_instance": analysis_instance, + "report_format": "json", + } + ) + return self.get("analysis", "get_result", params=params) + + def submit_url( + self, + url, + referer=None, + user_agent=None, + bypass_cache=False, + ): + """ + Upload an URL to be analyzed. + + :param str url: the url to analyze + :param str|None referer: the referer + :param str|None user_agent: the user agent + :param boolean bypass_cache: bypass_cache + :rtype: dict[str, any] + :return: a dictionary like the following if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + params = purge_none( + { + "url": url, + "bypass_cache": bypass_cache, + "referer": referer, + "user_agent": user_agent + } + ) + return self.post("analysis", "submit_url", params=params) + + def submit_file( + self, + file_data, + file_name=None, + password=None, + analysis_env=None, + allow_network_traffic=True, + analysis_timeout=None, + bypass_cache=False, + ): + """ + Upload a file to be analyzed. + + :param bytes file_data: the data as a byte sequence + :param str|None file_name: if set, represents the name of the file to submit + :param str|None password: if set, use it to extract the sample + :param str|None analysis_env: if set, e.g windowsxp + :param boolean allow_network_traffic: if set to False, deny network connections + :param int|None analysis_timeout: if set limit the duration of the analysis + :param boolean bypass_cache: whether to re-process a file (requires special permissions) + :rtype: dict[str, any] + :return: a dictionary in the following form if the analysis is already available: + { + "submission": "2019-11-17 09:33:23", + "child_tasks": [...], + "reports": [...], + "submission_timestamp": "2019-11-18 16:11:04", + "task_uuid": "86097fb8e4cd00100464cb001b97ecbe", + "score": 0, + "analysis_subject": { + "url": "https://www.google.com" + }, + "last_submission_timestamp": "2019-11-18 16:11:04" + } + + OR the following if the analysis is still pending: + + { + "submission_timestamp": "2019-11-18 13:59:25", + "task_uuid": "f3c0ae115d51001017ff8da768fa6049", + } + """ + params = purge_none( + { + "filename": file_name, + "password": password, + "analysis_env": analysis_env, + "allow_network_traffic": allow_network_traffic, + "analysis_timeout": analysis_timeout, + "bypass_cache": bypass_cache, + } + ) + files = {"file": (file_name, file_data, "application/octet-stream")} + return self.post("analysis", "submit_file", params=params, files=files) + + +class LastlineResultBaseParser(object): + """ + This is a parser to extract *basic* information from a Lastline result dictionary. + + Note: this is a *version 0*: the information we extract is merely related to the behaviors and + the HTTP connections. Further iterations will include host activity such as files, mutexes, + registry keys, strings, etc. + """ + + def __init__(self): + """Constructor.""" + self.misp_event = None + + @staticmethod + def _get_mitre_techniques(result): + return [ + "misp-galaxy:mitre-attack-pattern=\"{} - {}\"".format(w[0], w[1]) + for w in sorted(set([ + (y["id"], y["name"]) + for x in result.get("malicious_activity", []) + for y in result.get("activity_to_mitre_techniques", {}).get(x, []) + ])) + ] + + def parse(self, analysis_link, result): + """ + Parse the analysis result into a MISP event. + + :param str analysis_link: the analysis link + :param dict[str, any] result: the JSON returned by the analysis client. + :rtype: MISPEvent + :return: some results that can be consumed by MIPS. + """ + self.misp_event = pymisp.MISPEvent() + + # Add analysis subject info + if "url" in result["analysis_subject"]: + o = pymisp.MISPObject("url") + o.add_attribute("url", result["analysis_subject"]["url"]) + else: + o = pymisp.MISPObject("file") + o.add_attribute("md5", type="md5", value=result["analysis_subject"]["md5"]) + o.add_attribute("sha1", type="sha1", value=result["analysis_subject"]["sha1"]) + o.add_attribute("sha256", type="sha256", value=result["analysis_subject"]["sha256"]) + o.add_attribute( + "mimetype", + type="mime-type", + value=result["analysis_subject"]["mime_type"] + ) + self.misp_event.add_object(o) + + # Add HTTP requests from url analyses + network_dict = result.get("report", {}).get("analysis", {}).get("network", {}) + for request in network_dict.get("requests", []): + parsed_uri = parse.urlparse(request["url"]) + o = pymisp.MISPObject(name='http-request') + o.add_attribute('host', parsed_uri.netloc) + o.add_attribute('method', "GET") + o.add_attribute('uri', request["url"]) + o.add_attribute("ip", request["ip"]) + self.misp_event.add_object(o) + + # Add network behaviors from files + for subject in result.get("report", {}).get("analysis_subjects", []): + + # Add DNS requests + for dns_query in subject.get("dns_queries", []): + hostname = dns_query.get("hostname") + # Skip if it is an IP address + try: + if hostname == "wpad": + continue + _ = ipaddress.ip_address(hostname) + continue + except ValueError: + pass + + o = pymisp.MISPObject(name='dns-record') + o.add_attribute('queried-domain', hostname) + self.misp_event.add_object(o) + + # Add HTTP conversations (as network connection and as http request) + for http_conversation in subject.get("http_conversations", []): + o = pymisp.MISPObject(name="network-connection") + o.add_attribute("ip-src", http_conversation["src_ip"]) + o.add_attribute("ip-dst", http_conversation["dst_ip"]) + o.add_attribute("src-port", http_conversation["src_port"]) + o.add_attribute("dst-port", http_conversation["dst_port"]) + o.add_attribute("hostname-dst", http_conversation["dst_host"]) + o.add_attribute("layer3-protocol", "IP") + o.add_attribute("layer4-protocol", "TCP") + o.add_attribute("layer7-protocol", "HTTP") + self.misp_event.add_object(o) + + method, path, http_version = http_conversation["url"].split(" ") + if http_conversation["dst_port"] == 80: + uri = "http://{}{}".format(http_conversation["dst_host"], path) + else: + uri = "http://{}:{}{}".format( + http_conversation["dst_host"], + http_conversation["dst_port"], + path + ) + o = pymisp.MISPObject(name='http-request') + o.add_attribute('host', http_conversation["dst_host"]) + o.add_attribute('method', method) + o.add_attribute('uri', uri) + o.add_attribute('ip', http_conversation["dst_ip"]) + self.misp_event.add_object(o) + + # Add sandbox info like score and sandbox type + o = pymisp.MISPObject(name="sandbox-report") + sandbox_type = "saas" if is_task_hosted(analysis_link) else "on-premise" + o.add_attribute("score", result["score"]) + o.add_attribute("sandbox-type", sandbox_type) + o.add_attribute("{}-sandbox".format(sandbox_type), "lastline") + o.add_attribute("permalink", analysis_link) + self.misp_event.add_object(o) + + # Add behaviors + o = pymisp.MISPObject(name="sb-signature") + o.add_attribute("software", "Lastline") + for activity in result.get("malicious_activity", []): + a = pymisp.MISPAttribute() + a.from_dict(type="text", value=activity) + o.add_attribute("signature", **a) + self.misp_event.add_object(o) + + # Add mitre techniques + for technique in self._get_mitre_techniques(result): + self.misp_event.add_tag(technique) diff --git a/misp_modules/lib/vt_graph_parser/__init__.py b/misp_modules/lib/vt_graph_parser/__init__.py new file mode 100644 index 0000000..abc02c5 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/__init__.py @@ -0,0 +1,8 @@ +"""vt_graph_parser. + +This module provides methods to import graph from misp. +""" + + +from .helpers import * # noqa +from .importers import * # noqa diff --git a/misp_modules/lib/vt_graph_parser/errors.py b/misp_modules/lib/vt_graph_parser/errors.py new file mode 100644 index 0000000..a7e18e9 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/errors.py @@ -0,0 +1,20 @@ +"""vt_graph_parser.errors. + +This module provides custom errors for data importers. +""" + + +class GraphImportError(Exception): + pass + + +class InvalidFileFormatError(Exception): + pass + + +class MispEventNotFoundError(Exception): + pass + + +class ServerError(Exception): + pass diff --git a/misp_modules/lib/vt_graph_parser/helpers/__init__.py b/misp_modules/lib/vt_graph_parser/helpers/__init__.py new file mode 100644 index 0000000..8f9f660 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/__init__.py @@ -0,0 +1,7 @@ +"""vt_graph_parser.helpers. + +This modules provides functions and attributes to help MISP importers. +""" + + +__all__ = ["parsers", "rules", "wrappers"] diff --git a/misp_modules/lib/vt_graph_parser/helpers/parsers.py b/misp_modules/lib/vt_graph_parser/helpers/parsers.py new file mode 100644 index 0000000..8ca5745 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/parsers.py @@ -0,0 +1,88 @@ +"""vt_graph_parser.helpers.parsers. + +This module provides parsers for MISP inputs. +""" + + +from vt_graph_parser.helpers.wrappers import MispAttribute + + +MISP_INPUT_ATTR = [ + "hostname", + "domain", + "ip-src", + "ip-dst", + "md5", + "sha1", + "sha256", + "url", + "filename|md5", + "filename", + "target-user", + "target-email" +] + +VIRUSTOTAL_GRAPH_LINK_PREFIX = "https://www.virustotal.com/graph/" + + +def _parse_data(attributes, objects): + """Parse MISP event attributes and objects data. + + Args: + attributes (dict): dictionary which contains the MISP event attributes data. + objects (dict): dictionary which contains the MISP event objects data. + + Returns: + ([MispAttribute], str): MISP attributes and VTGraph link if exists. + Link defaults to "". + """ + attributes_data = [] + vt_graph_link = "" + + # Get simple MISP event attributes. + attributes_data += ( + [attr for attr in attributes + if attr.get("type") in MISP_INPUT_ATTR]) + + # Get attributes from MISP objects too. + if objects: + for object_ in objects: + object_attrs = object_.get("Attribute", []) + attributes_data += ( + [attr for attr in object_attrs + if attr.get("type") in MISP_INPUT_ATTR]) + + # Check if there is any VirusTotal Graph computed in MISP event. + vt_graph_links = ( + attr for attr in attributes if attr.get("type") == "link" + and attr.get("value", "").startswith(VIRUSTOTAL_GRAPH_LINK_PREFIX)) + + # MISP could have more than one VirusTotal Graph, so we will take + # the last one. + current_id = 0 # MISP attribute id is the number of the attribute. + vt_graph_link = "" + for link in vt_graph_links: + if int(link.get("id")) > current_id: + current_id = int(link.get("id")) + vt_graph_link = link.get("value") + + attributes = [ + MispAttribute(data["type"], data["category"], data["value"]) + for data in attributes_data] + return (attributes, + vt_graph_link.replace(VIRUSTOTAL_GRAPH_LINK_PREFIX, "")) + + +def parse_pymisp_response(payload): + """Get event attributes and VirusTotal Graph id from pymisp response. + + Args: + payload (dict): dictionary which contains pymisp response. + + Returns: + ([MispAttribute], str): MISP attributes and VTGraph link if exists. + Link defaults to "". + """ + event_attrs = payload.get("Attribute", []) + objects = payload.get("Object") + return _parse_data(event_attrs, objects) diff --git a/misp_modules/lib/vt_graph_parser/helpers/rules.py b/misp_modules/lib/vt_graph_parser/helpers/rules.py new file mode 100644 index 0000000..e3ed7f8 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/rules.py @@ -0,0 +1,304 @@ +"""vt_graph_parser.helpers.rules. + +This module provides rules that helps MISP importers to connect MISP attributes +between them using VirusTotal relationship. Check all available relationship +here: + +- File: https://developers.virustotal.com/v3/reference/#files-relationships +- URL: https://developers.virustotal.com/v3/reference/#urls-relationships +- Domain: https://developers.virustotal.com/v3/reference/#domains-relationships +- IP: https://developers.virustotal.com/v3/reference/#ip-relationships +""" + + +import abc + + +class MispEventRule(object): + """Rules for MISP event nodes connection object wrapper.""" + + def __init__(self, last_rule=None, node=None): + """Create a MispEventRule instance. + + MispEventRule is a collection of rules that can infer the relationships + between nodes from MISP events. + + Args: + last_rule (MispEventRule): previous rule. + node (Node): actual node. + """ + self.last_rule = last_rule + self.node = node + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def get_last_different_rule(self): + """Search the last rule whose event was different from actual. + + Returns: + MispEventRule: the last different rule. + """ + if not isinstance(self, self.last_rule.__class__): + return self.last_rule + else: + return self.last_rule.get_last_different_rule() + + def resolve_relation(self, graph, node, misp_category): + """Try to infer a relationship between two nodes. + + This method is based on a non-deterministic finite automaton for + this reason the future rule only depends on the actual rule and the input + node. + + For example if the actual rule is a MISPEventDomainRule and the given node + is an ip_address node, the connection type between them will be + `resolutions` and the this rule will transit to MISPEventIPRule. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + if node.node_type in self.relation_event: + return self.relation_event[node.node_type](graph, node, misp_category) + else: + return self.manual_link(graph, node) + + def manual_link(self, graph, node): + """Creates a manual link between self.node and the given node. + + We accept MISP types that VirusTotal does not know how to link, so we create + a end to end relationship instead of create an unknown relationship node. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + + Returns: + MispEventRule: the transited rule. + """ + graph.add_link(self.node.node_id, node.node_id, "manual") + return self + + @abc.abstractmethod + def __file_transition(self, graph, node, misp_category): + """Make a new transition due to file attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __ip_transition(self, graph, node, misp_category): + """Make a new transition due to ip attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __url_transition(self, graph, node, misp_category): + """Make a new transition due to url attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + @abc.abstractmethod + def __domain_transition(self, graph, node, misp_category): + """Make a new transition due to domain attribute event. + + Args: + graph (VTGraph): graph to be computed. + node (Node): the node to be linked. + misp_category: (str): MISP category of the given node. + + Returns: + MispEventRule: the transited rule. + """ + pass + + +class MispEventURLRule(MispEventRule): + """Rule for URL event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventURLRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "downloaded_files") + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_ips") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_domains") + return MispEventDomainRule(self, node) + + +class MispEventIPRule(MispEventRule): + """Rule for IP event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventIPRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + connection_type = "communicating_files" + if misp_category == "Artifacts dropped": + connection_type = "downloaded_files" + graph.add_link(self.node.node_id, node.node_id, connection_type) + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "resolutions") + return MispEventDomainRule(self, node) + + +class MispEventDomainRule(MispEventRule): + """Rule for domain event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventDomainRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + connection_type = "communicating_files" + if misp_category == "Artifacts dropped": + connection_type = "downloaded_files" + graph.add_link(self.node.node_id, node.node_id, connection_type) + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "resolutions") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + graph.add_link(self.node.node_id, node.node_id, "siblings") + return MispEventDomainRule(self, node) + + +class MispEventFileRule(MispEventRule): + """Rule for File event.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventFileRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + suitable_rule = self.get_last_different_rule() + if not isinstance(suitable_rule, MispEventInitialRule): + return suitable_rule.resolve_relation(graph, node, misp_category) + else: + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_ips") + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_urls") + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + graph.add_link(self.node.node_id, node.node_id, "contacted_domains") + return MispEventDomainRule(self, node) + + +class MispEventInitialRule(MispEventRule): + """Initial rule.""" + + def __init__(self, last_rule=None, node=None): + super(MispEventInitialRule, self).__init__(last_rule, node) + self.relation_event = { + "ip_address": self.__ip_transition, + "url": self.__url_transition, + "domain": self.__domain_transition, + "file": self.__file_transition + } + + def __file_transition(self, graph, node, misp_category): + return MispEventFileRule(self, node) + + def __ip_transition(self, graph, node, misp_category): + return MispEventIPRule(self, node) + + def __url_transition(self, graph, node, misp_category): + return MispEventURLRule(self, node) + + def __domain_transition(self, graph, node, misp_category): + return MispEventDomainRule(self, node) diff --git a/misp_modules/lib/vt_graph_parser/helpers/wrappers.py b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py new file mode 100644 index 0000000..d376d43 --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/helpers/wrappers.py @@ -0,0 +1,58 @@ +"""vt_graph_parser.helpers.wrappers. + +This module provides a Python object wrapper for MISP objects. +""" + + +class MispAttribute(object): + """Python object wrapper for MISP attribute. + + Attributes: + type (str): VirusTotal node type. + category (str): MISP attribute category. + value (str): node id. + label (str): node name. + misp_type (str): MISP node type. + """ + + MISP_TYPES_REFERENCE = { + "hostname": "domain", + "domain": "domain", + "ip-src": "ip_address", + "ip-dst": "ip_address", + "url": "url", + "filename|X": "file", + "filename": "file", + "md5": "file", + "sha1": "file", + "sha256": "file", + "target-user": "victim", + "target-email": "email" + } + + def __init__(self, misp_type, category, value, label=""): + """Constructor for a MispAttribute. + + Args: + misp_type (str): MISP type attribute. + category (str): MISP category attribute. + value (str): attribute value. + label (str): attribute label. + """ + if misp_type.startswith("filename|"): + label, value = value.split("|") + misp_type = "filename|X" + if misp_type == "filename": + label = value + + self.type = self.MISP_TYPES_REFERENCE.get(misp_type) + self.category = category + self.value = value + self.label = label + self.misp_type = misp_type + + def __eq__(self, other): + return (isinstance(other, self.__class__) and self.value == other.value and self.type == other.type) + + def __repr__(self): + return 'MispAttribute("{type}", "{category}", "{value}")'.format(type=self.type, category=self.category, value=self.value) diff --git a/misp_modules/lib/vt_graph_parser/importers/__init__.py b/misp_modules/lib/vt_graph_parser/importers/__init__.py new file mode 100644 index 0000000..c59197c --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/__init__.py @@ -0,0 +1,7 @@ +"""vt_graph_parser.importers. + +This module provides methods to import graphs from MISP. +""" + + +__all__ = ["base", "pymisp_response"] diff --git a/misp_modules/lib/vt_graph_parser/importers/base.py b/misp_modules/lib/vt_graph_parser/importers/base.py new file mode 100644 index 0000000..ed5c0fc --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/base.py @@ -0,0 +1,98 @@ +"""vt_graph_parser.importers.base. + +This module provides a common method to import graph from misp attributes. +""" + + +import vt_graph_api +from vt_graph_parser.helpers.rules import MispEventInitialRule + + +def import_misp_graph( + misp_attributes, graph_id, vt_api_key, fetch_information, name, + private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, + group_viewers, use_vt_to_connect_the_graph, max_api_quotas, + max_search_depth): + """Import VirusTotal Graph from MISP. + + Args: + misp_attributes ([MispAttribute]): list with the MISP attributes which + will be added to the returned graph. + graph_id: if supplied, the graph will be loaded instead of compute it again. + vt_api_key (str): VT API Key. + fetch_information (bool): whether the script will fetch + information for added nodes in VT. Defaults to True. + name (str): graph title. Defaults to "". + private (bool): True for private graphs. You need to have + Private Graph premium features enabled in your subscription. Defaults + to False. + fetch_vt_enterprise (bool, optional): if True, the graph will search any + available information using VirusTotal Intelligence for the node if there + is no normal information for it. Defaults to False. + user_editors ([str]): usernames that can edit the graph. + Defaults to None. + user_viewers ([str]): usernames that can view the graph. + Defaults to None. + group_editors ([str]): groups that can edit the graph. + Defaults to None. + group_viewers ([str]): groups that can view the graph. + Defaults to None. + use_vt_to_connect_the_graph (bool): if True, graph nodes will + be linked using VirusTotal API. Otherwise, the links will be generated + using production rules based on MISP attributes order. Defaults to + False. + max_api_quotas (int): maximum number of api quotas that could + be consumed to resolve graph using VirusTotal API. Defaults to 20000. + max_search_depth (int, optional): max search depth to explore + relationship between nodes when use_vt_to_connect_the_graph is True. + Defaults to 3. + + If use_vt_to_connect_the_graph is True, it will take some time to compute + graph. + + Returns: + vt_graph_api.graph.VTGraph: the imported graph. + """ + + rule = MispEventInitialRule() + + # Check if the event has been already computed in VirusTotal Graph. Otherwise + # a new graph will be created. + if not graph_id: + graph = vt_graph_api.VTGraph( + api_key=vt_api_key, name=name, private=private, + user_editors=user_editors, user_viewers=user_viewers, + group_editors=group_editors, group_viewers=group_viewers) + else: + graph = vt_graph_api.VTGraph.load_graph(graph_id, vt_api_key) + + attributes_to_add = [attr for attr in misp_attributes + if not graph.has_node(attr.value)] + + total_expandable_attrs = max(sum( + 1 for attr in attributes_to_add + if attr.type in vt_graph_api.Node.SUPPORTED_NODE_TYPES), + 1) + + max_quotas_per_search = max( + int(max_api_quotas / total_expandable_attrs), 1) + + previous_node_id = "" + for attr in attributes_to_add: + # Add the current attr as node to the graph. + added_node = graph.add_node( + attr.value, attr.type, fetch_information, fetch_vt_enterprise, + attr.label) + # If use_vt_to_connect_the_grap is True the nodes will be connected using + # VT API. + if use_vt_to_connect_the_graph: + if (attr.type not in vt_graph_api.Node.SUPPORTED_NODE_TYPES and previous_node_id): + graph.add_link(previous_node_id, attr.value, "manual") + else: + graph.connect_with_graph( + attr.value, max_quotas_per_search, max_search_depth, + fetch_info_collected_nodes=fetch_information) + else: + rule = rule.resolve_relation(graph, added_node, attr.category) + + return graph diff --git a/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py new file mode 100644 index 0000000..e0e834b --- /dev/null +++ b/misp_modules/lib/vt_graph_parser/importers/pymisp_response.py @@ -0,0 +1,73 @@ +"""vt_graph_parser.importers.pymisp_response. + +This modules provides a graph importer method for MISP event by using the +response payload giving by MISP API directly. +""" + + +from vt_graph_parser.helpers.parsers import parse_pymisp_response +from vt_graph_parser.importers.base import import_misp_graph + + +def from_pymisp_response( + payload, vt_api_key, fetch_information=True, + private=False, fetch_vt_enterprise=False, user_editors=None, + user_viewers=None, group_editors=None, group_viewers=None, + use_vt_to_connect_the_graph=False, max_api_quotas=1000, + max_search_depth=3, expand_node_one_level=False): + """Import VirusTotal Graph from MISP JSON file. + + Args: + payload (dict): dictionary which contains the request payload. + vt_api_key (str): VT API Key. + fetch_information (bool, optional): whether the script will fetch + information for added nodes in VT. Defaults to True. + name (str, optional): graph title. Defaults to "". + private (bool, optional): True for private graphs. You need to have + Private Graph premium features enabled in your subscription. Defaults + to False. + fetch_vt_enterprise (bool, optional): if True, the graph will search any + available information using VirusTotal Intelligence for the node if there + is no normal information for it. Defaults to False. + user_editors ([str], optional): usernames that can edit the graph. + Defaults to None. + user_viewers ([str], optional): usernames that can view the graph. + Defaults to None. + group_editors ([str], optional): groups that can edit the graph. + Defaults to None. + group_viewers ([str], optional): groups that can view the graph. + Defaults to None. + use_vt_to_connect_the_graph (bool, optional): if True, graph nodes will + be linked using VirusTotal API. Otherwise, the links will be generated + using production rules based on MISP attributes order. Defaults to + False. + max_api_quotas (int, optional): maximum number of api quotas that could + be consumed to resolve graph using VirusTotal API. Defaults to 20000. + max_search_depth (int, optional): max search depth to explore + relationship between nodes when use_vt_to_connect_the_graph is True. + Defaults to 3. + expand_one_level (bool, optional): expand entire graph one level. + Defaults to False. + + If use_vt_to_connect_the_graph is True, it will take some time to compute + graph. + + Raises: + LoaderError: if JSON file is invalid. + + Returns: + [vt_graph_api.graph.VTGraph: the imported graph]. + """ + graphs = [] + for event_payload in payload['data']: + misp_attrs, graph_id = parse_pymisp_response(event_payload) + name = "Graph created from MISP event" + graph = import_misp_graph( + misp_attrs, graph_id, vt_api_key, fetch_information, name, + private, fetch_vt_enterprise, user_editors, user_viewers, group_editors, + group_viewers, use_vt_to_connect_the_graph, max_api_quotas, + max_search_depth) + if expand_node_one_level: + graph.expand_n_level(1) + graphs.append(graph) + return graphs diff --git a/misp_modules/modules/__init__.py b/misp_modules/modules/__init__.py index 65ce6b2..47ddcbf 100644 --- a/misp_modules/modules/__init__.py +++ b/misp_modules/modules/__init__.py @@ -1,3 +1,3 @@ -from .expansion import * -from .import_mod import * -from .export_mod import * +from .expansion import * # noqa +from .import_mod import * # noqa +from .export_mod import * # noqa diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index b49c1dc..dbd3473 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,3 +1,21 @@ -from . import _vmray +from . import _vmray # noqa +import os +import sys -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator'] +sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) + +__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', + 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'eql', + 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', + 'whois', 'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep', + 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', + 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', + 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', + 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', + 'qrcode', 'ocr_enrich', 'pdf_enrich', 'docx_enrich', 'xlsx_enrich', 'pptx_enrich', + 'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus', + 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', + 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar', + 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich', + 'trustar_enrich'] diff --git a/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py index 0ab58e8..af3f204 100755 --- a/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py +++ b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py @@ -47,9 +47,11 @@ options = None locale.setlocale(locale.LC_ALL, '') + class QueryError(Exception): pass + class DnsdbClient(object): def __init__(self, server, apikey, limit=None, http_proxy=None, https_proxy=None): self.server = server @@ -81,7 +83,6 @@ class DnsdbClient(object): return self._query(path, before, after) def _query(self, path, before=None, after=None): - res = [] url = '%s/lookup/%s' % (self.server, path) params = {} @@ -120,12 +121,15 @@ class DnsdbClient(object): except (HTTPError, URLError) as e: raise QueryError(str(e), sys.exc_traceback) + def quote(path): return urllib_quote(path, safe='') + def sec_to_text(ts): return time.strftime('%Y-%m-%d %H:%M:%S -0000', time.gmtime(ts)) + def rrset_to_text(m): s = StringIO() @@ -155,9 +159,11 @@ def rrset_to_text(m): finally: s.close() + def rdata_to_text(m): return '%s IN %s %s' % (m['rrname'], m['rrtype'], m['rdata']) + def parse_config(cfg_files): config = {} @@ -172,6 +178,7 @@ def parse_config(cfg_files): return config + def time_parse(s): try: epoch = int(s) @@ -193,14 +200,15 @@ def time_parse(s): m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I) if m: - return -1*(int(m.group(1) or 0)*604800 + - int(m.group(2) or 0)*86400+ - int(m.group(3) or 0)*3600+ - int(m.group(4) or 0)*60+ - int(m.group(5) or 0)) + return -1 * (int(m.group(1) or 0) * 604800 + + int(m.group(2) or 0) * 86400 + + int(m.group(3) or 0) * 3600 + + int(m.group(4) or 0) * 60 + + int(m.group(5) or 0)) raise ValueError('Invalid time: "%s"' % s) + def epipe_wrapper(func): def f(*args, **kwargs): try: @@ -211,31 +219,23 @@ def epipe_wrapper(func): raise return f + @epipe_wrapper def main(): global cfg global options parser = optparse.OptionParser(epilog='Time formats are: "%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%d" (UNIX timestamp), "-%d" (Relative time in seconds), BIND format (e.g. 1w1h, (w)eek, (d)ay, (h)our, (m)inute, (s)econd)') - parser.add_option('-c', '--config', dest='config', - help='config file', action='append') - parser.add_option('-r', '--rrset', dest='rrset', type='string', - help='rrset [/[/BAILIWICK]]') - parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', - help='rdata name [/]') - parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', - help='rdata ip ') - parser.add_option('-t', '--rrtype', dest='rrtype', type='string', - help='rrset or rdata rrtype') - parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', - help='rrset bailiwick') + parser.add_option('-c', '--config', dest='config', help='config file', action='append') + parser.add_option('-r', '--rrset', dest='rrset', type='string', help='rrset [/[/BAILIWICK]]') + parser.add_option('-n', '--rdataname', dest='rdata_name', type='string', help='rdata name [/]') + parser.add_option('-i', '--rdataip', dest='rdata_ip', type='string', help='rdata ip ') + parser.add_option('-t', '--rrtype', dest='rrtype', type='string', help='rrset or rdata rrtype') + parser.add_option('-b', '--bailiwick', dest='bailiwick', type='string', help='rrset bailiwick') parser.add_option('-s', '--sort', dest='sort', type='string', help='sort key') - parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, - help='reverse sort') - parser.add_option('-j', '--json', dest='json', action='store_true', default=False, - help='output in JSON format') - parser.add_option('-l', '--limit', dest='limit', type='int', default=0, - help='limit number of results') + parser.add_option('-R', '--reverse', dest='reverse', action='store_true', default=False, help='reverse sort') + parser.add_option('-j', '--json', dest='json', action='store_true', default=False, help='output in JSON format') + parser.add_option('-l', '--limit', dest='limit', type='int', default=0, help='limit number of results') parser.add_option('', '--before', dest='before', type='string', help='only output results seen before this time') parser.add_option('', '--after', dest='after', type='string', help='only output results seen after this time') @@ -263,20 +263,20 @@ def main(): print(str(e), file=sys.stderr) sys.exit(1) - if not 'DNSDB_SERVER' in cfg: + if 'DNSDB_SERVER' not in cfg: cfg['DNSDB_SERVER'] = DEFAULT_DNSDB_SERVER - if not 'HTTP_PROXY' in cfg: + if 'HTTP_PROXY' not in cfg: cfg['HTTP_PROXY'] = DEFAULT_HTTP_PROXY - if not 'HTTPS_PROXY' in cfg: + if 'HTTPS_PROXY' not in cfg: cfg['HTTPS_PROXY'] = DEFAULT_HTTPS_PROXY - if not 'APIKEY' in cfg: + if 'APIKEY' not in cfg: sys.stderr.write('dnsdb_query: APIKEY not defined in config file\n') sys.exit(1) client = DnsdbClient(cfg['DNSDB_SERVER'], cfg['APIKEY'], - limit=options.limit, - http_proxy=cfg['HTTP_PROXY'], - https_proxy=cfg['HTTPS_PROXY']) + limit=options.limit, + http_proxy=cfg['HTTP_PROXY'], + https_proxy=cfg['HTTPS_PROXY']) if options.rrset: if options.rrtype or options.bailiwick: qargs = (options.rrset, options.rrtype, options.bailiwick) @@ -307,7 +307,7 @@ def main(): if options.sort: results = list(results) if len(results) > 0: - if not options.sort in results[0]: + if options.sort not in results[0]: sort_keys = results[0].keys() sort_keys.sort() sys.stderr.write('dnsdb_query: invalid sort key "%s". valid sort keys are %s\n' % (options.sort, ', '.join(sort_keys))) @@ -319,5 +319,6 @@ def main(): print(e.message, file=sys.stderr) sys.exit(1) + if __name__ == '__main__': main() diff --git a/misp_modules/modules/expansion/_ransomcoindb/__init__.py b/misp_modules/modules/expansion/_ransomcoindb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py b/misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py new file mode 100755 index 0000000..26cd2e3 --- /dev/null +++ b/misp_modules/modules/expansion/_ransomcoindb/ransomcoindb.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +import requests +import logging +import os +# import pprint + +copyright = """ + Copyright 2019 (C) by Aaron Kaplan , all rights reserved. + This file is part of the ransomwarecoindDB project and licensed under the AGPL 3.0 license +""" + +__version__ = 0.1 + + +baseurl = "https://ransomcoindb.concinnity-risks.com/api/v1/" +user_agent = "ransomcoindb client via python-requests/%s" % requests.__version__ + +urls = {'BTC': {'btc': baseurl + 'bin2btc/', + 'md5': baseurl + 'bin2btc/md5/', + 'sha1': baseurl + 'bin2btc/sha1/', + 'sha256': baseurl + 'bin2btc/sha256/', + }, + 'XMR': {'xmr': baseurl + 'bin2crypto/XMR/', + 'md5': baseurl + 'bin2crypto/XMR/md5/', + 'sha1': baseurl + 'bin2crypto/XMR/sha1/', + 'sha256': baseurl + 'bin2crypto/XMR/sha256/', + } + } + + +def get_data_by(coin: str, key: str, value: str, api_key: str): + """ + Abstract function to fetch data from the bin2btc/{key} endpoint. + This function must be made concrete by generating a relevant function. + See below for examples. + """ + + # pprint.pprint("api-key: %s" % api_key) + + headers = {'x-api-key': api_key, 'content-type': 'application/json'} + headers.update({'User-Agent': user_agent}) + + # check first if valid: + valid_coins = ['BTC', 'XMR'] + valid_keys = ['btc', 'md5', 'sha1', 'sha256'] + if coin not in valid_coins or key not in valid_keys: + logging.error("get_data_by_X(): not a valid key parameter. Must be a valid coin (i.e. from %r) and one of: %r" % (valid_coins, valid_keys)) + return None + try: + + url = urls[coin.upper()][key] + logging.debug("url = %s" % url) + if not url: + logging.error("Could not find a valid coin/key combination. Must be a valid coin (i.e. from %r) and one of: %r" % (valid_coins, valid_keys)) + return None + r = requests.get(url + "%s" % (value), headers=headers) + except Exception as ex: + logging.error("could not fetch from the service. Error: %s" % str(ex)) + + if r.status_code != 200: + logging.error("could not fetch from the service. Status code: %s" % + r.status_code) + return r.json() + + +def get_bin2btc_by_btc(btc_addr: str, api_key: str): + """ Function to fetch the data from the bin2btc/{btc} endpoint """ + return get_data_by('BTC', 'btc', btc_addr, api_key) + + +def get_bin2btc_by_md5(md5: str, api_key: str): + """ Function to fetch the data from the bin2btc/{md5} endpoint """ + return get_data_by('BTC', 'md5', md5, api_key) + + +def get_bin2btc_by_sha1(sha1: str, api_key: str): + """ Function to fetch the data from the bin2btc/{sha1} endpoint """ + return get_data_by('BTC', 'sha1', sha1, api_key) + + +def get_bin2btc_by_sha256(sha256: str, api_key: str): + """ Function to fetch the data from the bin2btc/{sha256} endpoint """ + return get_data_by('BTC', 'sha256', sha256, api_key) + + +if __name__ == "__main__": + """ Just for testing on the cmd line. """ + to_btc = "1KnuC7FdhGuHpvFNxtBpz299Q5QteUdNCq" + api_key = os.getenv('api_key') + r = get_bin2btc_by_btc(to_btc, api_key) + print(r) + r = get_bin2btc_by_md5("abc", api_key) + print(r) + r = get_data_by('XMR', 'md5', "452878CD7", api_key) + print(r) diff --git a/misp_modules/modules/expansion/apiosintds.py b/misp_modules/modules/expansion/apiosintds.py new file mode 100644 index 0000000..ac0dfa4 --- /dev/null +++ b/misp_modules/modules/expansion/apiosintds.py @@ -0,0 +1,147 @@ +import json +import logging +import sys +import os +from apiosintDS import apiosintDS + +log = logging.getLogger('apiosintDS') +log.setLevel(logging.DEBUG) +apiodbg = logging.StreamHandler(sys.stdout) +apiodbg.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +apiodbg.setFormatter(formatter) +log.addHandler(apiodbg) + +misperrors = {'error': 'Error'} + +mispattributes = {'input': ["domain", "domain|ip", "hostname", "ip-dst", "ip-src", "ip-dst|port", "ip-src|port", "url", + "md5", "sha1", "sha256", "filename|md5", "filename|sha1", "filename|sha256"], + 'output': ["domain", "ip-dst", "url", "comment", "md5", "sha1", "sha256"] + } + +moduleinfo = {'version': '0.1', 'author': 'Davide Baglieri aka davidonzo', + 'description': 'On demand query API for OSINT.digitalside.it project.', + 'module-type': ['expansion', 'hover']} + +moduleconfig = ['import_related_hashes', 'cache', 'cache_directory'] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + tosubmit = [] + if request.get('domain'): + tosubmit.append(request['domain']) + elif request.get('domain|ip'): + tosubmit.append(request['domain|ip'].split('|')[0]) + tosubmit.append(request['domain|ip'].split('|')[1]) + elif request.get('hostname'): + tosubmit.append(request['hostname']) + elif request.get('ip-dst'): + tosubmit.append(request['ip-dst']) + elif request.get('ip-src'): + tosubmit.append(request['ip-src']) + elif request.get('ip-dst|port'): + tosubmit.append(request['ip-dst|port'].split('|')[0]) + elif request.get('ip-src|port'): + tosubmit.append(request['ip-src|port'].split('|')[0]) + elif request.get('url'): + tosubmit.append(request['url']) + elif request.get('md5'): + tosubmit.append(request['md5']) + elif request.get('sha1'): + tosubmit.append(request['sha1']) + elif request.get('sha256'): + tosubmit.append(request['sha256']) + elif request.get('filename|md5'): + tosubmit.append(request['filename|md5'].split('|')[1]) + elif request.get('filename|sha1'): + tosubmit.append(request['filename|sha1'].split('|')[1]) + elif request.get('filename|sha256'): + tosubmit.append(request['filename|sha256'].split('|')[1]) + else: + return False + + submitcache = False + submitcache_directory = False + import_related_hashes = False + + r = {"results": []} + + if request.get('config'): + if request['config'].get('cache') and request['config']['cache'].lower() == "yes": + submitcache = True + if request['config'].get('import_related_hashes') and request['config']['import_related_hashes'].lower() == "yes": + import_related_hashes = True + if submitcache: + cache_directory = request['config'].get('cache_directory') + if cache_directory and len(cache_directory) > 0: + if os.access(cache_directory, os.W_OK): + submitcache_directory = cache_directory + else: + ErrorMSG = "Cache directory is not writable. Please fix it before." + log.debug(str(ErrorMSG)) + misperrors['error'] = ErrorMSG + return misperrors + else: + ErrorMSG = "Value for Plugin.Enrichment_apiosintds_cache_directory is empty but cache option is enabled as recommended. Please set a writable cache directory in plugin settings." + log.debug(str(ErrorMSG)) + misperrors['error'] = ErrorMSG + return misperrors + else: + log.debug("Cache option is set to " + str(submitcache) + ". You are not using the internal cache system and this is NOT recommended!") + log.debug("Please, consider to turn on the cache setting it to 'Yes' and specifing a writable directory for the cache directory option.") + try: + response = apiosintDS.request(entities=tosubmit, cache=submitcache, cachedirectory=submitcache_directory, verbose=True) + r["results"] += reversed(apiosintParser(response, import_related_hashes)) + except Exception as e: + log.debug(str(e)) + misperrors['error'] = str(e) + return r + + +def apiosintParser(response, import_related_hashes): + ret = [] + if isinstance(response, dict): + for key in response: + for item in response[key]["items"]: + if item["response"]: + comment = item["item"] + " IS listed by OSINT.digitalside.it. Date list: " + response[key]["list"]["date"] + if key == "url": + if "hashes" in item.keys(): + if "sha256" in item["hashes"].keys(): + ret.append({"types": ["sha256"], "values": [item["hashes"]["sha256"]]}) + if "sha1" in item["hashes"].keys(): + ret.append({"types": ["sha1"], "values": [item["hashes"]["sha1"]]}) + if "md5" in item["hashes"].keys(): + ret.append({"types": ["md5"], "values": [item["hashes"]["md5"]]}) + + if len(item["related_urls"]) > 0: + for urls in item["related_urls"]: + if isinstance(urls, dict): + itemToInclude = urls["url"] + if import_related_hashes: + if "hashes" in urls.keys(): + if "sha256" in urls["hashes"].keys(): + ret.append({"types": ["sha256"], "values": [urls["hashes"]["sha256"]], "comment": "Related to: " + itemToInclude}) + if "sha1" in urls["hashes"].keys(): + ret.append({"types": ["sha1"], "values": [urls["hashes"]["sha1"]], "comment": "Related to: " + itemToInclude}) + if "md5" in urls["hashes"].keys(): + ret.append({"types": ["md5"], "values": [urls["hashes"]["md5"]], "comment": "Related to: " + itemToInclude}) + ret.append({"types": ["url"], "values": [itemToInclude], "comment": "Related to: " + item["item"]}) + else: + ret.append({"types": ["url"], "values": [urls], "comment": "Related URL to: " + item["item"]}) + else: + comment = item["item"] + " IS NOT listed by OSINT.digitalside.it. Date list: " + response[key]["list"]["date"] + ret.append({"types": ["text"], "values": [comment]}) + return ret + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/apivoid.py b/misp_modules/modules/expansion/apivoid.py new file mode 100755 index 0000000..5d6395e --- /dev/null +++ b/misp_modules/modules/expansion/apivoid.py @@ -0,0 +1,90 @@ +import json +import requests +from pymisp import MISPAttribute, MISPEvent, MISPObject + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['domain', 'hostname'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'On demand query API for APIVoid.', + 'module-type': ['expansion', 'hover']} +moduleconfig = ['apikey'] + + +class APIVoidParser(): + def __init__(self, attribute): + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.misp_event.add_attribute(**self.attribute) + self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&' + + def get_results(self): + if hasattr(self, 'result'): + return self.result + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + def parse_domain(self, apikey): + feature = 'dnslookup' + if requests.get(f'{self.url.format(feature, apikey)}stats').json()['credits_remained'] < 0.13: + self.result = {'error': 'You do not have enough APIVoid credits to proceed your request.'} + return + mapping = {'A': 'resolution-of', 'MX': 'mail-server-of', 'NS': 'server-name-of'} + dnslookup = requests.get(f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}').json() + for item in dnslookup['data']['records']['items']: + record_type = item['type'] + try: + relationship = mapping[record_type] + except KeyError: + continue + self._handle_dns_record(item, record_type, relationship) + ssl = requests.get(f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}').json() + self._parse_ssl_certificate(ssl['data']['certificate']) + + def _handle_dns_record(self, item, record_type, relationship): + dns_record = MISPObject('dns-record') + dns_record.add_attribute('queried-domain', type='domain', value=item['host']) + attribute_type, feature = ('ip-dst', 'ip') if record_type == 'A' else ('domain', 'target') + dns_record.add_attribute(f'{record_type.lower()}-record', type=attribute_type, value=item[feature]) + dns_record.add_reference(self.attribute.uuid, relationship) + self.misp_event.add_object(**dns_record) + + def _parse_ssl_certificate(self, certificate): + x509 = MISPObject('x509') + fingerprint = 'x509-fingerprint-sha1' + x509.add_attribute(fingerprint, type=fingerprint, value=certificate['fingerprint']) + x509_mapping = {'subject': {'name': ('text', 'subject')}, + 'issuer': {'common_name': ('text', 'issuer')}, + 'signature': {'serial': ('text', 'serial-number')}, + 'validity': {'valid_from': ('datetime', 'validity-not-before'), + 'valid_to': ('datetime', 'validity-not-after')}} + certificate = certificate['details'] + for feature, subfeatures in x509_mapping.items(): + for subfeature, mapping in subfeatures.items(): + attribute_type, relation = mapping + x509.add_attribute(relation, type=attribute_type, value=certificate[feature][subfeature]) + x509.add_reference(self.attribute.uuid, 'seen-by') + self.misp_event.add_object(**x509) + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('config', {}).get('apikey'): + return {'error': 'An API key for APIVoid is required.'} + attribute = request.get('attribute') + apikey = request['config']['apikey'] + apivoid_parser = APIVoidParser(attribute) + apivoid_parser.parse_domain(apikey) + return apivoid_parser.get_results() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/assemblyline_query.py b/misp_modules/modules/expansion/assemblyline_query.py new file mode 100644 index 0000000..226e4dd --- /dev/null +++ b/misp_modules/modules/expansion/assemblyline_query.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +import json +from assemblyline_client import Client, ClientError +from collections import defaultdict +from pymisp import MISPAttribute, MISPEvent, MISPObject + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['link'], 'format': 'misp_standard'} + +moduleinfo = {'version': '1', 'author': 'Christian Studer', + 'description': 'Query AssemblyLine with a report URL to get the parsed data.', + 'module-type': ['expansion']} +moduleconfig = ["apiurl", "user_id", "apikey", "password"] + + +class AssemblyLineParser(): + def __init__(self): + self.misp_event = MISPEvent() + self.results = {} + self.attribute = {'to_ids': True} + self._results_mapping = {'NET_DOMAIN_NAME': 'domain', 'NET_FULL_URI': 'url', + 'NET_IP': 'ip-dst'} + self._file_mapping = {'entropy': {'type': 'float', 'object_relation': 'entropy'}, + 'md5': {'type': 'md5', 'object_relation': 'md5'}, + 'mime': {'type': 'mime-type', 'object_relation': 'mimetype'}, + 'sha1': {'type': 'sha1', 'object_relation': 'sha1'}, + 'sha256': {'type': 'sha256', 'object_relation': 'sha256'}, + 'size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, + 'ssdeep': {'type': 'ssdeep', 'object_relation': 'ssdeep'}} + + def get_submission(self, attribute, client): + sid = attribute['value'].split('=')[-1] + try: + if not client.submission.is_completed(sid): + self.results['error'] = 'Submission not completed, please try again later.' + return + except Exception as e: + self.results['error'] = f'Something went wrong while trying to check if the submission in AssemblyLine is completed: {e.__str__()}' + return + try: + submission = client.submission.full(sid) + except Exception as e: + self.results['error'] = f"Something went wrong while getting the submission from AssemblyLine: {e.__str__()}" + return + self._parse_report(submission) + + def finalize_results(self): + if 'error' in self.results: + return self.results + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object', 'Tag') if (key in event and event[key])} + return {'results': results} + + def _create_attribute(self, result, attribute_type): + attribute = MISPAttribute() + attribute.from_dict(type=attribute_type, value=result['value'], **self.attribute) + if result['classification'] != 'UNCLASSIFIED': + attribute.add_tag(result['classification'].lower()) + self.misp_event.add_attribute(**attribute) + return {'referenced_uuid': attribute.uuid, 'relationship_type': '-'.join(result['context'].lower().split(' '))} + + def _create_file_object(self, file_info): + file_object = MISPObject('file') + filename_attribute = {'type': 'filename'} + filename_attribute.update(self.attribute) + if file_info['classification'] != "UNCLASSIFIED": + tag = {'Tag': [{'name': file_info['classification'].lower()}]} + filename_attribute.update(tag) + for feature, attribute in self._file_mapping.items(): + attribute.update(tag) + file_object.add_attribute(value=file_info[feature], **attribute) + return filename_attribute, file_object + for feature, attribute in self._file_mapping.items(): + file_object.add_attribute(value=file_info[feature], **attribute) + return filename_attribute, file_object + + @staticmethod + def _get_results(submission_results): + results = defaultdict(list) + for k, values in submission_results.items(): + h = k.split('.')[0] + for t in values['result']['tags']: + if t['context'] is not None: + results[h].append(t) + return results + + def _get_scores(self, file_tree): + scores = {} + for h, f in file_tree.items(): + score = f['score'] + if score > 0: + scores[h] = {'name': f['name'], 'score': score} + if f['children']: + scores.update(self._get_scores(f['children'])) + return scores + + def _parse_report(self, submission): + if submission['classification'] != 'UNCLASSIFIED': + self.misp_event.add_tag(submission['classification'].lower()) + filtered_results = self._get_results(submission['results']) + scores = self._get_scores(submission['file_tree']) + for h, results in filtered_results.items(): + if h in scores: + attribute, file_object = self._create_file_object(submission['file_infos'][h]) + print(file_object) + for filename in scores[h]['name']: + file_object.add_attribute('filename', value=filename, **attribute) + for reference in self._parse_results(results): + file_object.add_reference(**reference) + self.misp_event.add_object(**file_object) + + def _parse_results(self, results): + references = [] + for result in results: + try: + attribute_type = self._results_mapping[result['type']] + except KeyError: + continue + references.append(self._create_attribute(result, attribute_type)) + return references + + +def parse_config(apiurl, user_id, config): + error = {"error": "Please provide your AssemblyLine API key or Password."} + if config.get('apikey'): + try: + return Client(apiurl, apikey=(user_id, config['apikey'])) + except ClientError as e: + error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' + if config.get('password'): + try: + return Client(apiurl, auth=(user_id, config['password'])) + except ClientError as e: + error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' + return error + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('config'): + return {"error": "Missing configuration."} + if not request['config'].get('apiurl'): + return {"error": "No AssemblyLine server address provided."} + apiurl = request['config']['apiurl'] + if not request['config'].get('user_id'): + return {"error": "Please provide your AssemblyLine User ID."} + user_id = request['config']['user_id'] + client = parse_config(apiurl, user_id, request['config']) + if isinstance(client, dict): + return client + assemblyline_parser = AssemblyLineParser() + assemblyline_parser.get_submission(request['attribute'], client) + return assemblyline_parser.finalize_results() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/assemblyline_submit.py b/misp_modules/modules/expansion/assemblyline_submit.py new file mode 100644 index 0000000..206f5c0 --- /dev/null +++ b/misp_modules/modules/expansion/assemblyline_submit.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +import json + +from assemblyline_client import Client, ClientError +from urllib.parse import urljoin + + +moduleinfo = {"version": 1, "author": "Christian Studer", "module-type": ["expansion"], + "description": "Submit files or URLs to AssemblyLine"} +moduleconfig = ["apiurl", "user_id", "apikey", "password"] +mispattributes = {"input": ["attachment", "malware-sample", "url"], + "output": ["link"]} + + +def parse_config(apiurl, user_id, config): + error = {"error": "Please provide your AssemblyLine API key or Password."} + if config.get('apikey'): + try: + return Client(apiurl, apikey=(user_id, config['apikey'])) + except ClientError as e: + error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' + if config.get('password'): + try: + return Client(apiurl, auth=(user_id, config['password'])) + except ClientError as e: + error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' + return error + + +def submit_content(client, filename, data): + try: + return client.submit(fname=filename, contents=data.encode()) + except Exception as e: + return {'error': f'Error while submitting content to AssemblyLine: {e.__str__()}'} + + +def submit_request(client, request): + if 'attachment' in request: + return submit_content(client, request['attachment'], request['data']) + if 'malware-sample' in request: + return submit_content(client, request['malware-sample'].split('|')[0], request['data']) + for feature in ('url', 'domain'): + if feature in request: + return submit_url(client, request[feature]) + return {"error": "No valid attribute type for this module has been provided."} + + +def submit_url(client, url): + try: + return client.submit(url=url) + except Exception as e: + return {'error': f'Error while submitting url to AssemblyLine: {e.__str__()}'} + + +def handler(q=False): + if q is False: + return q + request = json.loads(q) + if not request.get('config'): + return {"error": "Missing configuration."} + if not request['config'].get('apiurl'): + return {"error": "No AssemblyLine server address provided."} + apiurl = request['config']['apiurl'] + if not request['config'].get('user_id'): + return {"error": "Please provide your AssemblyLine User ID."} + user_id = request['config']['user_id'] + client = parse_config(apiurl, user_id, request['config']) + if isinstance(client, dict): + return client + submission = submit_request(client, request) + if 'error' in submission: + return submission + sid = submission['submission']['sid'] + return { + "results": [{ + "types": "link", + "categories": "External analysis", + "values": urljoin(apiurl, f'submission_detail.html?sid={sid}') + }] + } + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo["config"] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/backscatter_io.py b/misp_modules/modules/expansion/backscatter_io.py new file mode 100644 index 0000000..0796917 --- /dev/null +++ b/misp_modules/modules/expansion/backscatter_io.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +"""Backscatter.io Module.""" +import json +try: + from backscatter import Backscatter +except ImportError: + print("Backscatter.io library not installed.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} +moduleinfo = {'version': '1', 'author': 'brandon@backscatter.io', + 'description': 'Backscatter.io module to bring mass-scanning observations into MISP.', + 'module-type': ['expansion', 'hover']} +moduleconfig = ['api_key'] +query_playbook = [ + {'inputs': ['ip-src', 'ip-dst'], + 'services': ['observations', 'enrichment'], + 'name': 'generic'} +] + + +def check_query(request): + """Check the incoming request for a valid configuration.""" + output = {'success': False} + config = request.get('config', None) + if not config: + misperrors['error'] = "Configuration is missing from the request." + return output + for item in moduleconfig: + if config.get(item, None): + continue + misperrors['error'] = "Backscatter.io authentication is missing." + return output + if not request.get('ip-src') and request.get('ip-dst'): + misperrors['error'] = "Unsupported attributes type." + return output + profile = {'success': True, 'config': config, 'playbook': 'generic'} + if 'ip-src' in request: + profile.update({'value': request.get('ip-src')}) + else: + profile.update({'value': request.get('ip-dst')}) + return profile + + +def handler(q=False): + """Handle gathering data.""" + if not q: + return q + request = json.loads(q) + checks = check_query(request) + if not checks['success']: + return misperrors + + try: + bs = Backscatter(checks['config']['api_key']) + response = bs.get_observations(query=checks['value'], query_type='ip') + if not response['success']: + misperrors['error'] = '%s: %s' % (response['error'], response['message']) + return misperrors + output = {'results': [{'types': mispattributes['output'], 'values': [str(response)]}]} + except Exception as e: + misperrors['error'] = str(e) + return misperrors + + return output + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/asn_history.py b/misp_modules/modules/expansion/bgpranking.py similarity index 50% rename from misp_modules/modules/expansion/asn_history.py rename to misp_modules/modules/expansion/bgpranking.py index 9775f6d..b01088d 100755 --- a/misp_modules/modules/expansion/asn_history.py +++ b/misp_modules/modules/expansion/bgpranking.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import json -from asnhistory import ASNHistory +from datetime import date, timedelta +from pybgpranking import BGPRanking misperrors = {'error': 'Error'} mispattributes = {'input': ['AS'], 'output': ['freetext']} @@ -9,8 +10,6 @@ moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', 'description': 'Query an ASN Description history service (https://github.com/CIRCL/ASN-Description-History.git)', 'module-type': ['expansion', 'hover']} -moduleconfig = ['host', 'port', 'db'] - def handler(q=False): if q is False: @@ -22,19 +21,11 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('host') and - request['config'].get('port') and - request['config'].get('db')): - misperrors['error'] = 'ASN description history configuration is missing' - return misperrors - - asnhistory = ASNHistory(host=request['config'].get('host'), - port=request['config'].get('port'), db=request['config'].get('db')) - - values = ['{} {}'.format(date.isoformat(), description) for date, description in asnhistory.get_all_descriptions(toquery)] + bgpranking = BGPRanking() + values = bgpranking.query(toquery, date=(date.today() - timedelta(1)).isoformat()) if not values: - misperrors['error'] = 'Unable to find descriptions for this ASN' + misperrors['error'] = 'Unable to find the ASN in BGP Ranking' return misperrors return {'results': [{'types': mispattributes['output'], 'values': values}]} @@ -44,5 +35,4 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/btc_scam_check.py b/misp_modules/modules/expansion/btc_scam_check.py new file mode 100644 index 0000000..f551926 --- /dev/null +++ b/misp_modules/modules/expansion/btc_scam_check.py @@ -0,0 +1,44 @@ +import json +import sys + +try: + from dns.resolver import Resolver, NXDOMAIN + from dns.name import LabelTooLong + resolver = Resolver() + resolver.timeout = 1 + resolver.lifetime = 1 +except ImportError: + sys.exit("dnspython3 in missing. use 'pip install dnspython3' to install it.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['btc'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'Checks if a BTC address has been abused.', + 'module-type': ['hover']} +moduleconfig = [] + +url = 'bl.btcblack.it' + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + btc = request['btc'] + query = f"{btc}.{url}" + try: + result = ' - '.join([str(r) for r in resolver.query(query, 'TXT')])[1:-1] + except NXDOMAIN: + result = f"{btc} is not known as a scam address." + except LabelTooLong: + result = f"{btc} is probably not a valid BTC address." + return {'results': [{'types': mispattributes['output'], 'values': result}]} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/btc_steroids.py b/misp_modules/modules/expansion/btc_steroids.py new file mode 100755 index 0000000..04b7138 --- /dev/null +++ b/misp_modules/modules/expansion/btc_steroids.py @@ -0,0 +1,229 @@ +import json +import requests +import time + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['btc'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'BTC expansion service to \ + get quick information from MISP attributes', + 'module-type': ['hover']} + +moduleconfig = [] + +blockchain_firstseen = 'https://blockchain.info/q/addressfirstseen/' +blockchain_balance = 'https://blockchain.info/q/addressbalance/' +blockchain_totalreceived = 'https://blockchain.info/q/getreceivedbyaddress/' +blockchain_all = 'https://blockchain.info/rawaddr/{}?filter=5{}' +converter = 'https://min-api.cryptocompare.com/data/pricehistorical?fsym=BTC&tsyms=USD,EUR&ts={}' +converter_rls = 'https://min-api.cryptocompare.com/stats/rate/limit' +result_text = "" +g_rate_limit = 300 +start_time = 0 +conversion_rates = {} + + +def get_consumption(output=False): + try: + req = requests.get(converter_rls) + jreq = req.json() + minute = str(jreq['Data']['calls_left']['minute']) + hour = str(jreq['Data']['calls_left']['hour']) + except Exception: + minute = str(-1) + hour = str(-1) + # Debug out for the console + print("Calls left this minute / hour: " + minute + " / " + hour) + return minute, hour + + +def convert(btc, timestamp): + global g_rate_limit + global start_time + global now + global conversion_rates + date = time.strftime('%Y-%m-%d', time.localtime(timestamp)) + # Lookup conversion rates in the cache: + if date in conversion_rates: + (usd, eur) = conversion_rates[date] + else: + # If not cached, we have to get the converion rates + # We have to be careful with rate limiting on the server side + if g_rate_limit == 300: + minute, hour = get_consumption() + g_rate_limit -= 1 + now = time.time() + # delta = now - start_time + # print(g_rate_limit) + if g_rate_limit <= 10: + minute, hour = get_consumption(output=True) + if int(minute) <= 10: + # print(minute) + # get_consumption(output=True) + time.sleep(3) + else: + mprint(minute) + start_time = time.time() + g_rate_limit = int(minute) + try: + req = requests.get(converter.format(timestamp)) + jreq = req.json() + usd = jreq['BTC']['USD'] + eur = jreq['BTC']['EUR'] + # Since we have the rates, store them in the cache + conversion_rates[date] = (usd, eur) + except Exception as ex: + mprint(ex) + get_consumption(output=True) + # Actually convert and return the values + u = usd * btc + e = eur * btc + return u, e + + +def mprint(input): + # Prepare the final print + global result_text + result_text = result_text + "\n" + str(input) + + +def handler(q=False): + global result_text + global conversion_rates + result_text = "" + # start_time = time.time() + # now = time.time() + if q is False: + return False + request = json.loads(q) + click = False + # This means the magnifying glass has been clicked + if request.get('persistent') == 1: + click = True + # Otherwise the attribute was only hovered over + if request.get('btc'): + btc = request['btc'] + else: + return False + mprint("\nAddress:\t" + btc) + try: + req = requests.get(blockchain_all.format(btc, "&limit=50")) + jreq = req.json() + except Exception: + # print(e) + print(req.text) + result_text = "Not a valid BTC address" + r = { + 'results': [ + { + 'types': ['text'], + 'values':[ + str(result_text) + ] + } + ] + } + return r + + n_tx = jreq['n_tx'] + balance = float(jreq['final_balance'] / 100000000) + rcvd = float(jreq['total_received'] / 100000000) + sent = float(jreq['total_sent'] / 100000000) + output = 'Balance:\t{0:.10f} BTC (+{1:.10f} BTC / -{2:.10f} BTC)' + mprint(output.format(balance, rcvd, sent)) + if click is False: + mprint("Transactions:\t" + str(n_tx) + "\t (previewing up to 5 most recent)") + else: + mprint("Transactions:\t" + str(n_tx)) + if n_tx > 0: + mprint("======================================================================================") + i = 0 + while i < n_tx: + if click is False: + try: + req = requests.get(blockchain_all.format(btc, "&limit=5&offset={}".format(i))) + except Exception as e: + # Lazy retry - cries for a function + print(e) + time.sleep(3) + req = requests.get(blockchain_all.format(btc, "&limit=5&offset={}".format(i))) + if n_tx > 5: + n_tx = 5 + else: + try: + req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i))) + except Exception as e: + # Lazy retry - cries for a function + print(e) + time.sleep(3) + req = requests.get(blockchain_all.format(btc, "&limit=50&offset={}".format(i))) + jreq = req.json() + if jreq['txs']: + for transactions in jreq['txs']: + sum = 0 + sum_counter = 0 + for tx in transactions['inputs']: + script_old = tx['script'] + try: + addr_in = tx['prev_out']['addr'] + except KeyError: + addr_in = None + try: + prev_out = tx['prev_out']['value'] + except KeyError: + prev_out = None + if prev_out != 0 and addr_in == btc: + datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time']))) + value = float(tx['prev_out']['value'] / 100000000) + u, e = convert(value, transactions['time']) + mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0')) + if script_old != tx['script']: + i += 1 + else: + sum_counter += 1 + sum += value + if sum_counter > 1: + u, e = convert(sum, transactions['time']) + mprint("\t\t\t\t\t----------------------------------------------") + mprint("#" + str(n_tx - i) + "\t\t\t\t Sum:\t-{0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR\n".format(sum, u, e).rstrip('0')) + for tx in transactions['out']: + try: + addr_out = tx['addr'] + except KeyError: + addr_out = None + try: + prev_out = tx['prev_out']['value'] + except KeyError: + prev_out = None + if prev_out != 0 and addr_out == btc: + datetime = time.strftime("%d %b %Y %H:%M:%S %Z", time.localtime(int(transactions['time']))) + value = float(tx['value'] / 100000000) + u, e = convert(value, transactions['time']) + mprint("#" + str(n_tx - i) + "\t" + str(datetime) + "\t {0:10.8f} BTC {1:10.2f} USD\t{2:10.2f} EUR".format(value, u, e).rstrip('0')) + # i += 1 + i += 1 + + r = { + 'results': [ + { + 'types': ['text'], + 'values':[ + str(result_text) + ] + } + ] + } + # Debug output on the console + print(result_text) + # Unset the result for the next request + result_text = "" + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/censys_enrich.py b/misp_modules/modules/expansion/censys_enrich.py new file mode 100644 index 0000000..0fc61ae --- /dev/null +++ b/misp_modules/modules/expansion/censys_enrich.py @@ -0,0 +1,255 @@ +# encoding: utf-8 +import json +import base64 +import codecs +from dateutil.parser import isoparse +from pymisp import MISPAttribute, MISPEvent, MISPObject +try: + import censys.base + import censys.ipv4 + import censys.websites + import censys.certificates +except ImportError: + print("Censys module not installed. Try 'pip install censys'") + +misperrors = {'error': 'Error'} +moduleconfig = ['api_id', 'api_secret'] +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain', 'hostname', 'hostname|port', 'domain|ip', 'ip-dst|port', 'ip-src|port', + 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.1', 'author': 'Loïc Fortemps', + 'description': 'Censys.io expansion module', 'module-type': ['expansion', 'hover']} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if request.get('config'): + if (request['config'].get('api_id') is None) or (request['config'].get('api_secret') is None): + misperrors['error'] = "Censys API credentials are missing" + return misperrors + else: + misperrors['error'] = "Please provide config options" + return misperrors + + api_id = request['config']['api_id'] + api_secret = request['config']['api_secret'] + + if not request.get('attribute'): + return {'error': 'Unsupported input.'} + attribute = request['attribute'] + if not any(input_type == attribute['type'] for input_type in mispattributes['input']): + return {'error': 'Unsupported attributes type'} + + attribute = MISPAttribute() + attribute.from_dict(**request['attribute']) + # Lists to accomodate multi-types attribute + conn = list() + types = list() + values = list() + results = list() + + if "|" in attribute.type: + t_1, t_2 = attribute.type.split('|') + v_1, v_2 = attribute.value.split('|') + # We cannot use the port information + if t_2 == "port": + types.append(t_1) + values.append(v_1) + else: + types = [t_1, t_2] + values = [v_1, v_2] + else: + types.append(attribute.type) + values.append(attribute.value) + + for t in types: + # ip, ip-src or ip-dst + if t[:2] == "ip": + conn.append(censys.ipv4.CensysIPv4(api_id=api_id, api_secret=api_secret)) + elif t == 'domain' or t == "hostname": + conn.append(censys.websites.CensysWebsites(api_id=api_id, api_secret=api_secret)) + elif 'x509-fingerprint' in t: + conn.append(censys.certificates.CensysCertificates(api_id=api_id, api_secret=api_secret)) + + found = True + for c in conn: + val = values.pop(0) + try: + r = c.view(val) + results.append(parse_response(r, attribute)) + found = True + except censys.base.CensysNotFoundException: + found = False + except Exception: + misperrors['error'] = "Connection issue" + return misperrors + + if not found: + misperrors['error'] = "Nothing could be found on Censys" + return misperrors + + return {'results': remove_duplicates(results)} + + +def parse_response(censys_output, attribute): + misp_event = MISPEvent() + misp_event.add_attribute(**attribute) + # Generic fields (for IP/Websites) + if "autonomous_system" in censys_output: + cen_as = censys_output['autonomous_system'] + asn_object = MISPObject('asn') + asn_object.add_attribute('asn', value=cen_as["asn"]) + asn_object.add_attribute('description', value=cen_as['name']) + asn_object.add_attribute('subnet-announced', value=cen_as['routed_prefix']) + asn_object.add_attribute('country', value=cen_as['country_code']) + asn_object.add_reference(attribute.uuid, 'associated-to') + misp_event.add_object(**asn_object) + + if "ip" in censys_output and "ports" in censys_output: + ip_object = MISPObject('ip-port') + ip_object.add_attribute('ip', value=censys_output['ip']) + for p in censys_output['ports']: + ip_object.add_attribute('dst-port', value=p) + ip_object.add_reference(attribute.uuid, 'associated-to') + misp_event.add_object(**ip_object) + + # We explore all ports to find https or ssh services + for k in censys_output.keys(): + if not isinstance(censys_output[k], dict): + continue + if 'https' in censys_output[k]: + try: + cert = censys_output[k]['https']['tls']['certificate'] + cert_obj = get_certificate_object(cert, attribute) + misp_event.add_object(**cert_obj) + except KeyError: + print("Error !") + if 'ssh' in censys_output[k]: + try: + cert = censys_output[k]['ssh']['v2']['server_host_key'] + # TODO enable once the type is merged + # misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256']) + except KeyError: + pass + + # Info from certificate query + if "parsed" in censys_output: + cert_obj = get_certificate_object(censys_output, attribute) + misp_event.add_object(**cert_obj) + + # Location can be present for IP/Websites results + if "location" in censys_output: + loc_obj = MISPObject('geolocation') + loc = censys_output['location'] + loc_obj.add_attribute('latitude', value=loc['latitude']) + loc_obj.add_attribute('longitude', value=loc['longitude']) + if 'city' in loc: + loc_obj.add_attribute('city', value=loc['city']) + loc_obj.add_attribute('country', value=loc['country']) + if 'postal_code' in loc: + loc_obj.add_attribute('zipcode', value=loc['postal_code']) + if 'province' in loc: + loc_obj.add_attribute('region', value=loc['province']) + loc_obj.add_reference(attribute.uuid, 'associated-to') + misp_event.add_object(**loc_obj) + + event = json.loads(misp_event.to_json()) + return {'Object': event['Object'], 'Attribute': event['Attribute']} + + +# In case of multiple enrichment (ip and domain), we need to filter out similar objects +# TODO: make it more granular +def remove_duplicates(results): + # Only one enrichment was performed so no duplicate + if len(results) == 1: + return results[0] + elif len(results) == 2: + final_result = results[0] + obj_l2 = results[1]['Object'] + for o2 in obj_l2: + if o2['name'] == "asn": + key = "asn" + elif o2['name'] == "ip-port": + key = "ip" + elif o2['name'] == "x509": + key = "x509-fingerprint-sha256" + elif o2['name'] == "geolocation": + key = "latitude" + if not check_if_present(o2, key, final_result['Object']): + final_result['Object'].append(o2) + + return final_result + else: + return [] + + +def check_if_present(object, attribute_name, list_objects): + """ + Assert if a given object is present in the list. + + This function check if object (json format) is present in list_objects + using attribute_name for the matching + """ + for o in list_objects: + # We first look for a match on the name + if o['name'] == object['name']: + for attr in object['Attribute']: + # Within the attributes, we look for the one to compare + if attr['type'] == attribute_name: + # Then we check the attributes of the other object and look for a match + for attr2 in o['Attribute']: + if attr2['type'] == attribute_name and attr2['value'] == attr['value']: + return True + + return False + + +def get_certificate_object(cert, attribute): + parsed = cert['parsed'] + cert_object = MISPObject('x509') + cert_object.add_attribute('x509-fingerprint-sha256', value=parsed['fingerprint_sha256']) + cert_object.add_attribute('x509-fingerprint-sha1', value=parsed['fingerprint_sha1']) + cert_object.add_attribute('x509-fingerprint-md5', value=parsed['fingerprint_md5']) + cert_object.add_attribute('serial-number', value=parsed['serial_number']) + cert_object.add_attribute('version', value=parsed['version']) + cert_object.add_attribute('subject', value=parsed['subject_dn']) + cert_object.add_attribute('issuer', value=parsed['issuer_dn']) + cert_object.add_attribute('validity-not-before', value=isoparse(parsed['validity']['start'])) + cert_object.add_attribute('validity-not-after', value=isoparse(parsed['validity']['end'])) + cert_object.add_attribute('self_signed', value=parsed['signature']['self_signed']) + cert_object.add_attribute('signature_algorithm', value=parsed['signature']['signature_algorithm']['name']) + + cert_object.add_attribute('pubkey-info-algorithm', value=parsed['subject_key_info']['key_algorithm']['name']) + + if 'rsa_public_key' in parsed['subject_key_info']: + pub_key = parsed['subject_key_info']['rsa_public_key'] + cert_object.add_attribute('pubkey-info-size', value=pub_key['length']) + cert_object.add_attribute('pubkey-info-exponent', value=pub_key['exponent']) + hex_mod = codecs.encode(base64.b64decode(pub_key['modulus']), 'hex').decode() + cert_object.add_attribute('pubkey-info-modulus', value=hex_mod) + + if "extensions" in parsed and "subject_alt_name" in parsed["extensions"]: + san = parsed["extensions"]["subject_alt_name"] + if "dns_names" in san: + for dns in san['dns_names']: + cert_object.add_attribute('dns_names', value=dns) + if "ip_addresses" in san: + for ip in san['ip_addresses']: + cert_object.add_attribute('ip', value=ip) + + if "raw" in cert: + cert_object.add_attribute('raw-base64', value=cert['raw']) + + cert_object.add_reference(attribute.uuid, 'associated-to') + return cert_object + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/circl_passivedns.py b/misp_modules/modules/expansion/circl_passivedns.py index 3da5bac..d278a85 100755 --- a/misp_modules/modules/expansion/circl_passivedns.py +++ b/misp_modules/modules/expansion/circl_passivedns.py @@ -1,41 +1,71 @@ import json import pypdns +from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], 'output': ['freetext']} -moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy', 'description': 'Module to access CIRCL Passive DNS', 'module-type': ['expansion', 'hover']} +mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.2', 'author': 'Alexandre Dulaunoy', + 'description': 'Module to access CIRCL Passive DNS', + 'module-type': ['expansion', 'hover']} moduleconfig = ['username', 'password'] +class PassiveDNSParser(): + def __init__(self, attribute, authentication): + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.misp_event.add_attribute(**self.attribute) + self.pdns = pypdns.PyPDNS(basic_auth=authentication) + + def get_results(self): + if hasattr(self, 'result'): + return self.result + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + def parse(self): + value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value + + try: + results = self.pdns.query(value) + except Exception: + self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'} + return + + if not results: + self.result = {'error': 'Not found'} + return + + mapping = {'count': 'counter', 'origin': 'text', + 'time_first': 'datetime', 'rrtype': 'text', + 'rrname': 'text', 'rdata': 'text', + 'time_last': 'datetime'} + for result in results: + pdns_object = MISPObject('passive-dns') + for relation, attribute_type in mapping.items(): + pdns_object.add_attribute(relation, type=attribute_type, value=result[relation]) + pdns_object.add_reference(self.attribute.uuid, 'associated-to') + self.misp_event.add_object(**pdns_object) + + def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('hostname'): - toquery = request['hostname'] - elif request.get('domain'): - toquery = request['domain'] - elif request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip-dst'): - toquery = request['ip-dst'] - else: - misperrors['error'] = "Unsupported attributes type" - return misperrors - - if (request.get('config')): - if (request['config'].get('username') is None) or (request['config'].get('password') is None): - misperrors['error'] = 'CIRCL Passive DNS authentication is missing' - return misperrors - - x = pypdns.PyPDNS(basic_auth=(request['config']['username'], request['config']['password'])) - res = x.query(toquery) - out = '' - for v in res: - out = out + "{} ".format(v['rdata']) - - r = {'results': [{'types': mispattributes['output'], 'values': out}]} - return r + if not request.get('config'): + return {'error': 'CIRCL Passive DNS authentication is missing.'} + if not request['config'].get('username') or not request['config'].get('password'): + return {'error': 'CIRCL Passive DNS authentication is incomplete, please provide your username and password.'} + authentication = (request['config']['username'], request['config']['password']) + if not request.get('attribute'): + return {'error': 'Unsupported input.'} + attribute = request['attribute'] + if not any(input_type == attribute['type'] for input_type in mispattributes['input']): + return {'error': 'Unsupported attributes type'} + pdns_parser = PassiveDNSParser(attribute, authentication) + pdns_parser.parse() + return pdns_parser.get_results() def introspection(): diff --git a/misp_modules/modules/expansion/circl_passivessl.py b/misp_modules/modules/expansion/circl_passivessl.py index c6d5a3f..102bed8 100755 --- a/misp_modules/modules/expansion/circl_passivessl.py +++ b/misp_modules/modules/expansion/circl_passivessl.py @@ -1,35 +1,96 @@ import json import pypssl +from pymisp import MISPAttribute, MISPEvent, MISPObject -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} -moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', 'description': 'Module to access CIRCL Passive SSL', 'module-type': ['expansion', 'hover']} +mispattributes = {'input': ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.2', 'author': 'Raphaël Vinot', + 'description': 'Module to access CIRCL Passive SSL', + 'module-type': ['expansion', 'hover']} moduleconfig = ['username', 'password'] +class PassiveSSLParser(): + def __init__(self, attribute, authentication): + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.misp_event.add_attribute(**self.attribute) + self.pssl = pypssl.PyPSSL(basic_auth=authentication) + self.cert_hash = 'x509-fingerprint-sha1' + self.cert_type = 'pem' + self.mapping = {'issuer': ('text', 'issuer'), + 'keylength': ('text', 'pubkey-info-size'), + 'not_after': ('datetime', 'validity-not-after'), + 'not_before': ('datetime', 'validity-not-before'), + 'subject': ('text', 'subject')} + + def get_results(self): + if hasattr(self, 'result'): + return self.result + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + def parse(self): + value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value + + try: + results = self.pssl.query(value) + except Exception: + self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'} + return + + if not results: + self.result = {'error': 'Not found'} + return + + if 'error' in results: + self.result = {'error': results['error']} + return + + for ip_address, certificates in results.items(): + ip_uuid = self._handle_ip_attribute(ip_address) + for certificate in certificates['certificates']: + self._handle_certificate(certificate, ip_uuid) + + def _handle_certificate(self, certificate, ip_uuid): + x509 = MISPObject('x509') + x509.add_attribute(self.cert_hash, type=self.cert_hash, value=certificate) + cert_details = self.pssl.fetch_cert(certificate) + info = cert_details['info'] + for feature, mapping in self.mapping.items(): + attribute_type, object_relation = mapping + x509.add_attribute(object_relation, type=attribute_type, value=info[feature]) + x509.add_attribute(self.cert_type, type='text', value=self.cert_type) + x509.add_reference(ip_uuid, 'seen-by') + self.misp_event.add_object(**x509) + + def _handle_ip_attribute(self, ip_address): + if ip_address == self.attribute.value: + return self.attribute.uuid + ip_attribute = MISPAttribute() + ip_attribute.from_dict(**{'type': self.attribute.type, 'value': ip_address}) + self.misp_event.add_attribute(**ip_attribute) + return ip_attribute.uuid + + def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip-dst'): - toquery = request['ip-dst'] - else: - misperrors['error'] = "Unsupported attributes type" - return misperrors - - if request.get('config'): - if (request['config'].get('username') is None) or (request['config'].get('password') is None): - misperrors['error'] = 'CIRCL Passive SSL authentication is missing' - return misperrors - - x = pypssl.PyPSSL(basic_auth=(request['config']['username'], request['config']['password'])) - res = x.query(toquery) - out = res.get(toquery) - - r = {'results': [{'types': mispattributes['output'], 'values': out}]} - return r + if not request.get('config'): + return {'error': 'CIRCL Passive SSL authentication is missing.'} + if not request['config'].get('username') or not request['config'].get('password'): + return {'error': 'CIRCL Passive SSL authentication is incomplete, please provide your username and password.'} + authentication = (request['config']['username'], request['config']['password']) + if not request.get('attribute'): + return {'error': 'Unsupported input.'} + attribute = request['attribute'] + if not any(input_type == attribute['type'] for input_type in mispattributes['input']): + return {'error': 'Unsupported attributes type'} + pssl_parser = PassiveSSLParser(attribute, authentication) + pssl_parser.parse() + return pssl_parser.get_results() def introspection(): diff --git a/misp_modules/modules/expansion/countrycode.py b/misp_modules/modules/expansion/countrycode.py index 9f22c40..1de56e0 100755 --- a/misp_modules/modules/expansion/countrycode.py +++ b/misp_modules/modules/expansion/countrycode.py @@ -12,42 +12,41 @@ moduleinfo = {'version': '1', 'author': 'Hannah Ward', # config fields that your code expects from the site admin moduleconfig = [] -common_tlds = {"com":"Commercial (Worldwide)", - "org":"Organisation (Worldwide)", - "net":"Network (Worldwide)", - "int":"International (Worldwide)", - "edu":"Education (Usually USA)", - "gov":"Government (USA)" - } +common_tlds = {"com": "Commercial (Worldwide)", + "org": "Organisation (Worldwide)", + "net": "Network (Worldwide)", + "int": "International (Worldwide)", + "edu": "Education (Usually USA)", + "gov": "Government (USA)" + } + + +def parse_country_code(extension): + # Retrieve a json full of country info + try: + codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json() + except Exception: + return "http://www.geognos.com/api/en/countries/info/all.json not reachable" + if not codes.get('StatusMsg') or not codes["StatusMsg"] == "OK": + return 'Not able to get the countrycode references from http://www.geognos.com/api/en/countries/info/all.json' + for country in codes['Results'].values(): + if country['CountryCodes']['tld'] == extension: + return country['Name'] + return "Unknown" -codes = False def handler(q=False): - global codes - if not codes: - codes = requests.get("http://www.geognos.com/api/en/countries/info/all.json").json() if q is False: return False request = json.loads(q) - domain = request["domain"] + domain = request["domain"] if "domain" in request else request["hostname"] # Get the extension ext = domain.split(".")[-1] - # Check if if's a common, non country one - if ext in common_tlds.keys(): - val = common_tlds[ext] - else: - # Retrieve a json full of country info - if not codes["StatusMsg"] == "OK": - val = "Unknown" - else: - # Find our code based on TLD - codes = codes["Results"] - for code in codes.keys(): - if codes[code]["CountryCodes"]["tld"] == ext: - val = codes[code]["Name"] - r = {'results': [{'types':['text'], 'values':[val]}]} + # Check if it's a common, non country one + val = common_tlds[ext] if ext in common_tlds.keys() else parse_country_code(ext) + r = {'results': [{'types': ['text'], 'values':[val]}]} return r @@ -58,4 +57,3 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo - diff --git a/misp_modules/modules/expansion/cuckoo_submit.py b/misp_modules/modules/expansion/cuckoo_submit.py new file mode 100644 index 0000000..c1ded90 --- /dev/null +++ b/misp_modules/modules/expansion/cuckoo_submit.py @@ -0,0 +1,153 @@ +import base64 +import io +import json +import logging +import requests +import sys +import urllib.parse +import zipfile + +from requests.exceptions import RequestException + +log = logging.getLogger("cuckoo_submit") +log.setLevel(logging.DEBUG) +sh = logging.StreamHandler(sys.stdout) +sh.setLevel(logging.DEBUG) +fmt = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +sh.setFormatter(fmt) +log.addHandler(sh) + +moduleinfo = { + "version": "0.1", "author": "Evert Kors", + "description": "Submit files and URLs to Cuckoo Sandbox", + "module-type": ["expansion", "hover"] +} +misperrors = {"error": "Error"} +moduleconfig = ["api_url", "api_key"] +mispattributes = { + "input": ["attachment", "malware-sample", "url", "domain"], + "output": ["text"] +} + + +class APIKeyError(RequestException): + """Raised if the Cuckoo API returns a 401. This means no or an invalid + bearer token was supplied.""" + pass + + +class CuckooAPI(object): + + def __init__(self, api_url, api_key=""): + self.api_key = api_key + if not api_url.startswith("http"): + api_url = "https://{}".format(api_url) + + self.api_url = api_url + + def _post_api(self, endpoint, files=None, data={}): + data.update({ + "owner": "MISP" + }) + + try: + response = requests.post( + urllib.parse.urljoin(self.api_url, endpoint), + files=files, data=data, + headers={"Authorization": "Bearer {}".format(self.api_key)} + ) + except RequestException as e: + log.error("Failed to submit sample to Cuckoo Sandbox. %s", e) + return None + + if response.status_code == 401: + raise APIKeyError("Invalid or no Cuckoo Sandbox API key provided") + + if response.status_code != 200: + log.error("Invalid Cuckoo API response") + return None + + return response.json() + + def create_task(self, filename, fp): + response = self._post_api( + "/tasks/create/file", files={"file": (filename, fp)} + ) + if not response: + return False + + return response["task_id"] + + def create_url(self, url): + response = self._post_api( + "/tasks/create/url", data={"url": url} + ) + if not response: + return False + + return response["task_id"] + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + # See if the API URL was provided. The API key is optional, as it can + # be disabled in the Cuckoo API settings. + api_url = request["config"].get("api_url") + api_key = request["config"].get("api_key", "") + if not api_url: + misperrors["error"] = "No Cuckoo API URL provided" + return misperrors + + url = request.get("url") or request.get("domain") + data = request.get("data") + filename = None + if data: + data = base64.b64decode(data) + + if "malware-sample" in request: + filename = request.get("malware-sample").split("|", 1)[0] + with zipfile.ZipFile(io.BytesIO(data)) as zipf: + data = zipf.read(zipf.namelist()[0], pwd=b"infected") + + elif "attachment" in request: + filename = request.get("attachment") + + cuckoo_api = CuckooAPI(api_url=api_url, api_key=api_key) + task_id = None + try: + if url: + log.debug("Submitting URL to Cuckoo Sandbox %s", api_url) + task_id = cuckoo_api.create_url(url) + elif data and filename: + log.debug("Submitting file to Cuckoo Sandbox %s", api_url) + task_id = cuckoo_api.create_task( + filename=filename, fp=io.BytesIO(data) + ) + except APIKeyError as e: + misperrors["error"] = "Failed to submit to Cuckoo: {}".format(e) + return misperrors + + if not task_id: + misperrors["error"] = "File or URL submission failed" + return misperrors + + return { + "results": [ + {"types": "text", "values": "Cuckoo task id: {}".format(task_id)} + ] + } + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo["config"] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/cve.py b/misp_modules/modules/expansion/cve.py index 81d8db5..90c46bf 100755 --- a/misp_modules/modules/expansion/cve.py +++ b/misp_modules/modules/expansion/cve.py @@ -3,11 +3,15 @@ import requests misperrors = {'error': 'Error'} mispattributes = {'input': ['vulnerability'], 'output': ['text']} -moduleinfo = {'version': '0.2', 'author': 'Alexandre Dulaunoy', 'description': 'An expansion hover module to expand information about CVE id.', 'module-type': ['hover']} -moduleconfig = [] +moduleinfo = {'version': '0.3', 'author': 'Alexandre Dulaunoy', 'description': 'An expansion hover module to expand information about CVE id.', 'module-type': ['hover']} +moduleconfig = ["custom_API"] cveapi_url = 'https://cve.circl.lu/api/cve/' +def check_url(url): + return "{}/".format(url) if not url.endswith('/') else url + + def handler(q=False): if q is False: return False @@ -16,7 +20,8 @@ def handler(q=False): misperrors['error'] = 'Vulnerability id missing' return misperrors - r = requests.get(cveapi_url + request.get('vulnerability')) + api_url = check_url(request['config']['custom_API']) if request.get('config') and request['config'].get('custom_API') else cveapi_url + r = requests.get("{}{}".format(api_url, request.get('vulnerability'))) if r.status_code == 200: vulnerability = json.loads(r.text) if vulnerability: @@ -25,7 +30,7 @@ def handler(q=False): else: summary = 'Non existing CVE' else: - misperrors['error'] = 'cve.circl.lu API not accessible' + misperrors['error'] = 'API not accessible' return misperrors['error'] r = {'results': [{'types': mispattributes['output'], 'values': summary}]} diff --git a/misp_modules/modules/expansion/cve_advanced.py b/misp_modules/modules/expansion/cve_advanced.py new file mode 100644 index 0000000..86cba8c --- /dev/null +++ b/misp_modules/modules/expansion/cve_advanced.py @@ -0,0 +1,136 @@ +from collections import defaultdict +from pymisp import MISPEvent, MISPObject +import json +import requests + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'} +moduleinfo = {'version': '1', 'author': 'Christian Studer', + 'description': 'An expansion module to enrich a CVE attribute with the vulnerability information.', + 'module-type': ['expansion', 'hover']} +moduleconfig = ["custom_API"] +cveapi_url = 'https://cve.circl.lu/api/cve/' + + +class VulnerabilityParser(): + def __init__(self, attribute, vulnerability, api_url): + self.attribute = attribute + self.vulnerability = vulnerability + self.api_url = api_url + self.misp_event = MISPEvent() + self.misp_event.add_attribute(**attribute) + self.references = defaultdict(list) + self.capec_features = ('id', 'name', 'summary', 'prerequisites', 'solutions') + self.vulnerability_mapping = { + 'id': ('text', 'id'), 'summary': ('text', 'summary'), + 'vulnerable_configuration': ('text', 'vulnerable_configuration'), + 'vulnerable_configuration_cpe_2_2': ('text', 'vulnerable_configuration'), + 'Modified': ('datetime', 'modified'), 'Published': ('datetime', 'published'), + 'references': ('link', 'references'), 'cvss': ('float', 'cvss-score')} + self.weakness_mapping = {'name': 'name', 'description_summary': 'description', + 'status': 'status', 'weaknessabs': 'weakness-abs'} + + def get_result(self): + if self.references: + self.__build_references() + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + def parse_vulnerability_information(self): + vulnerability_object = MISPObject('vulnerability') + for feature in ('id', 'summary', 'Modified', 'cvss'): + value = self.vulnerability.get(feature) + if value: + attribute_type, relation = self.vulnerability_mapping[feature] + vulnerability_object.add_attribute(relation, **{'type': attribute_type, 'value': value}) + if 'Published' in self.vulnerability: + vulnerability_object.add_attribute('published', **{'type': 'datetime', 'value': self.vulnerability['Published']}) + vulnerability_object.add_attribute('state', **{'type': 'text', 'value': 'Published'}) + for feature in ('references', 'vulnerable_configuration', 'vulnerable_configuration_cpe_2_2'): + if feature in self.vulnerability: + attribute_type, relation = self.vulnerability_mapping[feature] + for value in self.vulnerability[feature]: + if isinstance(value, dict): + value = value['title'] + vulnerability_object.add_attribute(relation, **{'type': attribute_type, 'value': value}) + vulnerability_object.add_reference(self.attribute['uuid'], 'related-to') + self.misp_event.add_object(**vulnerability_object) + if 'cwe' in self.vulnerability and self.vulnerability['cwe'] not in ('Unknown', 'NVD-CWE-noinfo'): + self.__parse_weakness(vulnerability_object.uuid) + if 'capec' in self.vulnerability: + self.__parse_capec(vulnerability_object.uuid) + + def __build_references(self): + for object_uuid, references in self.references.items(): + for misp_object in self.misp_event.objects: + if misp_object.uuid == object_uuid: + for reference in references: + misp_object.add_reference(**reference) + break + + def __parse_capec(self, vulnerability_uuid): + attribute_type = 'text' + for capec in self.vulnerability['capec']: + capec_object = MISPObject('attack-pattern') + for feature in self.capec_features: + capec_object.add_attribute(feature, **dict(type=attribute_type, value=capec[feature])) + for related_weakness in capec['related_weakness']: + attribute = dict(type='weakness', value="CWE-{}".format(related_weakness)) + capec_object.add_attribute('related-weakness', **attribute) + self.misp_event.add_object(**capec_object) + self.references[vulnerability_uuid].append(dict(referenced_uuid=capec_object.uuid, + relationship_type='targeted-by')) + + def __parse_weakness(self, vulnerability_uuid): + attribute_type = 'text' + cwe_string, cwe_id = self.vulnerability['cwe'].split('-') + cwes = requests.get(self.api_url.replace('/cve/', '/cwe')) + if cwes.status_code == 200: + for cwe in cwes.json(): + if cwe['id'] == cwe_id: + weakness_object = MISPObject('weakness') + weakness_object.add_attribute('id', **dict(type=attribute_type, value='-'.join([cwe_string, cwe_id]))) + for feature, relation in self.weakness_mapping.items(): + if cwe.get(feature): + weakness_object.add_attribute(relation, **dict(type=attribute_type, value=cwe[feature])) + self.misp_event.add_object(**weakness_object) + self.references[vulnerability_uuid].append(dict(referenced_uuid=weakness_object.uuid, + relationship_type='weakened-by')) + break + + +def check_url(url): + return "{}/".format(url) if not url.endswith('/') else url + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + attribute = request.get('attribute') + if attribute.get('type') != 'vulnerability': + misperrors['error'] = 'Vulnerability id missing.' + return misperrors + api_url = check_url(request['config']['custom_API']) if request['config'].get('custom_API') else cveapi_url + r = requests.get("{}{}".format(api_url, attribute['value'])) + if r.status_code == 200: + vulnerability = r.json() + if not vulnerability: + misperrors['error'] = 'Non existing CVE' + return misperrors['error'] + else: + misperrors['error'] = 'API not accessible' + return misperrors['error'] + parser = VulnerabilityParser(attribute, vulnerability, api_url) + parser.parse_vulnerability_information() + return parser.get_result() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py new file mode 100755 index 0000000..9723ed6 --- /dev/null +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +''' +Cytomic Orion MISP Module +An expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion + + +''' + +from pymisp import MISPAttribute, MISPEvent, MISPObject +import json +import requests +import sys + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['md5'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.3', 'author': 'Koen Van Impe', + 'description': 'an expansion module to enrich attributes in MISP and share indicators of compromise with Cytomic Orion', + 'module-type': ['expansion']} +moduleconfig = ['api_url', 'token_url', 'clientid', 'clientsecret', 'clientsecret', 'username', 'password', 'upload_timeframe', 'upload_tag', 'delete_tag', 'upload_ttlDays', 'upload_threat_level_id', 'limit_upload_events', 'limit_upload_attributes'] +# There are more config settings in this module than used by the enrichment +# There is also a PyMISP module which reuses the module config, and requires additional configuration, for example used for pushing indicators to the API + + +class CytomicParser(): + def __init__(self, attribute, config_object): + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.misp_event.add_attribute(**self.attribute) + + self.config_object = config_object + + if self.config_object: + self.token = self.get_token() + else: + sys.exit('Missing configuration') + + def get_token(self): + try: + scope = self.config_object['scope'] + grant_type = self.config_object['grant_type'] + username = self.config_object['username'] + password = self.config_object['password'] + token_url = self.config_object['token_url'] + clientid = self.config_object['clientid'] + clientsecret = self.config_object['clientsecret'] + + if scope and grant_type and username and password: + data = {'scope': scope, 'grant_type': grant_type, 'username': username, 'password': password} + + if token_url and clientid and clientsecret: + access_token_response = requests.post(token_url, data=data, verify=False, allow_redirects=False, auth=(clientid, clientsecret)) + tokens = json.loads(access_token_response.text) + if 'access_token' in tokens: + return tokens['access_token'] + else: + self.result = {'error': 'No token received.'} + return + else: + self.result = {'error': 'No token_url, clientid or clientsecret supplied.'} + return + else: + self.result = {'error': 'No scope, grant_type, username or password supplied.'} + return + except Exception: + self.result = {'error': 'Unable to connect to token_url.'} + return + + def get_results(self): + if hasattr(self, 'result'): + return self.result + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + def parse(self, searchkey): + + if self.token: + + endpoint_fileinformation = self.config_object['endpoint_fileinformation'] + endpoint_machines = self.config_object['endpoint_machines'] + endpoint_machines_client = self.config_object['endpoint_machines_client'] + query_machines = self.config_object['query_machines'] + query_machine_info = self.config_object['query_machine_info'] + + # Update endpoint URLs + query_endpoint_fileinformation = endpoint_fileinformation.format(md5=searchkey) + query_endpoint_machines = endpoint_machines.format(md5=searchkey) + + # API calls + api_call_headers = {'Authorization': 'Bearer ' + self.token} + result_query_endpoint_fileinformation = requests.get(query_endpoint_fileinformation, headers=api_call_headers, verify=False) + json_result_query_endpoint_fileinformation = json.loads(result_query_endpoint_fileinformation.text) + + if json_result_query_endpoint_fileinformation: + + cytomic_object = MISPObject('cytomic-orion-file') + + cytomic_object.add_attribute('fileName', type='text', value=json_result_query_endpoint_fileinformation['fileName']) + cytomic_object.add_attribute('fileSize', type='text', value=json_result_query_endpoint_fileinformation['fileSize']) + cytomic_object.add_attribute('last-seen', type='datetime', value=json_result_query_endpoint_fileinformation['lastSeen']) + cytomic_object.add_attribute('first-seen', type='datetime', value=json_result_query_endpoint_fileinformation['firstSeen']) + cytomic_object.add_attribute('classification', type='text', value=json_result_query_endpoint_fileinformation['classification']) + cytomic_object.add_attribute('classificationName', type='text', value=json_result_query_endpoint_fileinformation['classificationName']) + self.misp_event.add_object(**cytomic_object) + + result_query_endpoint_machines = requests.get(query_endpoint_machines, headers=api_call_headers, verify=False) + json_result_query_endpoint_machines = json.loads(result_query_endpoint_machines.text) + + if query_machines and json_result_query_endpoint_machines and len(json_result_query_endpoint_machines) > 0: + for machine in json_result_query_endpoint_machines: + + if query_machine_info and machine['muid']: + query_endpoint_machines_client = endpoint_machines_client.format(muid=machine['muid']) + result_endpoint_machines_client = requests.get(query_endpoint_machines_client, headers=api_call_headers, verify=False) + json_result_endpoint_machines_client = json.loads(result_endpoint_machines_client.text) + + if json_result_endpoint_machines_client: + + cytomic_machine_object = MISPObject('cytomic-orion-machine') + + clienttag = [{'name': json_result_endpoint_machines_client['clientName']}] + + cytomic_machine_object.add_attribute('machineName', type='target-machine', value=json_result_endpoint_machines_client['machineName'], Tag=clienttag) + cytomic_machine_object.add_attribute('machineMuid', type='text', value=machine['muid']) + cytomic_machine_object.add_attribute('clientName', type='target-org', value=json_result_endpoint_machines_client['clientName'], Tag=clienttag) + cytomic_machine_object.add_attribute('clientId', type='text', value=machine['clientId']) + cytomic_machine_object.add_attribute('machinePath', type='text', value=machine['lastPath']) + cytomic_machine_object.add_attribute('first-seen', type='datetime', value=machine['firstSeen']) + cytomic_machine_object.add_attribute('last-seen', type='datetime', value=machine['lastSeen']) + cytomic_machine_object.add_attribute('creationDate', type='datetime', value=json_result_endpoint_machines_client['creationDate']) + cytomic_machine_object.add_attribute('clientCreationDateUTC', type='datetime', value=json_result_endpoint_machines_client['clientCreationDateUTC']) + cytomic_machine_object.add_attribute('lastSeenUtc', type='datetime', value=json_result_endpoint_machines_client['lastSeenUtc']) + self.misp_event.add_object(**cytomic_machine_object) + else: + self.result = {'error': 'No (valid) token.'} + return + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('attribute'): + return {'error': 'Unsupported input.'} + + attribute = request['attribute'] + if not any(input_type == attribute['type'] for input_type in mispattributes['input']): + return {'error': 'Unsupported attributes type'} + + if not request.get('config'): + return {'error': 'Missing configuration'} + + config_object = { + 'clientid': request["config"].get("clientid"), + 'clientsecret': request["config"].get("clientsecret"), + 'scope': 'orion.api', + 'password': request["config"].get("password"), + 'username': request["config"].get("username"), + 'grant_type': 'password', + 'token_url': request["config"].get("token_url"), + 'endpoint_fileinformation': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/info'), + 'endpoint_machines': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/md5/{md5}/muids'), + 'endpoint_machines_client': '{api_url}{endpoint}'.format(api_url=request["config"].get("api_url"), endpoint='/forensics/muid/{muid}/info'), + 'query_machines': True, + 'query_machine_info': True + } + + cytomic_parser = CytomicParser(attribute, config_object) + cytomic_parser.parse(attribute['value']) + + return cytomic_parser.get_results() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/dbl_spamhaus.py b/misp_modules/modules/expansion/dbl_spamhaus.py new file mode 100644 index 0000000..0cccfaf --- /dev/null +++ b/misp_modules/modules/expansion/dbl_spamhaus.py @@ -0,0 +1,65 @@ +import json +import sys + +try: + import dns.resolver + resolver = dns.resolver.Resolver() + resolver.timeout = 0.2 + resolver.lifetime = 0.2 +except ImportError: + print("dnspython3 is missing, use 'pip install dnspython3' to install it.") + sys.exit(0) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['domain', 'domain|ip', 'hostname', 'hostname|port'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'Checks Spamhaus DBL for a domain name.', + 'module-type': ['expansion', 'hover']} +moduleconfig = [] + +dbl = 'dbl.spamhaus.org' +dbl_mapping = {'127.0.1.2': 'spam domain', + '127.0.1.4': 'phish domain', + '127.0.1.5': 'malware domain', + '127.0.1.6': 'botnet C&C domain', + '127.0.1.102': 'abused legit spam', + '127.0.1.103': 'abused spammed redirector domain', + '127.0.1.104': 'abused legit phish', + '127.0.1.105': 'abused legit malware', + '127.0.1.106': 'abused legit botnet C&C', + '127.0.1.255': 'IP queries prohibited!'} + + +def fetch_requested_value(request): + for attribute_type in mispattributes['input']: + if request.get(attribute_type): + return request[attribute_type].split('|')[0] + return None + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + requested_value = fetch_requested_value(request) + if requested_value is None: + misperrors['error'] = "Unsupported attributes type" + return misperrors + query = "{}.{}".format(requested_value, dbl) + try: + query_result = resolver.query(query, 'A')[0] + result = "{} - {}".format(requested_value, dbl_mapping[str(query_result)]) + except dns.resolver.NXDOMAIN as e: + result = e.msg + except Exception: + return {'error': 'Not able to reach dbl.spamhaus.org or something went wrong'} + return {'results': [{'types': mispattributes.get('output'), 'values': result}]} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/dns.py b/misp_modules/modules/expansion/dns.py index 41da8b2..c5af9d6 100755 --- a/misp_modules/modules/expansion/dns.py +++ b/misp_modules/modules/expansion/dns.py @@ -43,7 +43,7 @@ def handler(q=False): except dns.exception.Timeout: misperrors['error'] = "Timeout" return misperrors - except: + except Exception: misperrors['error'] = "DNS resolving error" return misperrors diff --git a/misp_modules/modules/expansion/docx_enrich.py b/misp_modules/modules/expansion/docx_enrich.py new file mode 100644 index 0000000..d5da3f8 --- /dev/null +++ b/misp_modules/modules/expansion/docx_enrich.py @@ -0,0 +1,61 @@ +import json +import binascii +import np +import docx +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.docx to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + docx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + doc_content = "" + doc_file = io.BytesIO(docx_array) + try: + doc = docx.Document(doc_file) + for para in doc.paragraphs: + print(para.text) + doc_content = doc_content + "\n" + para.text + tables = doc.tables + for table in tables: + for row in table.rows: + for cell in row.cells: + for para in cell.paragraphs: + print(para.text) + doc_content = doc_content + "\n" + para.text + print(doc_content) + return {'results': [{'types': ['freetext'], 'values': doc_content, 'comment': ".docx-to-text from file " + filename}, + {'types': ['text'], 'values': doc_content, 'comment': ".docx-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .docx. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/eql.py b/misp_modules/modules/expansion/eql.py new file mode 100644 index 0000000..46cc05e --- /dev/null +++ b/misp_modules/modules/expansion/eql.py @@ -0,0 +1,84 @@ +""" +Export module for converting MISP events into Endgame EQL queries +""" +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "92 COS DOM", + "description": "Generates EQL queries from events", + "module-type": ["expansion"] +} + +# Map of MISP fields => Endgame fields +fieldmap = { + "ip-src": "source_address", + "ip-dst": "destination_address", + "filename": "file_name" +} + +# Describe what events have what fields +event_types = { + "source_address": "network", + "destination_address": "network", + "file_name": "file" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + for supportedType in fieldmap.keys(): + if request.get(supportedType): + attrType = supportedType + + if attrType: + eqlType = fieldmap[attrType] + event_type = event_types[eqlType] + fullEql = "{} where {} == \"{}\"".format(event_type, eqlType, request[attrType]) + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + response = [] + response.append({'types': ['comment'], 'categories': ['External analysis'], 'values': fullEql, 'comment': "Event EQL queries"}) + return {'results': response} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ + return mispattributes + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + return moduleinfo diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py index 2518771..5d32ea8 100755 --- a/misp_modules/modules/expansion/farsight_passivedns.py +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -16,10 +16,9 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if (request.get('config')): - if (request['config'].get('apikey') is None): - misperrors['error'] = 'Farsight DNSDB apikey is missing' - return misperrors + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = 'Farsight DNSDB apikey is missing' + return misperrors client = DnsdbClient(server, request['config']['apikey']) if request.get('hostname'): res = lookup_name(client, request['hostname']) @@ -51,7 +50,7 @@ def lookup_name(client, name): for i in item.get('rdata'): # grab email field and replace first dot by @ to convert to an email address yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1)) - except QueryError as e: + except QueryError: pass try: @@ -59,7 +58,7 @@ def lookup_name(client, name): for item in res: if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: yield(item.get('rrname').rstrip('.')) - except QueryError as e: + except QueryError: pass @@ -68,7 +67,7 @@ def lookup_ip(client, ip): res = client.query_rdata_ip(ip) for item in res: yield(item['rrname'].rstrip('.')) - except QueryError as e: + except QueryError: pass diff --git a/misp_modules/modules/expansion/geoip_asn.py b/misp_modules/modules/expansion/geoip_asn.py new file mode 100644 index 0000000..95d7ee7 --- /dev/null +++ b/misp_modules/modules/expansion/geoip_asn.py @@ -0,0 +1,64 @@ +import json +import geoip2.database +import sys +import logging + +log = logging.getLogger('geoip_asn') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} +moduleconfig = ['local_geolite_db'] +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '0.1', 'author': 'GlennHD', + 'description': 'Query a local copy of the Maxmind Geolite ASN database (MMDB format)', + 'module-type': ['expansion', 'hover']} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('local_geolite_db'): + return {'error': 'Please specify the path of your local copy of the Maxmind Geolite ASN database'} + path_to_geolite = request['config']['local_geolite_db'] + + if request.get('ip-dst'): + toquery = request['ip-dst'] + elif request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('domain|ip'): + toquery = request['domain|ip'].split('|')[1] + else: + return False + + try: + reader = geoip2.database.Reader(path_to_geolite) + except FileNotFoundError: + return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + log.debug(toquery) + try: + answer = reader.asn(toquery) + stringmap = 'ASN=' + str(answer.autonomous_system_number) + ', AS Org=' + str(answer.autonomous_system_organization) + except Exception as e: + misperrors['error'] = f"GeoIP resolving error: {e}" + return misperrors + + r = {'results': [{'types': mispattributes['output'], 'values': stringmap}]} + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/geoip_city.py b/misp_modules/modules/expansion/geoip_city.py new file mode 100644 index 0000000..01c0627 --- /dev/null +++ b/misp_modules/modules/expansion/geoip_city.py @@ -0,0 +1,65 @@ +import json +import geoip2.database +import sys +import logging + +log = logging.getLogger('geoip_city') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} +moduleconfig = ['local_geolite_db'] +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '0.1', 'author': 'GlennHD', + 'description': 'Query a local copy of the Maxmind Geolite City database (MMDB format)', + 'module-type': ['expansion', 'hover']} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('local_geolite_db'): + return {'error': 'Please specify the path of your local copy of Maxminds Geolite database'} + path_to_geolite = request['config']['local_geolite_db'] + + if request.get('ip-dst'): + toquery = request['ip-dst'] + elif request.get('ip-src'): + toquery = request['ip-src'] + elif request.get('domain|ip'): + toquery = request['domain|ip'].split('|')[1] + else: + return False + + try: + reader = geoip2.database.Reader(path_to_geolite) + except FileNotFoundError: + return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + log.debug(toquery) + try: + answer = reader.city(toquery) + stringmap = 'Continent=' + str(answer.continent.name) + ', Country=' + str(answer.country.name) + ', Subdivision=' + str(answer.subdivisions.most_specific.name) + ', City=' + str(answer.city.name) + + except Exception as e: + misperrors['error'] = f"GeoIP resolving error: {e}" + return misperrors + + r = {'results': [{'types': mispattributes['output'], 'values': stringmap}]} + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/geoip_country.cfg b/misp_modules/modules/expansion/geoip_country.cfg deleted file mode 100644 index 95037e5..0000000 --- a/misp_modules/modules/expansion/geoip_country.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[GEOIP] -database = /opt/misp-modules/var/GeoIP.dat - diff --git a/misp_modules/modules/expansion/geoip_country.py b/misp_modules/modules/expansion/geoip_country.py index 31c1b6a..d28e570 100644 --- a/misp_modules/modules/expansion/geoip_country.py +++ b/misp_modules/modules/expansion/geoip_country.py @@ -1,9 +1,7 @@ import json -import pygeoip +import geoip2.database import sys -import os import logging -import configparser log = logging.getLogger('geoip_country') log.setLevel(logging.DEBUG) @@ -15,27 +13,22 @@ log.addHandler(ch) misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['freetext']} - +moduleconfig = ['local_geolite_db'] # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', - 'description': 'Query a local copy of Maxminds Geolite database', +moduleinfo = {'version': '0.2', 'author': 'Andreas Muehlemann', + 'description': 'Query a local copy of Maxminds Geolite database, updated for MMDB format', 'module-type': ['expansion', 'hover']} -try: - # get current db from http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz - config = configparser.ConfigParser() - config.read(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'geoip_country.cfg')) - gi = pygeoip.GeoIP(config.get('GEOIP', 'database')) - enabled = True -except: - enabled = False - def handler(q=False): if q is False: return False request = json.loads(q) + if not request.get('config') or not request['config'].get('local_geolite_db'): + return {'error': 'Please specify the path of your local copy of Maxminds Geolite database'} + path_to_geolite = request['config']['local_geolite_db'] + if request.get('ip-dst'): toquery = request['ip-dst'] elif request.get('ip-src'): @@ -45,15 +38,18 @@ def handler(q=False): else: return False - log.debug(toquery) - try: - answer = gi.country_code_by_addr(toquery) - except: - misperrors['error'] = "GeoIP resolving error" + reader = geoip2.database.Reader(path_to_geolite) + except FileNotFoundError: + return {'error': f'Unable to locate the GeoLite database you specified ({path_to_geolite}).'} + log.debug(toquery) + try: + answer = reader.country(toquery) + except Exception as e: + misperrors['error'] = f"GeoIP resolving error: {e}" return misperrors - r = {'results': [{'types': mispattributes['output'], 'values': [str(answer)]}]} + r = {'results': [{'types': mispattributes['output'], 'values': [answer.country.iso_code]}]} return r @@ -63,5 +59,5 @@ def introspection(): def version(): - # moduleinfo['config'] = moduleconfig + moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/google_search.py b/misp_modules/modules/expansion/google_search.py new file mode 100644 index 0000000..b7b4e7a --- /dev/null +++ b/misp_modules/modules/expansion/google_search.py @@ -0,0 +1,34 @@ +import json +try: + from google import google +except ImportError: + print("GoogleAPI not installed. Command : pip install git+https://github.com/abenassi/Google-Search-API") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['url'], 'output': ['text']} +moduleinfo = {'author': 'Oun & Gindt', 'module-type': ['hover'], + 'description': 'An expansion hover module to expand google search information about an URL'} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('url'): + return {'error': "Unsupported attributes type"} + num_page = 1 + res = "" + search_results = google.search(request['url'], num_page) + for i in range(3): + res += "("+str(i+1)+")" + '\t' + res += json.dumps(search_results[i].description, ensure_ascii=False) + res += '\n\n' + return {'results': [{'types': mispattributes['output'], 'values':res}]} + + +def introspection(): + return mispattributes + + +def version(): + return moduleinfo diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py new file mode 100644 index 0000000..dd54158 --- /dev/null +++ b/misp_modules/modules/expansion/greynoise.py @@ -0,0 +1,43 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab ', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']} +moduleconfig = ['user-agent'] # TODO take this into account in the code + +greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip' +default_user_agent = 'MISP-Module' + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + ip = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + data = {'ip': ip} + r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) + response = r.text + if response: + return {'results': [{'types': mispattributes['output'], 'values': response}]} + elif r.status_code == 404: # Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]} + else: # Real error + misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/hashdd.py b/misp_modules/modules/expansion/hashdd.py index beeaf8e..42fc854 100755 --- a/misp_modules/modules/expansion/hashdd.py +++ b/misp_modules/modules/expansion/hashdd.py @@ -2,8 +2,8 @@ import json import requests misperrors = {'error': 'Error'} -mispattributes = {'input': ['md5'], 'output': ['text']} -moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy', 'description': 'An expansion module to check hashes against hashdd.com including NSLR dataset.', 'module-type': ['hover']} +mispattributes = {'input': ['md5', 'sha1', 'sha256'], 'output': ['text']} +moduleinfo = {'version': '0.2', 'author': 'Alexandre Dulaunoy', 'description': 'An expansion module to check hashes against hashdd.com including NSLR dataset.', 'module-type': ['hover']} moduleconfig = [] hashddapi_url = 'https://api.hashdd.com/' @@ -11,19 +11,19 @@ hashddapi_url = 'https://api.hashdd.com/' def handler(q=False): if q is False: return False + v = None request = json.loads(q) - if not request.get('md5'): - misperrors['error'] = 'MD5 hash value is missing missing' + for input_type in mispattributes['input']: + if request.get(input_type): + v = request[input_type].upper() + break + if v is None: + misperrors['error'] = 'Hash value is missing.' return misperrors - v = request.get('md5').upper() - r = requests.post(hashddapi_url, data={'hash':v}) + r = requests.post(hashddapi_url, data={'hash': v}) if r.status_code == 200: state = json.loads(r.text) - if state: - if state.get(v): - summary = state[v]['known_level'] - else: - summary = 'Unknown hash' + summary = state[v]['known_level'] if state and state.get(v) else 'Unknown hash' else: misperrors['error'] = '{} API not accessible'.format(hashddapi_url) return misperrors['error'] diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py new file mode 100644 index 0000000..8db3fa7 --- /dev/null +++ b/misp_modules/modules/expansion/hibp.py @@ -0,0 +1,43 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']} # All mails as input +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API.', 'module-type': ['hover']} +moduleconfig = ['user-agent'] # TODO take this into account in the code + +haveibeenpwned_api_url = 'https://api.haveibeenpwned.com/api/v2/breachedaccount/' +default_user_agent = 'MISP-Module' # User agent (must be set, requiered by API)) + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + email = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent}) # Real request + if r.status_code == 200: # OK (record found) + breaches = json.loads(r.text) + if breaches: + return {'results': [{'types': mispattributes['output'], 'values': breaches}]} + elif r.status_code == 404: # Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]} + else: # Real error + misperrors['error'] = 'haveibeenpwned.com API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/intel471.py b/misp_modules/modules/expansion/intel471.py new file mode 100755 index 0000000..bf95b2e --- /dev/null +++ b/misp_modules/modules/expansion/intel471.py @@ -0,0 +1,61 @@ +import json +from pyintel471 import PyIntel471 + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['hostname', 'domain', 'url', 'ip-src', 'ip-dst', 'email-src', + 'email-dst', 'target-email', 'whois-registrant-email', + 'whois-registrant-name', 'md5', 'sha1', 'sha256'], 'output': ['freetext']} +moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', 'description': 'Module to access Intel 471', + 'module-type': ['hover', 'expansion']} +moduleconfig = ['email', 'authkey'] + + +def cleanup(response): + '''The entries have uids that will be recognised as hashes when they shouldn't''' + j = response.json() + if j['iocTotalCount'] == 0: + return 'Nothing has been found.' + for ioc in j['iocs']: + ioc.pop('uid') + if ioc['links']['actorTotalCount'] > 0: + for actor in ioc['links']['actors']: + actor.pop('uid') + if ioc['links']['reportTotalCount'] > 0: + for report in ioc['links']['reports']: + report.pop('uid') + return json.dumps(j, indent=2) + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + to_query = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + + if (request.get('config')): + if (request['config'].get('email') is None) or (request['config'].get('authkey') is None): + misperrors['error'] = 'Intel 471 authentication is missing' + return misperrors + + intel471 = PyIntel471(email=request['config'].get('email'), authkey=request['config'].get('authkey')) + ioc_filters = intel471.iocs_filters(ioc=to_query) + res = intel471.iocs(filters=ioc_filters) + to_return = cleanup(res) + + r = {'results': [{'types': mispattributes['output'], 'values': to_return}]} + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py index 22c9a70..3c6867c 100755 --- a/misp_modules/modules/expansion/ipasn.py +++ b/misp_modules/modules/expansion/ipasn.py @@ -1,46 +1,52 @@ # -*- coding: utf-8 -*- import json -from ipasn_redis import IPASN +from pyipasnhistory import IPASNHistory +from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} -moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', +mispattributes = {'input': ['ip-src', 'ip-dst'], 'format': 'misp_standard'} +moduleinfo = {'version': '0.2', 'author': 'Raphaël Vinot', 'description': 'Query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git)', 'module-type': ['expansion', 'hover']} -moduleconfig = ['host', 'port', 'db'] + +def parse_result(attribute, values): + event = MISPEvent() + initial_attribute = MISPAttribute() + initial_attribute.from_dict(**attribute) + event.add_attribute(**initial_attribute) + mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')} + print(values) + for last_seen, response in values['response'].items(): + asn = MISPObject('asn') + asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen}) + for feature, attribute_fields in mapping.items(): + attribute_type, object_relation = attribute_fields + asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]}) + asn.add_reference(initial_attribute.uuid, 'related-to') + event.add_object(**asn) + event = json.loads(event.to_json()) + return {key: event[key] for key in ('Attribute', 'Object')} def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip-dst'): - toquery = request['ip-dst'] + if request.get('attribute') and request['attribute'].get('type') in mispattributes['input']: + toquery = request['attribute']['value'] else: misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('host') and - request['config'].get('port') and - request['config'].get('db')): - misperrors['error'] = 'IP ASN history configuration is missing' - return misperrors - - ipasn = IPASN(host=request['config'].get('host'), - port=request['config'].get('port'), db=request['config'].get('db')) - - values = [] - for first_seen, last_seen, asn, block in ipasn.aggregate_history(toquery): - values.append('{} {} {} {}'.format(first_seen.decode(), last_seen.decode(), asn.decode(), block)) + ipasn = IPASNHistory() + values = ipasn.query(toquery) if not values: misperrors['error'] = 'Unable to find the history of this IP' return misperrors - return {'results': [{'types': mispattributes['output'], 'values': values}]} + return {'results': parse_result(request['attribute'], values)} def introspection(): @@ -48,5 +54,4 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/iprep.py b/misp_modules/modules/expansion/iprep.py index 6073052..558dbdd 100755 --- a/misp_modules/modules/expansion/iprep.py +++ b/misp_modules/modules/expansion/iprep.py @@ -45,7 +45,7 @@ def parse_iprep(ip, api): url = 'https://www.packetmail.net/iprep.php/%s' % ip try: data = requests.get(url, params={'apikey': api}).json() - except: + except Exception: return ['Error pulling data'], rep # print '%s' % data for name, val in data.items(): @@ -71,11 +71,11 @@ def parse_iprep(ip, api): misp_val = context full_text += '\n%s' % context misp_comment = 'IPRep Source %s: %s' % (name, val['last_seen']) - rep.append({'types': mispattributes['output'], 'categories':['External analysis'], 'values': misp_val, 'comment': misp_comment}) - except: + rep.append({'types': mispattributes['output'], 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment}) + except Exception: err.append('Error parsing source: %s' % name) - rep.append({'types': ['freetext'], 'values': full_text , 'comment': 'Free text import of IPRep'}) + rep.append({'types': ['freetext'], 'values': full_text, 'comment': 'Free text import of IPRep'}) return err, rep diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py new file mode 100644 index 0000000..1ace259 --- /dev/null +++ b/misp_modules/modules/expansion/joesandbox_query.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +import jbxapi +import json +from joe_parser import JoeParser + +misperrors = {'error': 'Error'} + +inputSource = ['link'] + +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', + 'description': 'Query Joe Sandbox API with a report URL to get the parsed data.', + 'module-type': ['expansion']} +moduleconfig = ['apiurl', 'apikey', 'import_pe', 'import_mitre_attack'] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + apiurl = request['config'].get('apiurl') or 'https://jbxcloud.joesecurity.org/api' + apikey = request['config'].get('apikey') + parser_config = { + "import_pe": request["config"].get('import_pe', "false") == "true", + "mitre_attack": request["config"].get('import_mitre_attack', "false") == "true", + } + + if not apikey: + return {'error': 'No API key provided'} + + url = request['attribute']['value'] + if "/submissions/" not in url: + return {'error': "The URL does not point to a Joe Sandbox analysis."} + + submission_id = url.split('/')[-1] # The URL has the format https://example.net/submissions/12345 + joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent='MISP joesandbox_query') + + try: + joe_info = joe.submission_info(submission_id) + except jbxapi.ApiError as e: + return {'error': str(e)} + + if joe_info["status"] != "finished": + return {'error': "The analysis has not finished yet."} + + if joe_info['most_relevant_analysis'] is None: + return {'error': "No analysis belongs to this submission."} + + analysis_webid = joe_info['most_relevant_analysis']['webid'] + + joe_parser = JoeParser(parser_config) + joe_data = json.loads(joe.analysis_download(analysis_webid, 'jsonfixed')[1]) + joe_parser.parse_data(joe_data['analysis']) + joe_parser.finalize_results() + + return {'results': joe_parser.results} + + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['input'] = inputSource + except NameError: + pass + modulesetup['format'] = 'misp_standard' + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/joesandbox_submit.py b/misp_modules/modules/expansion/joesandbox_submit.py new file mode 100644 index 0000000..39b140e --- /dev/null +++ b/misp_modules/modules/expansion/joesandbox_submit.py @@ -0,0 +1,140 @@ +import jbxapi +import base64 +import io +import json +import logging +import sys +import zipfile +import re + +from urllib.parse import urljoin + + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) +sh = logging.StreamHandler(sys.stdout) +sh.setLevel(logging.DEBUG) +fmt = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +sh.setFormatter(fmt) +log.addHandler(sh) + +moduleinfo = { + "version": "1.0", + "author": "Joe Security LLC", + "description": "Submit files and URLs to Joe Sandbox", + "module-type": ["expansion", "hover"] +} +moduleconfig = [ + "apiurl", + "apikey", + "accept-tac", + "report-cache", + "systems", +] + +mispattributes = { + "input": ["attachment", "malware-sample", "url", "domain"], + "output": ["link"], +} + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + apiurl = request["config"].get("apiurl") or "https://jbxcloud.joesecurity.org/api" + apikey = request["config"].get("apikey") + + # systems + systems = request["config"].get("systems") or "" + systems = [s.strip() for s in re.split(r"[\s,;]", systems) if s.strip()] + + try: + accept_tac = _parse_bool(request["config"].get("accept-tac"), "accept-tac") + report_cache = _parse_bool(request["config"].get("report-cache"), "report-cache") + except _ParseError as e: + return {"error": str(e)} + + params = { + "report-cache": report_cache, + "systems": systems, + } + + if not apikey: + return {"error": "No API key provided"} + + joe = jbxapi.JoeSandbox(apiurl=apiurl, apikey=apikey, user_agent="MISP joesandbox_submit", accept_tac=accept_tac) + + try: + is_url_submission = "url" in request or "domain" in request + + if is_url_submission: + url = request.get("url") or request.get("domain") + + log.info("Submitting URL: %s", url) + result = joe.submit_url(url, params=params) + else: + if "malware-sample" in request: + filename = request.get("malware-sample").split("|", 1)[0] + data = _decode_malware(request["data"], True) + elif "attachment" in request: + filename = request["attachment"] + data = _decode_malware(request["data"], False) + + data_fp = io.BytesIO(data) + log.info("Submitting sample: %s", filename) + result = joe.submit_sample((filename, data_fp), params=params) + + assert "submission_id" in result + except jbxapi.JoeException as e: + return {"error": str(e)} + + link_to_analysis = urljoin(apiurl, "../submissions/{}".format(result["submission_id"])) + + return { + "results": [{ + "types": "link", + "categories": "External analysis", + "values": link_to_analysis, + }] + } + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo["config"] = moduleconfig + return moduleinfo + + +def _decode_malware(data, is_encrypted): + data = base64.b64decode(data) + + if is_encrypted: + with zipfile.ZipFile(io.BytesIO(data)) as zipf: + data = zipf.read(zipf.namelist()[0], pwd=b"infected") + + return data + + +class _ParseError(Exception): + pass + + +def _parse_bool(value, name="bool"): + if value is None or value == "": + return None + + if value == "true": + return True + + if value == "false": + return False + + raise _ParseError("Cannot parse {}. Must be 'true' or 'false'".format(name)) diff --git a/misp_modules/modules/expansion/lastline_query.py b/misp_modules/modules/expansion/lastline_query.py new file mode 100644 index 0000000..4ce4e47 --- /dev/null +++ b/misp_modules/modules/expansion/lastline_query.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +""" +Module (type "expansion") to query a Lastline report from an analysis link. +""" +import json + +import lastline_api + + +misperrors = { + "error": "Error", +} + +mispattributes = { + "input": [ + "link", + ], + "output": ["text"], + "format": "misp_standard", +} + +moduleinfo = { + "version": "0.1", + "author": "Stefano Ortolani", + "description": "Get a Lastline report from an analysis link.", + "module-type": ["expansion"], +} + +moduleconfig = [ + "username", + "password", + "verify_ssl", +] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo["config"] = moduleconfig + return moduleinfo + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + # Parse the init parameters + try: + config = request["config"] + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) + analysis_link = request['attribute']['value'] + # The API url changes based on the analysis link host name + api_url = lastline_api.get_portal_url_from_task_link(analysis_link) + except Exception as e: + misperrors["error"] = "Error parsing configuration: {}".format(e) + return misperrors + + # Parse the call parameters + try: + task_uuid = lastline_api.get_uuid_from_task_link(analysis_link) + except (KeyError, ValueError) as e: + misperrors["error"] = "Error processing input parameters: {}".format(e) + return misperrors + + # Make the API calls + try: + api_client = lastline_api.PortalClient(api_url, auth_data, verify_ssl=config.get('verify_ssl', True).lower() in ("true")) + response = api_client.get_progress(task_uuid) + if response.get("completed") != 1: + raise ValueError("Analysis is not finished yet.") + + response = api_client.get_result(task_uuid) + if not response: + raise ValueError("Analysis report is empty.") + + except Exception as e: + misperrors["error"] = "Error issuing the API call: {}".format(e) + return misperrors + + # Parse and return + result_parser = lastline_api.LastlineResultBaseParser() + result_parser.parse(analysis_link, response) + + event = result_parser.misp_event + event_dictionary = json.loads(event.to_json()) + + return { + "results": { + key: event_dictionary[key] + for key in ('Attribute', 'Object', 'Tag') + if (key in event and event[key]) + } + } + + +if __name__ == "__main__": + """Test querying information from a Lastline analysis link.""" + import argparse + import configparser + + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config-file", dest="config_file") + parser.add_argument("-s", "--section-name", dest="section_name") + args = parser.parse_args() + c = configparser.ConfigParser() + c.read(args.config_file) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) + + j = json.dumps( + { + "config": a, + "attribute": { + "value": ( + "https://user.lastline.com/portal#/analyst/task/" + "1fcbcb8f7fb400100772d6a7b62f501b/overview" + ) + } + } + ) + print(json.dumps(handler(j), indent=4, sort_keys=True)) + + j = json.dumps( + { + "config": a, + "attribute": { + "value": ( + "https://user.lastline.com/portal#/analyst/task/" + "f3c0ae115d51001017ff8da768fa6049/overview" + ) + } + } + ) + print(json.dumps(handler(j), indent=4, sort_keys=True)) diff --git a/misp_modules/modules/expansion/lastline_submit.py b/misp_modules/modules/expansion/lastline_submit.py new file mode 100644 index 0000000..1572955 --- /dev/null +++ b/misp_modules/modules/expansion/lastline_submit.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +""" +Module (type "expansion") to submit files and URLs to Lastline for analysis. +""" +import base64 +import io +import json +import zipfile + +import lastline_api + + +misperrors = { + "error": "Error", +} + +mispattributes = { + "input": [ + "attachment", + "malware-sample", + "url", + ], + "output": [ + "link", + ], +} + +moduleinfo = { + "version": "0.1", + "author": "Stefano Ortolani", + "description": "Submit files and URLs to Lastline analyst", + "module-type": ["expansion", "hover"], +} + +moduleconfig = [ + "url", + "api_token", + "key", +] + + +DEFAULT_ZIP_PASSWORD = b"infected" + + +def __unzip(zipped_data, password=None): + data_file_object = io.BytesIO(zipped_data) + with zipfile.ZipFile(data_file_object) as zip_file: + sample_hashname = zip_file.namelist()[0] + data_zipped = zip_file.read(sample_hashname, password) + return data_zipped + + +def __str_to_bool(x): + return x in ("True", "true", True) + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo["config"] = moduleconfig + return moduleinfo + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + # Parse the init parameters + try: + config = request.get("config", {}) + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) + api_url = config.get("url", lastline_api.DEFAULT_LL_ANALYSIS_API_URL) + except Exception as e: + misperrors["error"] = "Error parsing configuration: {}".format(e) + return misperrors + + # Parse the call parameters + try: + call_args = {} + if "url" in request: + # URLs are text strings + api_method = lastline_api.AnalysisClient.submit_url + call_args["url"] = request.get("url") + else: + data = request.get("data") + # Malware samples are zip-encrypted and then base64 encoded + if "malware-sample" in request: + api_method = lastline_api.AnalysisClient.submit_file + call_args["file_data"] = __unzip(base64.b64decode(data), DEFAULT_ZIP_PASSWORD) + call_args["file_name"] = request.get("malware-sample").split("|", 1)[0] + call_args["password"] = DEFAULT_ZIP_PASSWORD + # Attachments are just base64 encoded + elif "attachment" in request: + api_method = lastline_api.AnalysisClient.submit_file + call_args["file_data"] = base64.b64decode(data) + call_args["file_name"] = request.get("attachment") + + else: + raise ValueError("Input parameters do not specify either an URL or a file") + + except Exception as e: + misperrors["error"] = "Error processing input parameters: {}".format(e) + return misperrors + + # Make the API call + try: + api_client = lastline_api.AnalysisClient(api_url, auth_data) + response = api_method(api_client, **call_args) + task_uuid = response.get("task_uuid") + if not task_uuid: + raise ValueError("Unable to process returned data") + if response.get("score") is not None: + tags = ["workflow:state='complete'"] + else: + tags = ["workflow:state='incomplete'"] + + except Exception as e: + misperrors["error"] = "Error issuing the API call: {}".format(e) + return misperrors + + # Assemble and return + analysis_link = lastline_api.get_task_link(task_uuid, analysis_url=api_url) + + return { + "results": [ + { + "types": "link", + "categories": ["External analysis"], + "values": analysis_link, + "tags": tags, + }, + ] + } + + +if __name__ == "__main__": + """Test submitting a test subject to the Lastline backend.""" + import argparse + import configparser + + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config-file", dest="config_file") + parser.add_argument("-s", "--section-name", dest="section_name") + args = parser.parse_args() + c = configparser.ConfigParser() + c.read(args.config_file) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) + + j = json.dumps( + { + "config": a, + "url": "https://www.google.exe.com", + } + ) + print(json.dumps(handler(j), indent=4, sort_keys=True)) + + with open("./tests/test_files/test.docx", "rb") as f: + data = f.read() + + j = json.dumps( + { + "config": a, + "data": base64.b64encode(data).decode("utf-8"), + "attachment": "test.docx", + } + ) + print(json.dumps(handler(j), indent=4, sort_keys=True)) diff --git a/misp_modules/modules/expansion/macaddress_io.py b/misp_modules/modules/expansion/macaddress_io.py new file mode 100644 index 0000000..72f950a --- /dev/null +++ b/misp_modules/modules/expansion/macaddress_io.py @@ -0,0 +1,123 @@ +import json + +from maclookup import ApiClient, exceptions + +misperrors = { + 'error': 'Error' +} + +mispattributes = { + 'input': ['mac-address'], +} + +moduleinfo = { + 'version': '1.0', + 'author': 'CodeLine OY - macaddress.io', + 'description': 'MISP hover module for macaddress.io', + 'module-type': ['hover'] +} + +moduleconfig = ['api_key'] + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + if request.get('mac-address'): + mac_address = request['mac-address'] + else: + return False + + if request.get('config') and request['config'].get('api_key'): + api_key = request['config'].get('api_key') + else: + misperrors['error'] = 'Authorization required' + return misperrors + + api_client = ApiClient(api_key) + + try: + response = api_client.get(mac_address) + + except exceptions.EmptyResponseException: + misperrors['error'] = 'Empty response' + return misperrors + + except exceptions.UnparsableResponseException: + misperrors['error'] = 'Unparsable response' + return misperrors + + except exceptions.ServerErrorException: + misperrors['error'] = 'Internal server error' + return misperrors + + except exceptions.UnknownOutputFormatException: + misperrors['error'] = 'Unknown output' + return misperrors + + except exceptions.AuthorizationRequiredException: + misperrors['error'] = 'Authorization required' + return misperrors + + except exceptions.AccessDeniedException: + misperrors['error'] = 'Access denied' + return misperrors + + except exceptions.InvalidMacOrOuiException: + misperrors['error'] = 'Invalid MAC or OUI' + return misperrors + + except exceptions.NotEnoughCreditsException: + misperrors['error'] = 'Not enough credits' + return misperrors + + except Exception: + misperrors['error'] = 'Unknown error' + return misperrors + + date_created = \ + response.block_details.date_created.strftime('%d %B %Y') if response.block_details.date_created else None + + date_updated = \ + response.block_details.date_updated.strftime('%d %B %Y') if response.block_details.date_updated else None + + results = { + 'results': + [{'types': ['text'], 'values': + { + # Mac address details + 'Valid MAC address': "True" if response.mac_address_details.is_valid else "False", + 'Transmission type': response.mac_address_details.transmission_type, + 'Administration type': response.mac_address_details.administration_type, + + # Vendor details + 'OUI': response.vendor_details.oui, + 'Vendor details are hidden': "True" if response.vendor_details.is_private else "False", + 'Company name': response.vendor_details.company_name, + 'Company\'s address': response.vendor_details.company_address, + 'County code': response.vendor_details.country_code, + + # Block details + 'Block found': "True" if response.block_details.block_found else "False", + 'The left border of the range': response.block_details.border_left, + 'The right border of the range': response.block_details.border_right, + 'The total number of MAC addresses in this range': response.block_details.block_size, + 'Assignment block size': response.block_details.assignment_block_size, + 'Date when the range was allocated': date_created, + 'Date when the range was last updated': date_updated + }}] + } + + return results + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/macvendors.py b/misp_modules/modules/expansion/macvendors.py new file mode 100644 index 0000000..bb98366 --- /dev/null +++ b/misp_modules/modules/expansion/macvendors.py @@ -0,0 +1,43 @@ +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['mac-address'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access Macvendors API.', 'module-type': ['hover']} +moduleconfig = ['user-agent'] + +macvendors_api_url = 'https://api.macvendors.com/' +default_user_agent = 'MISP-Module' + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + for input_type in mispattributes['input']: + if input_type in request: + mac = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type" + return misperrors + user_agent = request['config']['user-agent'] if request.get('config') and request['config'].get('user-agent') else default_user_agent + r = requests.get(macvendors_api_url + mac, headers={'user-agent': user_agent}) # Real request + if r.status_code == 200: # OK (record found) + response = r.text + if response: + return {'results': [{'types': mispattributes['output'], 'values': response}]} + elif r.status_code == 404: # Not found (not an error) + return {'results': [{'types': mispattributes['output'], 'values': 'Not found'}]} + else: # Real error + misperrors['error'] = 'MacVendors API not accessible (HTTP ' + str(r.status_code) + ')' + return misperrors['error'] + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/malwarebazaar.py b/misp_modules/modules/expansion/malwarebazaar.py new file mode 100644 index 0000000..4574b75 --- /dev/null +++ b/misp_modules/modules/expansion/malwarebazaar.py @@ -0,0 +1,52 @@ +import json +import requests +from pymisp import MISPEvent, MISPObject + +mispattributes = {'input': ['md5', 'sha1', 'sha256'], + 'format': 'misp_standard'} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'Query Malware Bazaar to get additional information about the input hash.', + 'module-type': ['expansion', 'hover']} +moduleconfig = [] + + +def parse_response(response): + mapping = {'file_name': {'type': 'filename', 'object_relation': 'filename'}, + 'file_size': {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}, + 'file_type_mime': {'type': 'mime-type', 'object_relation': 'mimetype'}, + 'md5_hash': {'type': 'md5', 'object_relation': 'md5'}, + 'sha1_hash': {'type': 'sha1', 'object_relation': 'sha1'}, + 'sha256_hash': {'type': 'sha256', 'object_relation': 'sha256'}, + 'ssdeep': {'type': 'ssdeep', 'object_relation': 'ssdeep'}} + misp_event = MISPEvent() + for data in response: + misp_object = MISPObject('file') + for feature, attribute in mapping.items(): + if feature in data: + misp_attribute = {'value': data[feature]} + misp_attribute.update(attribute) + misp_object.add_attribute(**misp_attribute) + misp_event.add_object(**misp_object) + return {'results': {'Object': [json.loads(misp_object.to_json()) for misp_object in misp_event.objects]}} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + attribute = request['attribute'] + url = 'https://mb-api.abuse.ch/api/v1/' + response = requests.post(url, data={'query': 'get_info', 'hash': attribute['value']}).json() + query_status = response['query_status'] + if query_status == 'ok': + return parse_response(response['data']) + return {'error': 'Hash not found on MALWAREbazzar' if query_status == 'hash_not_found' else f'Problem encountered during the query: {query_status}'} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/modules/expansion/module.py.skeleton b/misp_modules/modules/expansion/module.py.skeleton similarity index 100% rename from modules/expansion/module.py.skeleton rename to misp_modules/modules/expansion/module.py.skeleton diff --git a/misp_modules/modules/expansion/ocr_enrich.py b/misp_modules/modules/expansion/ocr_enrich.py new file mode 100644 index 0000000..cd6baca --- /dev/null +++ b/misp_modules/modules/expansion/ocr_enrich.py @@ -0,0 +1,50 @@ +import json +import binascii +import cv2 +import np +import pytesseract + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'OCR decoder', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + img_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + image = img_array + image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) + try: + decoded = pytesseract.image_to_string(image) + return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename}, + {'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file type. Only images are supported right now." + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/ods_enrich.py b/misp_modules/modules/expansion/ods_enrich.py new file mode 100644 index 0000000..b247c44 --- /dev/null +++ b/misp_modules/modules/expansion/ods_enrich.py @@ -0,0 +1,56 @@ +import json +import binascii +import np +import ezodf +import pandas_ods_reader +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.ods to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + ods_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + ods_content = "" + ods_file = io.BytesIO(ods_array) + doc = ezodf.opendoc(ods_file) + num_sheets = len(doc.sheets) + try: + for i in range(0, num_sheets): + ods = pandas_ods_reader.read_ods(ods_file, i, headers=False) + ods_content = ods_content + "\n" + ods.to_string(max_rows=None) + print(ods_content) + return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}, + {'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .ods. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/odt_enrich.py b/misp_modules/modules/expansion/odt_enrich.py new file mode 100644 index 0000000..c4513ae --- /dev/null +++ b/misp_modules/modules/expansion/odt_enrich.py @@ -0,0 +1,51 @@ +import json +import binascii +import np +from ODTReader.odtreader import odtToText +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.odt to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + odt_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + odt_content = "" + odt_file = io.BytesIO(odt_array) + try: + odt_content = odtToText(odt_file) + print(odt_content) + return {'results': [{'types': ['freetext'], 'values': odt_content, 'comment': ".odt-to-text from file " + filename}, + {'types': ['text'], 'values': odt_content, 'comment': ".odt-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .odt. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/onyphe.py b/misp_modules/modules/expansion/onyphe.py index 86abe7a..d8db477 100644 --- a/misp_modules/modules/expansion/onyphe.py +++ b/misp_modules/modules/expansion/onyphe.py @@ -1,4 +1,3 @@ -import json # -*- coding: utf-8 -*- import json @@ -9,7 +8,8 @@ except ImportError: misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domains'], 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']} +mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']} # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', 'description': 'Query on Onyphe', @@ -24,7 +24,7 @@ def handler(q=False): request = json.loads(q) - if not request.get('config') and not (request['config'].get('apikey')): + if not request.get('config') or not request['config'].get('apikey'): misperrors['error'] = 'Onyphe authentication is missing' return misperrors @@ -54,7 +54,7 @@ def handle_expansion(api, ip, misperrors): misperrors['error'] = result['message'] return misperrors - categories = list(set([item['@category'] for item in result['results']])) + # categories = list(set([item['@category'] for item in result['results']])) result_filtered = {"results": []} urls_pasties = [] @@ -65,16 +65,16 @@ def handle_expansion(api, ip, misperrors): for r in result['results']: if r['@category'] == 'pastries': - if r['@type'] == 'pastebin': + if r['source'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % r['key']) elif r['@category'] == 'synscan': asn_list.append(r['asn']) os_target = r['os'] if os_target != 'Unknown': os_list.append(r['os']) - elif r['@category'] == 'resolver' and r['@type'] =='reverse': + elif r['@category'] == 'resolver' and r['type'] == 'reverse': domains_resolver.append(r['reverse']) - elif r['@category'] == 'resolver' and r['@type'] =='forward': + elif r['@category'] == 'resolver' and r['type'] == 'forward': domains_forward.append(r['forward']) result_filtered['results'].append({'types': ['url'], 'values': urls_pasties, @@ -90,7 +90,7 @@ def handle_expansion(api, ip, misperrors): result_filtered['results'].append({'types': ['domain'], 'values': list(set(domains_resolver)), 'categories': ['Network activity'], - 'comment': 'resolver to %s' % ip }) + 'comment': 'resolver to %s' % ip}) result_filtered['results'].append({'types': ['domain'], 'values': list(set(domains_forward)), @@ -105,4 +105,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo diff --git a/misp_modules/modules/expansion/onyphe_full.py b/misp_modules/modules/expansion/onyphe_full.py index 7a05d12..3b1c554 100644 --- a/misp_modules/modules/expansion/onyphe_full.py +++ b/misp_modules/modules/expansion/onyphe_full.py @@ -1,4 +1,3 @@ -import json # -*- coding: utf-8 -*- import json @@ -10,7 +9,7 @@ except ImportError: misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'], - 'output': ['hostname', 'domain', 'ip-src', 'ip-dst','url']} + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url']} # possible module-types: 'expansion', 'hover' or both moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', @@ -26,7 +25,7 @@ def handler(q=False): request = json.loads(q) - if not request.get('config') and not (request['config'].get('apikey')): + if not request.get('config') or not request['config'].get('apikey'): misperrors['error'] = 'Onyphe authentication is missing' return misperrors @@ -38,10 +37,10 @@ def handler(q=False): ip = '' if request.get('ip-src'): ip = request['ip-src'] - return handle_ip(api ,ip, misperrors) + return handle_ip(api, ip, misperrors) elif request.get('ip-dst'): ip = request['ip-dst'] - return handle_ip(api,ip,misperrors) + return handle_ip(api, ip, misperrors) elif request.get('domain'): domain = request['domain'] return handle_domain(api, domain, misperrors) @@ -91,11 +90,11 @@ def handle_ip(api, ip, misperrors): r, status_ok = expand_syscan(api, ip, misperrors) if status_ok: - result_filtered['results'].extend(r) + result_filtered['results'].extend(r) else: - misperrors['error'] = "Error syscan result" + misperrors['error'] = "Error syscan result" - r, status_ok = expand_pastries(api,misperrors,ip=ip) + r, status_ok = expand_pastries(api, misperrors, ip=ip) if status_ok: result_filtered['results'].extend(r) @@ -185,11 +184,11 @@ def expand_syscan(api, ip, misperror): return r, status_ok -def expand_datascan(api, misperror,**kwargs): +def expand_datascan(api, misperror, **kwargs): status_ok = False r = [] - ip = '' - query ='' + # ip = '' + query = '' asn_list = [] geoloc = [] orgs = [] @@ -311,11 +310,11 @@ def expand_pastries(api, misperror, **kwargs): query = kwargs.get('domain') result = api.search_pastries('domain:%s' % query) - if result['status'] =='ok': + if result['status'] == 'ok': status_ok = True for item in result['results']: if item['@category'] == 'pastries': - if item['@type'] == 'pastebin': + if item['source'] == 'pastebin': urls_pasties.append('https://pastebin.com/raw/%s' % item['key']) if 'domain' in item: @@ -328,7 +327,7 @@ def expand_pastries(api, misperror, **kwargs): r.append({'types': ['url'], 'values': urls_pasties, 'categories': ['External analysis'], - 'comment':'URLs of pasties where %s has found' % query}) + 'comment': 'URLs of pasties where %s has found' % query}) r.append({'types': ['domain'], 'values': list(set(domains)), 'categories': ['Network activity'], 'comment': 'Domains found in pasties of Onyphe'}) @@ -340,7 +339,7 @@ def expand_pastries(api, misperror, **kwargs): return r, status_ok -def expand_threatlist(api, misperror,**kwargs): +def expand_threatlist(api, misperror, **kwargs): status_ok = False r = [] @@ -366,7 +365,8 @@ def expand_threatlist(api, misperror,**kwargs): 'comment': '%s is present in threatlist' % query }) - return r,status_ok + return r, status_ok + def introspection(): return mispattributes @@ -374,4 +374,4 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig - return moduleinfo \ No newline at end of file + return moduleinfo diff --git a/misp_modules/modules/expansion/otx.py b/misp_modules/modules/expansion/otx.py index 214e7f0..e586180 100755 --- a/misp_modules/modules/expansion/otx.py +++ b/misp_modules/modules/expansion/otx.py @@ -15,9 +15,10 @@ moduleinfo = {'version': '1', 'author': 'chrisdoman', # We're not actually using the API key yet moduleconfig = ["apikey"] + # Avoid adding windows update to enrichment etc. def isBlacklisted(value): - blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.' , 'time.windows.com'] + blacklist = ['0.0.0.0', '8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com'] for b in blacklist: if value in b: @@ -25,28 +26,31 @@ def isBlacklisted(value): return True + def valid_ip(ip): m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip) return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups())) + def findAll(data, keys): a = [] if isinstance(data, dict): - for key in data.keys(): + for key, value in data.items(): if key == keys: - a.append(data[key]) + a.append(value) else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) + if isinstance(value, (dict, list)): + a.extend(findAll(value, keys)) if isinstance(data, list): for i in data: - a += findAll(i, keys) - + a.extend(findAll(i, keys)) return a + def valid_email(email): return bool(re.search(r"[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?", email)) + def handler(q=False): if q is False: return False @@ -82,10 +86,10 @@ def handler(q=False): return r -def getHash(hash, key): +def getHash(_hash, key): ret = [] - req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + hash).text) + req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/file/analysis/" + _hash).text) for ip in findAll(req, "dst"): if not isBlacklisted(ip) and valid_ip(ip): @@ -100,19 +104,17 @@ def getHash(hash, key): def getIP(ip, key): ret = [] - req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text ) + req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/malware/" + ip + "?limit=1000").text) - for hash in findAll(req, "hash"): - ret.append({"types": ["sha256"], "values": [hash]}) + for _hash in findAll(req, "hash"): + ret.append({"types": ["sha256"], "values": [_hash]}) - - req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text ) + req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/ip/passive_dns/" + ip).text) for hostname in findAll(req, "hostname"): if not isBlacklisted(hostname): ret.append({"types": ["hostname"], "values": [hostname]}) - return ret @@ -120,23 +122,23 @@ def getDomain(domain, key): ret = [] - req = json.loads( requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text ) + req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/malware/" + domain + "?limit=1000").text) - for hash in findAll(req, "hash"): - ret.append({"types": ["sha256"], "values": [hash]}) + for _hash in findAll(req, "hash"): + ret.append({"types": ["sha256"], "values": [_hash]}) req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/domain/whois/" + domain).text) - for domain in findAll(req, "domain"): - ret.append({"types": ["hostname"], "values": [domain]}) + for _domain in findAll(req, "domain"): + ret.append({"types": ["hostname"], "values": [_domain]}) for email in findAll(req, "value"): if valid_email(email): - ret.append({"types": ["email"], "values": [domain]}) + ret.append({"types": ["email"], "values": [email]}) - for domain in findAll(req, "hostname"): - if "." in domain and not isBlacklisted(domain): - ret.append({"types": ["hostname"], "values": [domain]}) + for _domain in findAll(req, "hostname"): + if "." in _domain and not isBlacklisted(_domain): + ret.append({"types": ["hostname"], "values": [_domain]}) req = json.loads(requests.get("https://otx.alienvault.com/otxapi/indicator/hostname/passive_dns/" + domain).text) for ip in findAll(req, "address"): @@ -145,6 +147,7 @@ def getDomain(domain, key): return ret + def introspection(): return mispattributes diff --git a/misp_modules/modules/expansion/passivetotal.py b/misp_modules/modules/expansion/passivetotal.py index 5325d19..dfcedad 100755 --- a/misp_modules/modules/expansion/passivetotal.py +++ b/misp_modules/modules/expansion/passivetotal.py @@ -125,16 +125,14 @@ def process_ssl_details(instance, query): """Process details for a specific certificate.""" log.debug("SSL Details: starting") values = list() - _ = instance.get_ssl_certificate_details(query=query) - err = _has_error(_) + details = instance.get_ssl_certificate_details(query=query) + err = _has_error(details) if err: raise Exception("We hit an error, time to bail!") - - for key, value in _.items(): - if not value: - continue - values.append(value) - txt = [{'types': ['ssl-cert-attributes'], 'values': list(set(values))}] + if details.get('message') and details['message'].startswith('quota_exceeded'): + raise Exception("API quota exceeded.") + values = {value for value in details.values() if value} + txt = [{'types': ['ssl-cert-attributes'], 'values': list(values)}] log.debug("SSL Details: ending") return txt @@ -151,12 +149,13 @@ def process_ssl_history(instance, query): } hits = {'ip': list(), 'sha1': list(), 'domain': list()} - _ = instance.get_ssl_certificate_history(query=query) - err = _has_error(_) + history = instance.get_ssl_certificate_history(query=query) + err = _has_error(history) if err: raise Exception("We hit an error, time to bail!") - - for item in _.get('results', []): + if history.get('message') and history['message'].startswith('quota_exceeded'): + raise Exception("API quota exceeded.") + for item in history.get('results', []): hits['ip'] += item.get('ipAddresses', []) hits['sha1'].append(item['sha1']) hits['domain'] += item.get('domains', []) @@ -175,21 +174,22 @@ def process_whois_details(instance, query): """Process the detail from the WHOIS record.""" log.debug("WHOIS Details: starting") tmp = list() - _ = instance.get_whois_details(query=query, compact_record=True) - err = _has_error(_) + details = instance.get_whois_details(query=query, compact_record=True) + err = _has_error(details) if err: raise Exception("We hit an error, time to bail!") - - if _.get('contactEmail', None): - tmp.append({'types': ['whois-registrant-email'], 'values': [_.get('contactEmail')]}) - phones = _['compact']['telephone']['raw'] + if details.get('message') and details['message'].startswith('quota_exceeded'): + raise Exception("API quota exceeded.") + if details.get('contactEmail', None): + tmp.append({'types': ['whois-registrant-email'], 'values': [details.get('contactEmail')]}) + phones = details['compact']['telephone']['raw'] tmp.append({'types': ['whois-registrant-phone'], 'values': phones}) - names = _['compact']['name']['raw'] + names = details['compact']['name']['raw'] tmp.append({'types': ['whois-registrant-name'], 'values': names}) - if _.get('registrar', None): - tmp.append({'types': ['whois-registrar'], 'values': [_.get('registrar')]}) - if _.get('registered', None): - tmp.append({'types': ['whois-creation-date'], 'values': [_.get('registered')]}) + if details.get('registrar', None): + tmp.append({'types': ['whois-registrar'], 'values': [details.get('registrar')]}) + if details.get('registered', None): + tmp.append({'types': ['whois-creation-date'], 'values': [details.get('registered')]}) log.debug("WHOIS Details: ending") return tmp @@ -206,12 +206,13 @@ def process_whois_search(instance, query, qtype): field_type = 'name' domains = list() - _ = instance.search_whois_by_field(field=field_type, query=query) - err = _has_error(_) + search = instance.search_whois_by_field(field=field_type, query=query) + err = _has_error(search) if err: raise Exception("We hit an error, time to bail!") - - for item in _.get('results', []): + if search.get('message') and search['message'].startswith('quota_exceeded'): + raise Exception("API quota exceeded.") + for item in search.get('results', []): domain = item.get('domain', None) if not domain: continue @@ -227,15 +228,16 @@ def process_passive_dns(instance, query): """Process passive DNS data.""" log.debug("Passive DNS: starting") tmp = list() - _ = instance.get_unique_resolutions(query=query) - err = _has_error(_) + pdns = instance.get_unique_resolutions(query=query) + err = _has_error(pdns) if err: raise Exception("We hit an error, time to bail!") - + if pdns.get('message') and pdns['message'].startswith('quota_exceeded'): + raise Exception("API quota exceeded.") if is_ip(query): - tmp = [{'types': ['domain', 'hostname'], 'values': _.get('results', [])}] + tmp = [{'types': ['domain', 'hostname'], 'values': pdns.get('results', [])}] else: - tmp = [{'types': ['ip-src', 'ip-dst'], 'values': _.get('results', [])}] + tmp = [{'types': ['ip-src', 'ip-dst'], 'values': pdns.get('results', [])}] log.debug("Passive DNS: ending") return tmp @@ -245,12 +247,13 @@ def process_osint(instance, query): """Process OSINT links.""" log.debug("OSINT: starting") urls = list() - _ = instance.get_osint(query=query) - err = _has_error(_) + osint = instance.get_osint(query=query) + err = _has_error(osint) if err: raise Exception("We hit an error, time to bail!") - - for item in _.get('results', []): + if osint.get('message') and osint['message'].startswith('quota_exceeded'): + raise Exception("API quota exceeded.") + for item in osint.get('results', []): urls.append(item['sourceUrl']) tmp = [{'types': ['link'], 'values': urls}] @@ -263,12 +266,13 @@ def process_malware(instance, query): """Process malware samples.""" log.debug("Malware: starting") content = {'hashes': list(), 'urls': list()} - _ = instance.get_malware(query=query) - err = _has_error(_) + malware = instance.get_malware(query=query) + err = _has_error(malware) if err: raise Exception("We hit an error, time to bail!") - - for item in _.get('results', []): + if malware.get('message') and malware['message'].startswith('quota_exceeded'): + raise Exception("API quota exceeded.") + for item in malware.get('results', []): content['hashes'].append(item['sample']) content['urls'].append(item['sourceUrl']) @@ -331,7 +335,8 @@ def handler(q=False): output['results'] += results else: log.error("Unsupported query pattern issued.") - except: + except Exception as e: + misperrors['error'] = e.__str__() return misperrors return output diff --git a/misp_modules/modules/expansion/pdf_enrich.py b/misp_modules/modules/expansion/pdf_enrich.py new file mode 100644 index 0000000..ef85fde --- /dev/null +++ b/misp_modules/modules/expansion/pdf_enrich.py @@ -0,0 +1,48 @@ +import json +import binascii +import np +import pdftotext +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'PDF to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + pdf_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + pdf_file = io.BytesIO(pdf_array) + try: + pdf_content = "\n\n".join(pdftotext.PDF(pdf_file)) + return {'results': [{'types': ['freetext'], 'values': pdf_content, 'comment': "PDF-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as PDF. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/pptx_enrich.py b/misp_modules/modules/expansion/pptx_enrich.py new file mode 100644 index 0000000..816e439 --- /dev/null +++ b/misp_modules/modules/expansion/pptx_enrich.py @@ -0,0 +1,55 @@ +import json +import binascii +import np +from pptx import Presentation +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.pptx to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + pptx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + ppt_content = "" + ppt_file = io.BytesIO(pptx_array) + try: + ppt = Presentation(ppt_file) + for slide in ppt.slides: + for shape in slide.shapes: + if hasattr(shape, "text"): + print(shape.text) + ppt_content = ppt_content + "\n" + shape.text + return {'results': [{'types': ['freetext'], 'values': ppt_content, 'comment': ".pptx-to-text from file " + filename}, + {'types': ['text'], 'values': ppt_content, 'comment': ".pptx-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .pptx. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/qrcode.py b/misp_modules/modules/expansion/qrcode.py new file mode 100644 index 0000000..9a62827 --- /dev/null +++ b/misp_modules/modules/expansion/qrcode.py @@ -0,0 +1,89 @@ +import json +from pyzbar import pyzbar +import cv2 +import re +import binascii +import np + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['url', 'btc']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'QR code decoder', + 'module-type': ['expansion', 'hover']} + +debug = True +debug_prefix = "[DEBUG] QR Code module: " +# format example: bitcoin:1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh?amount=0.15424 +# format example: http://example.com +cryptocurrencies = ['bitcoin'] +schemas = ['http://', 'https://', 'ftp://'] +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + img_array = np.fromstring(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + print(e) + return misperrors + image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) + if q: + barcodes = pyzbar.decode(image) + for item in barcodes: + try: + result = item.data.decode() + except Exception as e: + print(e) + return + if debug: + print(debug_prefix + result) + for item in cryptocurrencies: + if item in result: + try: + currency, address, extra = re.split(r'\:|\?', result) + except Exception as e: + print(e) + if currency in cryptocurrencies: + try: + amount = re.split('=', extra)[1] + if debug: + print(debug_prefix + address) + print(debug_prefix + amount) + return {'results': [{'types': ['btc'], 'values': address, 'comment': "BTC: " + amount + " from file " + filename}]} + except Exception as e: + print(e) + else: + print(address) + for item in schemas: + if item in result: + try: + url = result + if debug: + print(debug_prefix + url) + return {'results': [{'types': ['url'], 'values': url, 'comment': "from QR code of file " + filename}]} + except Exception as e: + print(e) + else: + try: + return {'results': [{'types': ['text'], 'values': result, 'comment': "from QR code of file " + filename}]} + except Exception as e: + print(e) + misperrors['error'] = "Couldn't decode QR code in attachment." + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/ransomcoindb.py b/misp_modules/modules/expansion/ransomcoindb.py new file mode 100644 index 0000000..2b9b566 --- /dev/null +++ b/misp_modules/modules/expansion/ransomcoindb.py @@ -0,0 +1,66 @@ +import json +from ._ransomcoindb import ransomcoindb +from pymisp import MISPObject + +copyright = """ + Copyright 2019 (C) by Aaron Kaplan , all rights reserved. + This file is part of the ransomwarecoindDB project and licensed under the AGPL 3.0 license +""" + +__version__ = 0.1 + + +debug = False + +misperrors = {'error': 'Error'} +# mispattributes = {'input': ['sha1', 'sha256', 'md5', 'btc', 'xmr', 'dash' ], 'output': ['btc', 'sha1', 'sha256', 'md5', 'freetext']} +mispattributes = {'input': ['sha1', 'sha256', 'md5', 'btc'], 'output': ['btc', 'sha1', 'sha256', 'md5', 'freetext'], 'format': 'misp_standard'} +moduleinfo = {'version': __version__, 'author': 'Aaron Kaplan', 'description': 'Module to access the ransomcoinDB (see https://ransomcoindb.concinnity-risks.com)', 'module-type': ['expansion', 'hover']} +moduleconfig = ['api-key'] + + +def handler(q=False): + """ the main handler function which gets a JSON dict as input and returns a results dict """ + + if q is False: + return False + + q = json.loads(q) + if "config" not in q or "api-key" not in q["config"]: + return {"error": "Ransomcoindb API key is missing"} + api_key = q["config"]["api-key"] + r = {"results": []} + + """ the "q" query coming in should look something like this: + {'config': {'api-key': ''}, + 'md5': 'md5 or sha1 or sha256 or btc', + 'module': 'ransomcoindb', + 'persistent': 1} + """ + attribute = q['attribute'] + answer = ransomcoindb.get_data_by('BTC', attribute['type'], attribute['value'], api_key) + """ The results data type should be: + r = { 'results': [ {'types': 'md5', 'values': [ a list of all md5s or all binaries related to this btc address ] } ] } + """ + if attribute['type'] in ['md5', 'sha1', 'sha256']: + r['results'].append({'types': 'btc', 'values': [a['btc'] for a in answer]}) + elif attribute['type'] == 'btc': + # better: create a MISP object + files = [] + for a in answer: + obj = MISPObject('file') + obj.add_attribute('md5', a['md5']) + obj.add_attribute('sha1', a['sha1']) + obj.add_attribute('sha256', a['sha256']) + files.append(obj) + r['results'] = {'Object': [json.loads(f.to_json()) for f in files]} + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index da8c5fb..4d7bba5 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -1,12 +1,12 @@ import json -import datetime +import sys try: import dns.resolver resolver = dns.resolver.Resolver() resolver.timeout = 0.2 resolver.lifetime = 0.2 -except: +except ImportError: print("dnspython3 is missing, use 'pip install dnspython3' to install it.") sys.exit(0) @@ -18,64 +18,65 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', moduleconfig = [] rbls = { - 'spam.spamrats.com': 'http://www.spamrats.com', - 'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/', - 'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'web.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net', - 'virus.rbl.jp': 'http://www.rbl.jp', - 'dul.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/', - 'psbl.surriel.com': 'http://psbl.surriel.com', - 'misc.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'httpbl.abuse.ch': 'http://dnsbl.abuse.ch', - 'combined.njabl.org': 'http://combined.njabl.org', - 'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'korea.services.net': 'http://korea.services.net', - 'drone.abuse.ch': 'http://dnsbl.abuse.ch', - 'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org', - 'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', - 'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request', - 'bl.spamcannibal.org': 'http://www.spamcannibal.org', - 'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/', - 'zen.spamhaus.org': 'http://www.spamhaus.org/zen/', - 'rbl.suresupport.com': 'http://suresupport.com/postmaster', - 'db.wpbl.info': 'http://www.wpbl.info', - 'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/', - 'http.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index', - 'rbl.interserver.net': 'http://rbl.interserver.net', - 'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/', - 'dnsbl.sorbs.net': 'http://www.sorbs.net', - 'virbl.bit.nl': 'http://virbl.bit.nl', - 'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/', - 'socks.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'short.rbl.jp': 'http://www.rbl.jp', - 'dnsbl.dronebl.org': 'http://www.dronebl.org', - 'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp', - 'dyna.spamrats.com': 'http://www.spamrats.com', - 'spamrbl.imp.ch': 'http://antispam.imp.ch', - 'spam.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'wormrbl.imp.ch': 'http://antispam.imp.ch', - 'query.senderbase.org': 'http://www.senderbase.org/about', - 'opm.tornevall.org': 'http://dnsbl.tornevall.org', - 'netblock.pedantic.org': 'http://pedantic.org', - 'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33', - 'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', - 'multi.surbl.org': 'http://www.surbl.org', - 'noptr.spamrats.com': 'http://www.spamrats.com', - 'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en', - 'bl.spamcop.net': 'http://bl.spamcop.net', - 'cbl.abuseat.org': 'http://cbl.abuseat.org', - 'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php', - 'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'dnsbl.njabl.org': 'http://dnsbl.njabl.org', - 'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl', - 'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html' + 'spam.spamrats.com': 'http://www.spamrats.com', + 'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/', + 'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', + 'web.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net', + 'virus.rbl.jp': 'http://www.rbl.jp', + 'dul.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/', + 'psbl.surriel.com': 'http://psbl.surriel.com', + 'misc.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'httpbl.abuse.ch': 'http://dnsbl.abuse.ch', + 'combined.njabl.org': 'http://combined.njabl.org', + 'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'korea.services.net': 'http://korea.services.net', + 'drone.abuse.ch': 'http://dnsbl.abuse.ch', + 'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org', + 'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', + 'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request', + 'bl.spamcannibal.org': 'http://www.spamcannibal.org', + 'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/', + 'zen.spamhaus.org': 'http://www.spamhaus.org/zen/', + 'rbl.suresupport.com': 'http://suresupport.com/postmaster', + 'db.wpbl.info': 'http://www.wpbl.info', + 'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/', + 'http.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index', + 'rbl.interserver.net': 'http://rbl.interserver.net', + 'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/', + 'dnsbl.sorbs.net': 'http://www.sorbs.net', + 'virbl.bit.nl': 'http://virbl.bit.nl', + 'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/', + 'socks.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'short.rbl.jp': 'http://www.rbl.jp', + 'dnsbl.dronebl.org': 'http://www.dronebl.org', + 'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', + 'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp', + 'dyna.spamrats.com': 'http://www.spamrats.com', + 'spamrbl.imp.ch': 'http://antispam.imp.ch', + 'spam.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'wormrbl.imp.ch': 'http://antispam.imp.ch', + 'query.senderbase.org': 'http://www.senderbase.org/about', + 'opm.tornevall.org': 'http://dnsbl.tornevall.org', + 'netblock.pedantic.org': 'http://pedantic.org', + 'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33', + 'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', + 'multi.surbl.org': 'http://www.surbl.org', + 'noptr.spamrats.com': 'http://www.spamrats.com', + 'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en', + 'bl.spamcop.net': 'http://bl.spamcop.net', + 'cbl.abuseat.org': 'http://cbl.abuseat.org', + 'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php', + 'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net', + 'dnsbl.njabl.org': 'http://dnsbl.njabl.org', + 'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', + 'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl', + 'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html' } + def handler(q=False): if q is False: return False @@ -87,26 +88,27 @@ def handler(q=False): else: misperrors['error'] = "Unsupported attributes type" return misperrors - listed = [] - info = [] + listeds = [] + infos = [] + ipRev = '.'.join(ip.split('.')[::-1]) for rbl in rbls: - ipRev = '.'.join(ip.split('.')[::-1]) query = '{}.{}'.format(ipRev, rbl) try: - txt = resolver.query(query,'TXT') - listed.append(query) - info.append(str(txt[0])) - except: + txt = resolver.query(query, 'TXT') + listeds.append(query) + infos.append([str(t) for t in txt]) + except Exception: continue - result = {} - for l, i in zip(listed, info): - result[l] = i - r = {'results': [{'types': mispattributes.get('output'), 'values': json.dumps(result)}]} - return r + result = "\n".join([f"{listed}: {' - '.join(info)}" for listed, info in zip(listeds, infos)]) + if not result: + return {'error': 'No data found by querying known RBLs'} + return {'results': [{'types': mispattributes.get('output'), 'values': result}]} + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/reversedns.py b/misp_modules/modules/expansion/reversedns.py index 46fe94a..3b945a7 100644 --- a/misp_modules/modules/expansion/reversedns.py +++ b/misp_modules/modules/expansion/reversedns.py @@ -1,5 +1,5 @@ import json -import dns.reversename, dns.resolver +from dns import reversename, resolver, exception misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst', 'domain|ip'], 'output': ['hostname']} @@ -12,6 +12,7 @@ moduleinfo = {'version': '0.1', 'author': 'Andreas Muehlemann', # config fields that your code expects from the site admin moduleconfig = ['nameserver'] + def handler(q=False): if q is False: return False @@ -26,9 +27,9 @@ def handler(q=False): return False # reverse lookup for ip - revname = dns.reversename.from_address(toquery) + revname = reversename.from_address(toquery) - r = dns.resolver.Resolver() + r = resolver.Resolver() r.timeout = 2 r.lifetime = 2 @@ -42,13 +43,13 @@ def handler(q=False): try: answer = r.query(revname, 'PTR') - except dns.resolver.NXDOMAIN: + except resolver.NXDOMAIN: misperrors['error'] = "NXDOMAIN" return misperrors - except dns.exception.Timeout: + except exception.Timeout: misperrors['error'] = "Timeout" return misperrors - except: + except Exception: misperrors['error'] = "DNS resolving error" return misperrors @@ -56,9 +57,11 @@ def handler(q=False): 'values':[str(answer[0])]}]} return r + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/securitytrails.py b/misp_modules/modules/expansion/securitytrails.py new file mode 100644 index 0000000..f5750e1 --- /dev/null +++ b/misp_modules/modules/expansion/securitytrails.py @@ -0,0 +1,561 @@ +import json +import logging +import sys +import time + +from dnstrails import APIError +from dnstrails import DnsTrails + +log = logging.getLogger('dnstrails') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +misperrors = {'error': 'Error'} +mispattributes = { + 'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'dns-soa-email', + 'whois-registrant-email', 'whois-registrant-phone', + 'whois-registrant-name', + 'whois-registrar', 'whois-creation-date', 'domain'] +} + +moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven', + 'description': 'Query on securitytrails.com', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ['apikey'] + + +def handler(q=False): + if q: + + request = json.loads(q) + + if not request.get('config') or not (request['config'].get('apikey')): + misperrors['error'] = 'SecurityTrails authentication is missing' + return misperrors + + api = DnsTrails(request['config'].get('apikey')) + + if not api: + misperrors['error'] = 'SecurityTrails Error instance api' + return misperrors + if request.get('ip-src'): + ip = request['ip-src'] + return handle_ip(api, ip, misperrors) + elif request.get('ip-dst'): + ip = request['ip-dst'] + return handle_ip(api, ip, misperrors) + elif request.get('domain'): + domain = request['domain'] + return handle_domain(api, domain, misperrors) + elif request.get('hostname'): + hostname = request['hostname'] + return handle_domain(api, hostname, misperrors) + else: + misperrors['error'] = "Unsupported attributes types" + return misperrors + else: + return False + + +def handle_domain(api, domain, misperrors): + result_filtered = {"results": []} + + r, status_ok = expand_domain_info(api, misperrors, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error DNS result' + return misperrors + + time.sleep(1) + r, status_ok = expand_subdomains(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error subdomains result' + return misperrors + + time.sleep(1) + r, status_ok = expand_whois(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + + time.sleep(1) + r, status_ok = expand_history_ipv4_ipv6(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error history ipv4' + return misperrors + + time.sleep(1) + + r, status_ok = expand_history_dns(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error in expand History DNS' + return misperrors + + r, status_ok = expand_history_whois(api, domain) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] = misperrors['error'] + ' Error in expand History Whois' + return misperrors + + return result_filtered + + +def handle_ip(api, ip, misperrors): + result_filtered = {"results": []} + + r, status_ok = expand_searching_domain(api, ip) + + if status_ok: + if r: + result_filtered['results'].extend(r) + else: + misperrors['error'] += ' Error in expand searching domain' + return misperrors + + return result_filtered + + +def expand_domain_info(api, misperror, domain): + r = [] + status_ok = False + ns_servers = [] + list_ipv4 = [] + list_ipv6 = [] + servers_mx = [] + soa_hostnames = [] + + try: + results = api.domain(domain) + except APIError as e: + misperrors['error'] = e.value + return [], False + + if results: + status_ok = True + if 'current_dns' in results: + if 'values' in results['current_dns']['ns']: + ns_servers = [ns_entry['nameserver'] for ns_entry in + results['current_dns']['ns']['values'] + if 'nameserver' in ns_entry] + if 'values' in results['current_dns']['a']: + list_ipv4 = [a_entry['ip'] for a_entry in + results['current_dns']['a']['values'] if + 'ip' in a_entry] + + if 'values' in results['current_dns']['aaaa']: + list_ipv6 = [ipv6_entry['ipv6'] for ipv6_entry in + results['current_dns']['aaaa']['values'] if + 'ipv6' in ipv6_entry] + + if 'values' in results['current_dns']['mx']: + servers_mx = [mx_entry['hostname'] for mx_entry in + results['current_dns']['mx']['values'] if + 'hostname' in mx_entry] + if 'values' in results['current_dns']['soa']: + soa_hostnames = [soa_entry['email'] for soa_entry in + results['current_dns']['soa']['values'] if + 'email' in soa_entry] + + if ns_servers: + r.append({'types': ['domain'], + 'values': ns_servers, + 'categories': ['Network activity'], + 'comment': 'List of name servers of %s first seen %s ' % + (domain, + results['current_dns']['ns']['first_seen']) + }) + + if list_ipv4: + r.append({'types': ['domain|ip'], + 'values': ['%s|%s' % (domain, ipv4) for ipv4 in + list_ipv4], + 'categories': ['Network activity'], + + 'comment': ' List ipv4 of %s first seen %s' % + (domain, + results['current_dns']['a']['first_seen']) + + }) + if list_ipv6: + r.append({'types': ['domain|ip'], + 'values': ['%s|%s' % (domain, ipv6) for ipv6 in + list_ipv6], + 'categories': ['Network activity'], + 'comment': ' List ipv6 of %s first seen %s' % + (domain, + results['current_dns']['aaaa']['first_seen']) + + }) + + if servers_mx: + r.append({'types': ['domain'], + 'values': servers_mx, + 'categories': ['Network activity'], + 'comment': ' List mx of %s first seen %s' % + (domain, + results['current_dns']['mx']['first_seen']) + + }) + if soa_hostnames: + r.append({'types': ['domain'], + 'values': soa_hostnames, + 'categories': ['Network activity'], + 'comment': ' List soa of %s first seen %s' % + (domain, + results['current_dns']['soa']['first_seen']) + }) + + return r, status_ok + + +def expand_subdomains(api, domain): + r = [] + status_ok = False + + try: + results = api.subdomains(domain) + + if results: + status_ok = True + if 'subdomains' in results: + r.append({ + 'types': ['domain'], + 'values': ['%s.%s' % (sub, domain) + for sub in results['subdomains']], + 'categories': ['Network activity'], + 'comment': 'subdomains of %s' % domain + } + + ) + except APIError as e: + misperrors['error'] = e.value + return [], False + + return r, status_ok + + +def expand_whois(api, domain): + r = [] + status_ok = False + + try: + results = api.whois(domain) + + if results: + status_ok = True + item_registrant = __select_registrant_item(results) + if item_registrant: + + if 'email' in item_registrant[0]: + r.append( + { + 'types': ['whois-registrant-email'], + 'values': [item_registrant[0]['email']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'telephone' in item_registrant[0]: + r.append( + { + 'types': ['whois-registrant-phone'], + 'values': [item_registrant[0]['telephone']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'name' in item_registrant[0]: + r.append( + { + 'types': ['whois-registrant-name'], + 'values': [item_registrant[0]['name']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'registrarName' in item_registrant[0]: + r.append( + { + 'types': ['whois-registrar'], + 'values': [item_registrant[0]['registrarName']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + if 'createdDate' in item_registrant[0]: + r.append( + { + 'types': ['whois-creation-date'], + 'values': [item_registrant[0]['createdDate']], + 'categories': ['Attribution'], + 'comment': 'Whois information of %s by securitytrails' + % domain + } + ) + + except APIError as e: + misperrors['error'] = e.value + return [], False + + return r, status_ok + + +def expand_history_ipv4_ipv6(api, domain): + r = [] + status_ok = False + + try: + results = api.history_dns_ipv4(domain) + + if results: + status_ok = True + r.extend(__history_ip(results, domain)) + + time.sleep(1) + results = api.history_dns_aaaa(domain) + + if results: + status_ok = True + r.extend(__history_ip(results, domain, type_ip='ipv6')) + + except APIError as e: + misperrors['error'] = e.value + return [], False + + return r, status_ok + + +def expand_history_dns(api, domain): + r = [] + status_ok = False + + try: + + results = api.history_dns_ns(domain) + if results: + r.extend(__history_dns(results, domain, 'nameserver', 'ns')) + + time.sleep(1) + + results = api.history_dns_soa(domain) + + if results: + r.extend(__history_dns(results, domain, 'email', 'soa')) + + time.sleep(1) + + results = api.history_dns_mx(domain) + + if results: + status_ok = True + r.extend(__history_dns(results, domain, 'host', 'mx')) + + except APIError as e: + misperrors['error'] = e.value + return [], False + + status_ok = True + + return r, status_ok + + +def expand_history_whois(api, domain): + r = [] + status_ok = False + try: + results = api.history_whois(domain) + + if results: + + if 'items' in results['result']: + for item in results['result']['items']: + item_registrant = __select_registrant_item(item) + + r.append( + { + 'types': ['domain'], + 'values': item['nameServers'], + 'categories': ['Network activity'], + 'comment': 'Whois history Name Servers of %s ' + 'Status: %s ' % ( + domain, ' '.join(item['status'])) + + } + ) + if item_registrant: + + if 'email' in item_registrant[0]: + r.append( + { + 'types': ['whois-registrant-email'], + 'values': [item_registrant[0]['email']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant email of %s' + 'Status: %s' % ( + domain, + ' '.join(item['status'])) + } + ) + + if 'telephone' in item_registrant[0]: + r.append( + { + 'types': ['whois-registrant-phone'], + 'values': [item_registrant[0]['telephone']], + 'categories': ['Attribution'], + 'comment': 'Whois history registrant phone of %s' + 'Status: %s' % ( + domain, + ' '.join(item['status'])) + } + ) + + except APIError as e: + misperrors['error'] = e.value + return [], False + status_ok = True + + return r, status_ok + + +def __history_ip(results, domain, type_ip='ip'): + r = [] + if 'records' in results: + for record in results['records']: + if 'values' in record: + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': ['%s|%s' % (domain, item[type_ip])], + 'categories': ['Network activity'], + 'comment': 'History IP on securitytrails %s ' + 'last seen: %s first seen: %s' % + (domain, record['last_seen'], + record['first_seen']) + } + ) + + return r + + +def __history_dns(results, domain, type_serv, service): + r = [] + + if 'records' in results: + for record in results['records']: + if 'values' in record: + values = record['values'] + if type(values) is list: + + for item in record['values']: + r.append( + {'types': ['domain|ip'], + 'values': [item[type_serv]], + 'categories': ['Network activity'], + 'comment': 'history %s of %s last seen: %s first seen: %s' % + (service, domain, record['last_seen'], + record['first_seen']) + } + ) + else: + r.append( + {'types': ['domain|ip'], + 'values': [values[type_serv]], + 'categories': ['Network activity'], + 'comment': 'history %s of %s last seen: %s first seen: %s' % + (service, domain, record['last_seen'], + record['first_seen']) + } + ) + return r + + +def expand_searching_domain(api, ip): + r = [] + status_ok = False + + try: + results = api.searching_domains(ipv4=ip) + + if results: + if 'records' in results: + res = [(r['host_provider'], r['hostname'], r['whois']) + for r in results['records']] + + for host_provider, hostname, whois in res: + comment = 'domain for %s by %s' % (ip, host_provider[0]) + if whois['registrar']: + comment = comment + ' registrar %s' % whois['registrar'] + + r.append( + { + 'types': ['domain'], + 'category': ['Network activity'], + 'values': [hostname], + 'comment': comment + + } + ) + status_ok = True + except APIError as e: + misperrors['error'] = e.value + return [], False + + return r, status_ok + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + + +def __select_registrant_item(entry): + res = None + if 'contacts' in entry: + res = list(filter(lambda x: x['type'] == 'registrant', + entry['contacts'])) + + if 'contact' in entry: + res = list(filter(lambda x: x['type'] == 'registrant', + entry['contact'])) + + return res diff --git a/misp_modules/modules/expansion/shodan.py b/misp_modules/modules/expansion/shodan.py index fbdf5cd..5a4b792 100755 --- a/misp_modules/modules/expansion/shodan.py +++ b/misp_modules/modules/expansion/shodan.py @@ -27,8 +27,8 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('apikey')): - misperrors['error'] = 'shodan authentication is missing' + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = 'Shodan authentication is missing' return misperrors api = shodan.Shodan(request['config'].get('apikey')) diff --git a/misp_modules/modules/expansion/sigma_queries.py b/misp_modules/modules/expansion/sigma_queries.py new file mode 100644 index 0000000..d17a100 --- /dev/null +++ b/misp_modules/modules/expansion/sigma_queries.py @@ -0,0 +1,50 @@ +import io +import json +try: + from sigma.parser.collection import SigmaCollectionParser + from sigma.configuration import SigmaConfiguration + from sigma.backends.discovery import getBackend +except ImportError: + print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['sigma'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'], + 'description': 'An expansion hover module to display the result of sigma queries.'} +moduleconfig = [] +sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'mdatp', 'splunkxml', 'arcsight', 'qualys') + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('sigma'): + misperrors['error'] = 'Sigma rule missing' + return misperrors + config = SigmaConfiguration() + f = io.TextIOWrapper(io.BytesIO(request.get('sigma').encode()), encoding='utf-8') + parser = SigmaCollectionParser(f, config) + targets = [] + results = [] + for t in sigma_targets: + backend = getBackend(t)(config, {'rulecomment': False}) + try: + parser.generate(backend) + result = backend.finalize() + if result: + results.append(result) + targets.append(t) + except Exception: + continue + d_result = {t: r.strip() for t, r in zip(targets, results)} + return {'results': [{'types': mispattributes['output'], 'values': d_result}]} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/sigma_syntax_validator.py b/misp_modules/modules/expansion/sigma_syntax_validator.py index 0d5226f..658b4d3 100644 --- a/misp_modules/modules/expansion/sigma_syntax_validator.py +++ b/misp_modules/modules/expansion/sigma_syntax_validator.py @@ -1,9 +1,9 @@ import json try: import yaml - from sigma.parser import SigmaParser - from sigma.config import SigmaConfiguration -except ModuleNotFoundError: + from sigma.parser.rule import SigmaParser + from sigma.configuration import SigmaConfiguration +except ImportError: print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") misperrors = {'error': 'Error'} @@ -12,6 +12,7 @@ moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['e 'description': 'An expansion hover module to perform a syntax check on sigma rules'} moduleconfig = [] + def handler(q=False): if q is False: return False @@ -21,15 +22,17 @@ def handler(q=False): return misperrors config = SigmaConfiguration() try: - parser = SigmaParser(yaml.load(request.get('sigma')), config) + parser = SigmaParser(yaml.safe_load(request.get('sigma')), config) result = ("Syntax valid: {}".format(parser.values)) except Exception as e: result = ("Syntax error: {}".format(str(e))) return {'results': [{'types': mispattributes['output'], 'values': result}]} + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/sophoslabs_intelix.py b/misp_modules/modules/expansion/sophoslabs_intelix.py new file mode 100644 index 0000000..017683a --- /dev/null +++ b/misp_modules/modules/expansion/sophoslabs_intelix.py @@ -0,0 +1,124 @@ +from pymisp import MISPEvent, MISPObject +import json +import requests +import base64 +from urllib.parse import quote + +moduleinfo = {'version': '1.0', + 'author': 'Ben Verschaeren', + 'description': 'SOPHOSLabs Intelix Integration', + 'module-type': ['expansion']} + +moduleconfig = ['client_id', 'client_secret'] + +misperrors = {'error': 'Error'} + +misp_types_in = ['sha256', 'ip', 'ip-src', 'ip-dst', 'uri', 'url', 'domain', 'hostname'] + +mispattributes = {'input': misp_types_in, + 'format': 'misp_standard'} + + +class SophosLabsApi(): + def __init__(self, client_id, client_secret): + self.misp_event = MISPEvent() + self.client_id = client_id + self.client_secret = client_secret + self.authToken = f"{self.client_id}:{self.client_secret}" + self.baseurl = 'de.api.labs.sophos.com' + d = {'grant_type': 'client_credentials'} + h = {'Authorization': f"Basic {base64.b64encode(self.authToken.encode('UTF-8')).decode('ascii')}", + 'Content-Type': 'application/x-www-form-urlencoded'} + r = requests.post('https://api.labs.sophos.com/oauth2/token', headers=h, data=d) + if r.status_code == 200: + j = json.loads(r.text) + self.accessToken = j['access_token'] + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + def hash_lookup(self, filehash): + sophos_object = MISPObject('SOPHOSLabs Intelix SHA256 Report') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/files/v1/{filehash}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'reputationScore' in j: + sophos_object.add_attribute('Reputation Score', type='text', value=j['reputationScore']) + if 0 <= j['reputationScore'] <= 19: + sophos_object.add_attribute('Decision', type='text', value='This file is malicious') + if 20 <= j['reputationScore'] <= 29: + sophos_object.add_attribute('Decision', type='text', value='This file is potentially unwanted') + if 30 <= j['reputationScore'] <= 69: + sophos_object.add_attribute('Decision', type='text', value='This file is unknown and suspicious') + if 70 <= j['reputationScore'] <= 100: + sophos_object.add_attribute('Decision', type='text', value='This file is known good') + if 'detectionName' in j: + sophos_object.add_attribute('Detection Name', type='text', value=j['detectionName']) + else: + sophos_object.add_attribute('Detection Name', type='text', value='No name associated with this IoC') + self.misp_event.add_object(**sophos_object) + + def ip_lookup(self, ip): + sophos_object = MISPObject('SOPHOSLabs Intelix IP Category Lookup') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/ips/v1/{ip}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'category' in j: + for c in j['category']: + sophos_object.add_attribute('IP Address Categorisation', type='text', value=c) + else: + sophos_object.add_attribute('IP Address Categorisation', type='text', value='No category assocaited with IoC') + self.misp_event.add_object(**sophos_object) + + def url_lookup(self, url): + sophos_object = MISPObject('SOPHOSLabs Intelix URL Lookup') + h = {"Authorization": f"{self.accessToken}"} + r = requests.get(f"https://{self.baseurl}/lookup/urls/v1/{quote(url, safe='')}", headers=h) + if r.status_code == 200: + j = json.loads(r.text) + if 'productivityCategory' in j: + sophos_object.add_attribute('URL Categorisation', type='text', value=j['productivityCategory']) + else: + sophos_object.add_attribute('URL Categorisation', type='text', value='No category assocaited with IoC') + + if 'riskLevel' in j: + sophos_object.add_attribute('URL Risk Level', type='text', value=j['riskLevel']) + else: + sophos_object.add_attribute('URL Risk Level', type='text', value='No risk level associated with IoC') + + if 'securityCategory' in j: + sophos_object.add_attribute('URL Security Category', type='text', value=j['securityCategory']) + else: + sophos_object.add_attribute('URL Security Category', type='text', value='No Security Category associated with IoC') + self.misp_event.add_object(**sophos_object) + + +def handler(q=False): + if q is False: + return False + j = json.loads(q) + if not j.get('config') or not j['config'].get('client_id') or not j['config'].get('client_secret'): + misperrors['error'] = "Missing client_id or client_secret value for SOPHOSLabs Intelix. \ + It's free to sign up here https://aws.amazon.com/marketplace/pp/B07SLZPMCS." + return misperrors + client = SophosLabsApi(j['config']['client_id'], j['config']['client_secret']) + if j['attribute']['type'] == "sha256": + client.hash_lookup(j['attribute']['value1']) + if j['attribute']['type'] in ['ip-dst', 'ip-src', 'ip']: + client.ip_lookup(j["attribute"]["value1"]) + if j['attribute']['type'] in ['uri', 'url', 'domain', 'hostname']: + client.url_lookup(j["attribute"]["value1"]) + return client.get_result() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py new file mode 100644 index 0000000..842217a --- /dev/null +++ b/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py @@ -0,0 +1,45 @@ +import json +try: + from stix2patterns.validator import run_validator +except ImportError: + print("stix2 patterns python library is missing, use 'pip3 install stix2-patterns' to install it.") + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['stix2-pattern'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['hover'], + 'description': 'An expansion hover module to perform a syntax check on stix2 patterns.'} +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('stix2-pattern'): + misperrors['error'] = 'STIX2 pattern missing' + return misperrors + pattern = request.get('stix2-pattern') + syntax_errors = [] + for p in pattern[1:-1].split(' AND '): + syntax_validator = run_validator("[{}]".format(p)) + if syntax_validator: + for error in syntax_validator: + syntax_errors.append(error) + if syntax_errors: + s = 's' if len(syntax_errors) > 1 else '' + s_errors = "" + for error in syntax_errors: + s_errors += "{}\n".format(error[6:]) + result = "Syntax error{}: \n{}".format(s, s_errors[:-1]) + else: + result = "Syntax valid" + return {'results': [{'types': mispattributes['output'], 'values': result}]} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/threatcrowd.py b/misp_modules/modules/expansion/threatcrowd.py index 9187ca5..268832f 100644 --- a/misp_modules/modules/expansion/threatcrowd.py +++ b/misp_modules/modules/expansion/threatcrowd.py @@ -17,7 +17,7 @@ moduleconfig = [] # Avoid adding windows update to enrichment etc. def isBlacklisted(value): - blacklist = ['8.8.8.8', '255.255.255.255', '192.168.56.' , 'time.windows.com'] + blacklist = ['8.8.8.8', '255.255.255.255', '192.168.56.', 'time.windows.com'] for b in blacklist: if value in b: @@ -25,28 +25,31 @@ def isBlacklisted(value): return False + def valid_ip(ip): m = re.match(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$", ip) return bool(m) and all(map(lambda n: 0 <= int(n) <= 255, m.groups())) + def valid_domain(hostname): if len(hostname) > 255: return False if hostname[-1] == ".": - hostname = hostname[:-1] # strip exactly one dot from the right, if present - allowed = re.compile("(?!-)[A-Z\d-]{1,63}(? Value of entity. + """ + report_links = list() + trustar_reports = self.ts_client.search_reports(entity_value) + for report in trustar_reports: + report_links.append(self.REPORT_BASE_URL.format(report.id)) + + return report_links + + def parse_indicator_summary(self, summaries): + """ + Converts a response from the TruSTAR /1.3/indicators/summaries endpoint + a MISP trustar_report object and adds the summary data and links as attributes. + + :param summaries: A TruSTAR Python SDK Page.generator object for generating + indicator summaries pages. + """ + + for summary in summaries: + trustar_obj = MISPObject('trustar_report') + indicator_type = summary.indicator_type + indicator_value = summary.value + if indicator_type in self.ENTITY_TYPE_MAPPINGS: + trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], + value=indicator_value) + trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", + value=json.dumps(summary.to_dict(), sort_keys=True, indent=4)) + report_links = self.generate_trustar_links(indicator_value) + for link in report_links: + trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=link) + self.misp_event.add_object(**trustar_obj) + + +def handler(q=False): + """ + MISP handler function. A user's API key and secret will be retrieved from the MISP + request and used to create a TruSTAR API client. If enclave IDs are provided, only + those enclaves will be queried for data. Otherwise, all of the enclaves a user has + access to will be queried. + """ + + if q is False: + return False + + request = json.loads(q) + + config = request.get('config', {}) + if not config.get('user_api_key') or not config.get('user_api_secret'): + misperrors['error'] = "Your TruSTAR API key and secret are required for indicator enrichment." + return misperrors + + attribute = request['attribute'] + trustar_parser = TruSTARParser(attribute, config) + + try: + summaries = list( + trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE)) + except Exception as e: + misperrors['error'] = "Unable to retrieve TruSTAR summary data: {}".format(e) + return misperrors + + trustar_parser.parse_indicator_summary(summaries) + return trustar_parser.get_results() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py new file mode 100644 index 0000000..baaaaf6 --- /dev/null +++ b/misp_modules/modules/expansion/urlhaus.py @@ -0,0 +1,148 @@ +from pymisp import MISPAttribute, MISPEvent, MISPObject +import json +import requests + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['domain', 'hostname', 'ip-src', 'ip-dst', 'md5', 'sha256', 'url'], + 'output': ['url', 'filename', 'md5', 'sha256'], + 'format': 'misp_standard'} +moduleinfo = {'version': '0.1', 'author': 'Christian Studer', + 'description': 'Query of the URLhaus API to get additional information about some attributes.', + 'module-type': ['expansion', 'hover']} +moduleconfig = [] + +file_keys = ('filename', 'response_size', 'response_md5', 'response_sha256') +file_relations = ('filename', 'size-in-bytes', 'md5', 'sha256') +vt_keys = ('result', 'link') +vt_types = ('text', 'link') +vt_relations = ('detection-ratio', 'permalink') + + +class URLhaus(): + def __init__(self): + super(URLhaus, self).__init__() + self.misp_event = MISPEvent() + + @staticmethod + def _create_vt_object(virustotal): + vt_object = MISPObject('virustotal-report') + for key, vt_type, relation in zip(vt_keys, vt_types, vt_relations): + vt_object.add_attribute(relation, **{'type': vt_type, 'value': virustotal[key]}) + return vt_object + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + def parse_error(self, query_status): + if query_status == 'no_results': + return {'error': f'No results found on URLhaus for this {self.attribute.type} attribute'} + return {'error': f'Error encountered during the query of URLhaus: {query_status}'} + + +class HostQuery(URLhaus): + def __init__(self, attribute): + super(HostQuery, self).__init__() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.url = 'https://urlhaus-api.abuse.ch/v1/host/' + + def query_api(self): + response = requests.post(self.url, data={'host': self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) + if 'urls' in response and response['urls']: + for url in response['urls']: + self.misp_event.add_attribute(type='url', value=url['url']) + return self.get_result() + + +class PayloadQuery(URLhaus): + def __init__(self, attribute): + super(PayloadQuery, self).__init__() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.url = 'https://urlhaus-api.abuse.ch/v1/payload/' + + def query_api(self): + hash_type = self.attribute.type + file_object = MISPObject('file') + if hasattr(self.attribute, 'object_id') and hasattr(self.attribute, 'event_id') and self.attribute.event_id != '0': + file_object.id = self.attribute.object_id + response = requests.post(self.url, data={'{}_hash'.format(hash_type): self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) + other_hash_type = 'md5' if hash_type == 'sha256' else 'sha256' + for key, relation in zip(('{}_hash'.format(other_hash_type), 'file_size'), (other_hash_type, 'size-in-bytes')): + if response[key]: + file_object.add_attribute(relation, **{'type': relation, 'value': response[key]}) + if response['virustotal']: + vt_object = self._create_vt_object(response['virustotal']) + file_object.add_reference(vt_object.uuid, 'analyzed-with') + self.misp_event.add_object(**vt_object) + _filename_ = 'filename' + for url in response['urls']: + attribute = MISPAttribute() + attribute.from_dict(type='url', value=url['url']) + self.misp_event.add_attribute(**attribute) + file_object.add_reference(attribute.uuid, 'retrieved-from') + if url[_filename_]: + file_object.add_attribute(_filename_, **{'type': _filename_, 'value': url[_filename_]}) + if any((file_object.attributes, file_object.references)): + self.misp_event.add_object(**file_object) + return self.get_result() + + +class UrlQuery(URLhaus): + def __init__(self, attribute): + super(UrlQuery, self).__init__() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.url = 'https://urlhaus-api.abuse.ch/v1/url/' + + @staticmethod + def _create_file_object(payload): + file_object = MISPObject('file') + for key, relation in zip(file_keys, file_relations): + if payload[key]: + file_object.add_attribute(relation, **{'type': relation, 'value': payload[key]}) + return file_object + + def query_api(self): + response = requests.post(self.url, data={'url': self.attribute.value}).json() + if response['query_status'] != 'ok': + return self.parse_error(response['query_status']) + if 'payloads' in response and response['payloads']: + for payload in response['payloads']: + file_object = self._create_file_object(payload) + if payload['virustotal']: + vt_object = self._create_vt_object(payload['virustotal']) + file_object.add_reference(vt_object.uuid, 'analyzed-with') + self.misp_event.add_object(**vt_object) + if any((file_object.attributes, file_object.references)): + self.misp_event.add_object(**file_object) + return self.get_result() + + +_misp_type_mapping = {'url': UrlQuery, 'md5': PayloadQuery, 'sha256': PayloadQuery, + 'domain': HostQuery, 'hostname': HostQuery, + 'ip-src': HostQuery, 'ip-dst': HostQuery} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + attribute = request['attribute'] + urlhaus_parser = _misp_type_mapping[attribute['type']](attribute) + return urlhaus_parser.query_api() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/urlscan.py b/misp_modules/modules/expansion/urlscan.py new file mode 100644 index 0000000..e6af7f6 --- /dev/null +++ b/misp_modules/modules/expansion/urlscan.py @@ -0,0 +1,262 @@ +import json +import requests +import logging +import sys +import time + +log = logging.getLogger('urlscan') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + +moduleinfo = { + 'version': '0.1', + 'author': 'Dave Johnson', + 'description': 'Module to query urlscan.io', + 'module-type': ['expansion'] +} + +moduleconfig = ['apikey'] +misperrors = {'error': 'Error'} +mispattributes = { + 'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url'], + 'output': ['hostname', 'domain', 'ip-src', 'ip-dst', 'url', 'text', 'link', 'hash'] +} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = 'Urlscan apikey is missing' + return misperrors + client = urlscanAPI(request['config']['apikey']) + + r = {'results': []} + + if 'ip-src' in request: + r['results'] += lookup_indicator(client, request['ip-src']) + if 'ip-dst' in request: + r['results'] += lookup_indicator(client, request['ip-dst']) + if 'domain' in request: + r['results'] += lookup_indicator(client, request['domain']) + if 'hostname' in request: + r['results'] += lookup_indicator(client, request['hostname']) + if 'url' in request: + r['results'] += lookup_indicator(client, request['url']) + + # Return any errors generated from lookup to the UI and remove duplicates + + uniq = [] + log.debug(r['results']) + for item in r['results']: + log.debug(item) + if 'error' in item: + misperrors['error'] = item['error'] + return misperrors + if item not in uniq: + uniq.append(item) + r['results'] = uniq + return r + + +def lookup_indicator(client, query): + result = client.search_url(query) + log.debug('RESULTS: ' + json.dumps(result)) + r = [] + misp_comment = "{}: Enriched via the urlscan module".format(query) + + # Determine if the page is reachable + for request in result['data']['requests']: + if request['response'].get('failed'): + if request['response']['failed']['errorText']: + log.debug('The page could not load') + r.append( + {'error': 'Domain could not be resolved: {}'.format(request['response']['failed']['errorText'])}) + + if result.get('page'): + if result['page'].get('domain'): + misp_val = result['page']['domain'] + r.append({'types': 'domain', + 'categories': ['Network activity'], + 'values': misp_val, + 'comment': misp_comment}) + + if result['page'].get('ip'): + misp_val = result['page']['ip'] + r.append({'types': 'ip-dst', + 'categories': ['Network activity'], + 'values': misp_val, + 'comment': misp_comment}) + + if result['page'].get('country'): + misp_val = 'country: ' + result['page']['country'] + if result['page'].get('city'): + misp_val += ', city: ' + result['page']['city'] + r.append({'types': 'text', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result['page'].get('asn'): + misp_val = result['page']['asn'] + r.append({'types': 'AS', 'categories': ['External analysis'], 'values': misp_val, 'comment': misp_comment}) + + if result['page'].get('asnname'): + misp_val = result['page']['asnname'] + r.append({'types': 'text', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result.get('stats'): + if result['stats'].get('malicious'): + log.debug('There is something in results > stats > malicious') + threat_list = set() + + if 'matches' in result['meta']['processors']['gsb']['data']: + for item in result['meta']['processors']['gsb']['data']['matches']: + if item['threatType']: + threat_list.add(item['threatType']) + + threat_list = ', '.join(threat_list) + log.debug('threat_list values are: \'' + threat_list + '\'') + + if threat_list: + misp_val = '{} threat(s) detected'.format(threat_list) + r.append({'types': 'text', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result.get('lists'): + if result['lists'].get('urls'): + for url in result['lists']['urls']: + url = url.lower() + if 'office' in url: + misp_val = "Possible Office-themed phishing" + elif 'o365' in url or '0365' in url: + misp_val = "Possible O365-themed phishing" + elif 'microsoft' in url: + misp_val = "Possible Microsoft-themed phishing" + elif 'paypal' in url: + misp_val = "Possible PayPal-themed phishing" + elif 'onedrive' in url: + misp_val = "Possible OneDrive-themed phishing" + elif 'docusign' in url: + misp_val = "Possible DocuSign-themed phishing" + r.append({'types': 'text', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result.get('task'): + if result['task'].get('reportURL'): + misp_val = result['task']['reportURL'] + r.append({'types': 'link', + 'categories': ['External analysis'], + 'values': misp_val, + 'comment': misp_comment}) + + if result['task'].get('screenshotURL'): + image_url = result['task']['screenshotURL'] + r.append({'types': 'link', + 'categories': ['External analysis'], + 'values': image_url, + 'comment': misp_comment}) + # ## TO DO ### + # ## Add ability to add an in-line screenshot of the target website into an attribute + # screenshot = requests.get(image_url).content + # r.append({'types': ['attachment'], + # 'categories': ['External analysis'], + # 'values': image_url, + # 'image': str(base64.b64encode(screenshot), 'utf-8'), + # 'comment': 'Screenshot of website'}) + + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo + + +class urlscanAPI(): + def __init__(self, apikey=None, uuid=None): + self.key = apikey + self.uuid = uuid + + def request(self, query): + log.debug('From request function with the parameter: ' + query) + payload = {'url': query} + headers = {'API-Key': self.key, + 'Content-Type': "application/json", + 'Cache-Control': "no-cache"} + + # Troubleshooting problems with initial search request + log.debug('PAYLOAD: ' + json.dumps(payload)) + log.debug('HEADERS: ' + json.dumps(headers)) + + search_url_string = "https://urlscan.io/api/v1/scan/" + response = requests.request("POST", + search_url_string, + data=json.dumps(payload), + headers=headers) + + # HTTP 400 - Bad Request + if response.status_code == 400: + raise Exception('HTTP Error 400 - Bad Request') + + # HTTP 404 - Not found + if response.status_code == 404: + raise Exception('HTTP Error 404 - These are not the droids you\'re looking for') + + # Any other status code + if response.status_code != 200: + raise Exception('HTTP Error ' + str(response.status_code)) + + if response.text: + response = json.loads(response.content.decode("utf-8")) + time.sleep(3) + self.uuid = response['uuid'] + + # Strings for to check for errors on the results page + # Null response string for any unavailable resources + null_response_string = '"status": 404' + # Redirect string accounting for 301/302/303/307/308 status codes + redirect_string = '"status": 30' + # Normal response string with 200 status code + normal_response_string = '"status": 200' + + results_url_string = "https://urlscan.io/api/v1/result/" + self.uuid + log.debug('Results URL: ' + results_url_string) + + # Need to wait for results to process and check if they are valid + tries = 10 + while tries >= 0: + results = requests.request("GET", results_url_string) + log.debug('Made a GET request') + results = results.content.decode("utf-8") + # checking if there is a 404 status code and no available resources + if null_response_string in results and \ + redirect_string not in results and \ + normal_response_string not in results: + log.debug('Results not processed. Please check again later.') + time.sleep(3) + tries -= 1 + else: + return json.loads(results) + + raise Exception('Results contained a 404 status error and could not be processed.') + + def search_url(self, query): + log.debug('From search_url with parameter: ' + query) + return self.request(query) diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py old mode 100755 new mode 100644 index 3997ee6..f47a2e3 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -1,196 +1,210 @@ +from pymisp import MISPAttribute, MISPEvent, MISPObject import json import requests -from requests import HTTPError -import base64 misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "sha512"], - 'output': ['domain', "ip-src", "ip-dst", "text", "md5", "sha1", "sha256", "sha512", "ssdeep", - "authentihash", "filename"] - } +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], + 'format': 'misp_standard'} # possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '2', 'author': 'Hannah Ward', - 'description': 'Get information from virustotal', +moduleinfo = {'version': '4', 'author': 'Hannah Ward', + 'description': 'Get information from VirusTotal', 'module-type': ['expansion']} # config fields that your code expects from the site admin moduleconfig = ["apikey", "event_limit"] -limit = 5 # Default -comment = '%s: Enriched via VT' + + +class VirusTotalParser(object): + def __init__(self, apikey, limit): + self.apikey = apikey + self.limit = limit + self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report" + self.misp_event = MISPEvent() + self.parsed_objects = {} + self.input_types_mapping = {'ip-src': self.parse_ip, 'ip-dst': self.parse_ip, + 'domain': self.parse_domain, 'hostname': self.parse_domain, + 'md5': self.parse_hash, 'sha1': self.parse_hash, + 'sha256': self.parse_hash, 'url': self.parse_url} + + def query_api(self, attribute): + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + return self.input_types_mapping[self.attribute.type](self.attribute.value, recurse=True) + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + ################################################################################ + #### Main parsing functions #### # noqa + ################################################################################ + + def parse_domain(self, domain, recurse=False): + req = requests.get(self.base_url.format('domain'), params={'apikey': self.apikey, 'domain': domain}) + if req.status_code != 200: + return req.status_code + req = req.json() + hash_type = 'sha256' + whois = 'whois' + feature_types = {'communicating': 'communicates-with', + 'downloaded': 'downloaded-from', + 'referrer': 'referring'} + siblings = (self.parse_siblings(domain) for domain in req['domain_siblings']) + uuid = self.parse_resolutions(req['resolutions'], req['subdomains'], siblings) + for feature_type, relationship in feature_types.items(): + for feature in ('undetected_{}_samples', 'detected_{}_samples'): + for sample in req.get(feature.format(feature_type), [])[:self.limit]: + status_code = self.parse_hash(sample[hash_type], False, uuid, relationship) + if status_code != 200: + return status_code + if req.get(whois): + whois_object = MISPObject(whois) + whois_object.add_attribute('text', type='text', value=req[whois]) + self.misp_event.add_object(**whois_object) + return self.parse_related_urls(req, recurse, uuid) + + def parse_hash(self, sample, recurse=False, uuid=None, relationship=None): + req = requests.get(self.base_url.format('file'), params={'apikey': self.apikey, 'resource': sample}) + status_code = req.status_code + if req.status_code == 200: + req = req.json() + vt_uuid = self.parse_vt_object(req) + file_attributes = [] + for hash_type in ('md5', 'sha1', 'sha256'): + if req.get(hash_type): + file_attributes.append({'type': hash_type, 'object_relation': hash_type, + 'value': req[hash_type]}) + if file_attributes: + file_object = MISPObject('file') + for attribute in file_attributes: + file_object.add_attribute(**attribute) + file_object.add_reference(vt_uuid, 'analyzed-with') + if uuid and relationship: + file_object.add_reference(uuid, relationship) + self.misp_event.add_object(**file_object) + return status_code + + def parse_ip(self, ip, recurse=False): + req = requests.get(self.base_url.format('ip-address'), params={'apikey': self.apikey, 'ip': ip}) + if req.status_code != 200: + return req.status_code + req = req.json() + if req.get('asn'): + asn_mapping = {'network': ('ip-src', 'subnet-announced'), + 'country': ('text', 'country')} + asn_object = MISPObject('asn') + asn_object.add_attribute('asn', type='AS', value=req['asn']) + for key, value in asn_mapping.items(): + if req.get(key): + attribute_type, relation = value + asn_object.add_attribute(relation, type=attribute_type, value=req[key]) + self.misp_event.add_object(**asn_object) + uuid = self.parse_resolutions(req['resolutions']) if req.get('resolutions') else None + return self.parse_related_urls(req, recurse, uuid) + + def parse_url(self, url, recurse=False, uuid=None): + req = requests.get(self.base_url.format('url'), params={'apikey': self.apikey, 'resource': url}) + status_code = req.status_code + if req.status_code == 200: + req = req.json() + vt_uuid = self.parse_vt_object(req) + if not recurse: + feature = 'url' + url_object = MISPObject(feature) + url_object.add_attribute(feature, type=feature, value=url) + url_object.add_reference(vt_uuid, 'analyzed-with') + if uuid: + url_object.add_reference(uuid, 'hosted-in') + self.misp_event.add_object(**url_object) + return status_code + + ################################################################################ + #### Additional parsing functions #### # noqa + ################################################################################ + + def parse_related_urls(self, query_result, recurse, uuid=None): + if recurse: + for feature in ('detected_urls', 'undetected_urls'): + if feature in query_result: + for url in query_result[feature]: + value = url['url'] if isinstance(url, dict) else url[0] + status_code = self.parse_url(value, False, uuid) + if status_code != 200: + return status_code + else: + for feature in ('detected_urls', 'undetected_urls'): + if feature in query_result: + for url in query_result[feature]: + value = url['url'] if isinstance(url, dict) else url[0] + self.misp_event.add_attribute('url', value) + return 200 + + def parse_resolutions(self, resolutions, subdomains=None, uuids=None): + domain_ip_object = MISPObject('domain-ip') + if self.attribute.type == 'domain': + domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value) + attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address') + else: + domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value) + attribute_type, relation, key = ('domain', 'domain', 'hostname') + for resolution in resolutions: + domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key]) + if subdomains: + for subdomain in subdomains: + attribute = MISPAttribute() + attribute.from_dict(**dict(type='domain', value=subdomain)) + self.misp_event.add_attribute(**attribute) + domain_ip_object.add_reference(attribute.uuid, 'subdomain') + if uuids: + for uuid in uuids: + domain_ip_object.add_reference(uuid, 'sibling-of') + self.misp_event.add_object(**domain_ip_object) + return domain_ip_object.uuid + + def parse_siblings(self, domain): + attribute = MISPAttribute() + attribute.from_dict(**dict(type='domain', value=domain)) + self.misp_event.add_attribute(**attribute) + return attribute.uuid + + def parse_vt_object(self, query_result): + if query_result['response_code'] == 1: + vt_object = MISPObject('virustotal-report') + vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) + detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) + vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) + self.misp_event.add_object(**vt_object) + return vt_object.uuid + + +def parse_error(status_code): + status_mapping = {204: 'VirusTotal request rate limit exceeded.', + 400: 'Incorrect request, please check the arguments.', + 403: 'You don\'t have enough privileges to make the request.'} + if status_code in status_mapping: + return status_mapping[status_code] + return "VirusTotal may not be accessible." def handler(q=False): - global limit if q is False: return False - - q = json.loads(q) - - key = q["config"]["apikey"] - limit = int(q["config"].get("event_limit", 5)) - - r = {"results": []} - - if "ip-src" in q: - r["results"] += getIP(q["ip-src"], key) - if "ip-dst" in q: - r["results"] += getIP(q["ip-dst"], key) - if "domain" in q: - r["results"] += getDomain(q["domain"], key) - if 'hostname' in q: - r["results"] += getDomain(q['hostname'], key) - if 'md5' in q: - r["results"] += getHash(q['md5'], key) - if 'sha1' in q: - r["results"] += getHash(q['sha1'], key) - if 'sha256' in q: - r["results"] += getHash(q['sha256'], key) - if 'sha512' in q: - r["results"] += getHash(q['sha512'], key) - - uniq = [] - for res in r["results"]: - if res not in uniq: - uniq.append(res) - r["results"] = uniq - return r - - -def getHash(hash, key, do_not_recurse=False): - req = requests.get("https://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, 'resource': hash}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) + request = json.loads(q) + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = "A VirusTotal api key is required for this module." return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - return getMoreInfo(req, key) - - -def getIP(ip, key, do_not_recurse=False): - global limit - toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/ip-address/report", - params={"ip": ip, "apikey": key}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) + event_limit = request['config'].get('event_limit') + if not isinstance(event_limit, int): + event_limit = 5 + parser = VirusTotalParser(request['config']['apikey'], event_limit) + attribute = request['attribute'] + status = parser.query_api(attribute) + if status != 200: + misperrors['error'] = parse_error(status) return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append({"types": ["domain"], "values": [res["hostname"]], "comment": comment % ip}) - # Pivot from here to find all domain info - if not do_not_recurse: - toReturn += getDomain(res["hostname"], key, True) - - toReturn += getMoreInfo(req, key) - return toReturn - - -def getDomain(domain, key, do_not_recurse=False): - global limit - toReturn = [] - req = requests.get("https://www.virustotal.com/vtapi/v2/domain/report", - params={"domain": domain, "apikey": key}) - try: - req.raise_for_status() - req = req.json() - except HTTPError as e: - misperrors['error'] = str(e) - return misperrors - - if req["response_code"] == 0: - # Nothing found - return [] - - if "resolutions" in req: - for res in req["resolutions"][:limit]: - toReturn.append({"types": ["ip-dst", "ip-src"], "values": [res["ip_address"]], "comment": comment % domain}) - # Pivot from here to find all info on IPs - if not do_not_recurse: - toReturn += getIP(res["ip_address"], key, True) - if "subdomains" in req: - for subd in req["subdomains"]: - toReturn.append({"types": ["domain"], "values": [subd], "comment": comment % domain}) - toReturn += getMoreInfo(req, key) - return toReturn - - -def findAll(data, keys): - a = [] - if isinstance(data, dict): - for key in data.keys(): - if key in keys: - a.append(data[key]) - else: - if isinstance(data[key], (dict, list)): - a += findAll(data[key], keys) - if isinstance(data, list): - for i in data: - a += findAll(i, keys) - - return a - - -def getMoreInfo(req, key): - global limit - r = [] - # Get all hashes first - hashes = [] - hashes = findAll(req, ["md5", "sha1", "sha256", "sha512"]) - r.append({"types": ["freetext"], "values": hashes}) - for hsh in hashes[:limit]: - # Search VT for some juicy info - try: - data = requests.get("http://www.virustotal.com/vtapi/v2/file/report", - params={"allinfo": 1, "apikey": key, "resource": hsh} - ).json() - except: - continue - - # Go through each key and check if it exists - if "submission_names" in data: - r.append({'types': ["filename"], "values": data["submission_names"], "comment": comment % hsh}) - - if "ssdeep" in data: - r.append({'types': ["ssdeep"], "values": [data["ssdeep"]], "comment": comment % hsh}) - - if "authentihash" in data: - r.append({"types": ["authentihash"], "values": [data["authentihash"]], "comment": comment % hsh}) - - if "ITW_urls" in data: - r.append({"types": ["url"], "values": data["ITW_urls"], "comment": comment % hsh}) - - # Get the malware sample - sample = requests.get("https://www.virustotal.com/vtapi/v2/file/download", - params={"hash": hsh, "apikey": key}) - - malsample = sample.content - - # It is possible for VT to not give us any submission names - if "submission_names" in data: - r.append({"types": ["malware-sample"], - "categories": ["Payload delivery"], - "values": data["submission_names"], - "data": str(base64.b64encode(malsample), 'utf-8') - } - ) - - return r + return parser.get_result() def introspection(): diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py new file mode 100644 index 0000000..e7c2e96 --- /dev/null +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -0,0 +1,196 @@ +from pymisp import MISPAttribute, MISPEvent, MISPObject +import json +import requests + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], + 'format': 'misp_standard'} +moduleinfo = {'version': '1', 'author': 'Christian Studer', + 'description': 'Get information from VirusTotal public API v2.', + 'module-type': ['expansion', 'hover']} + +moduleconfig = ['apikey'] + + +class VirusTotalParser(): + def __init__(self): + super(VirusTotalParser, self).__init__() + self.misp_event = MISPEvent() + + def declare_variables(self, apikey, attribute): + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.apikey = apikey + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + def parse_urls(self, query_result): + for feature in ('detected_urls', 'undetected_urls'): + if feature in query_result: + for url in query_result[feature]: + value = url['url'] if isinstance(url, dict) else url[0] + self.misp_event.add_attribute('url', value) + + def parse_resolutions(self, resolutions, subdomains=None, uuids=None): + domain_ip_object = MISPObject('domain-ip') + if self.attribute.type == 'domain': + domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value) + attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address') + else: + domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value) + attribute_type, relation, key = ('domain', 'domain', 'hostname') + for resolution in resolutions: + domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key]) + if subdomains: + for subdomain in subdomains: + attribute = MISPAttribute() + attribute.from_dict(**dict(type='domain', value=subdomain)) + self.misp_event.add_attribute(**attribute) + domain_ip_object.add_reference(attribute.uuid, 'subdomain') + if uuids: + for uuid in uuids: + domain_ip_object.add_reference(uuid, 'sibling-of') + self.misp_event.add_object(**domain_ip_object) + + def parse_vt_object(self, query_result): + if query_result['response_code'] == 1: + vt_object = MISPObject('virustotal-report') + vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) + detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) + vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) + self.misp_event.add_object(**vt_object) + + def get_query_result(self, query_type): + params = {query_type: self.attribute.value, 'apikey': self.apikey} + return requests.get(self.base_url, params=params) + + +class DomainQuery(VirusTotalParser): + def __init__(self, apikey, attribute): + super(DomainQuery, self).__init__() + self.base_url = "https://www.virustotal.com/vtapi/v2/domain/report" + self.declare_variables(apikey, attribute) + + def parse_report(self, query_result): + hash_type = 'sha256' + whois = 'whois' + for feature_type in ('referrer', 'downloaded', 'communicating'): + for feature in ('undetected_{}_samples', 'detected_{}_samples'): + for sample in query_result.get(feature.format(feature_type), []): + self.misp_event.add_attribute(hash_type, sample[hash_type]) + if query_result.get(whois): + whois_object = MISPObject(whois) + whois_object.add_attribute('text', type='text', value=query_result[whois]) + self.misp_event.add_object(**whois_object) + if 'domain_siblings' in query_result: + siblings = (self.parse_siblings(domain) for domain in query_result['domain_siblings']) + if 'subdomains' in query_result: + self.parse_resolutions(query_result['resolutions'], query_result['subdomains'], siblings) + self.parse_urls(query_result) + + def parse_siblings(self, domain): + attribute = MISPAttribute() + attribute.from_dict(**dict(type='domain', value=domain)) + self.misp_event.add_attribute(**attribute) + return attribute.uuid + + +class HashQuery(VirusTotalParser): + def __init__(self, apikey, attribute): + super(HashQuery, self).__init__() + self.base_url = "https://www.virustotal.com/vtapi/v2/file/report" + self.declare_variables(apikey, attribute) + + def parse_report(self, query_result): + file_attributes = [] + for hash_type in ('md5', 'sha1', 'sha256'): + if query_result.get(hash_type): + file_attributes.append({'type': hash_type, 'object_relation': hash_type, + 'value': query_result[hash_type]}) + if file_attributes: + file_object = MISPObject('file') + for attribute in file_attributes: + file_object.add_attribute(**attribute) + self.misp_event.add_object(**file_object) + self.parse_vt_object(query_result) + + +class IpQuery(VirusTotalParser): + def __init__(self, apikey, attribute): + super(IpQuery, self).__init__() + self.base_url = "https://www.virustotal.com/vtapi/v2/ip-address/report" + self.declare_variables(apikey, attribute) + + def parse_report(self, query_result): + if query_result.get('asn'): + asn_mapping = {'network': ('ip-src', 'subnet-announced'), + 'country': ('text', 'country')} + asn_object = MISPObject('asn') + asn_object.add_attribute('asn', type='AS', value=query_result['asn']) + for key, value in asn_mapping.items(): + if query_result.get(key): + attribute_type, relation = value + asn_object.add_attribute(relation, type=attribute_type, value=query_result[key]) + self.misp_event.add_object(**asn_object) + self.parse_urls(query_result) + if query_result.get('resolutions'): + self.parse_resolutions(query_result['resolutions']) + + +class UrlQuery(VirusTotalParser): + def __init__(self, apikey, attribute): + super(UrlQuery, self).__init__() + self.base_url = "https://www.virustotal.com/vtapi/v2/url/report" + self.declare_variables(apikey, attribute) + + def parse_report(self, query_result): + self.parse_vt_object(query_result) + + +domain = ('domain', DomainQuery) +ip = ('ip', IpQuery) +file = ('resource', HashQuery) +misp_type_mapping = {'domain': domain, 'hostname': domain, 'ip-src': ip, + 'ip-dst': ip, 'md5': file, 'sha1': file, 'sha256': file, + 'url': ('resource', UrlQuery)} + + +def parse_error(status_code): + status_mapping = {204: 'VirusTotal request rate limit exceeded.', + 400: 'Incorrect request, please check the arguments.', + 403: 'You don\'t have enough privileges to make the request.'} + if status_code in status_mapping: + return status_mapping[status_code] + return "VirusTotal may not be accessible." + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('config') or not request['config'].get('apikey'): + misperrors['error'] = "A VirusTotal api key is required for this module." + return misperrors + attribute = request['attribute'] + query_type, to_call = misp_type_mapping[attribute['type']] + parser = to_call(request['config']['apikey'], attribute) + query_result = parser.get_query_result(query_type) + status_code = query_result.status_code + if status_code == 200: + parser.parse_report(query_result.json()) + else: + misperrors['error'] = parse_error(status_code) + return misperrors + return parser.get_result() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/vmray_submit.py b/misp_modules/modules/expansion/vmray_submit.py index 15b163b..1c0d553 100644 --- a/misp_modules/modules/expansion/vmray_submit.py +++ b/misp_modules/modules/expansion/vmray_submit.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 ''' -Submit sample to VMRay. +Submit sample to VMRay. -Submit a sample to VMRay +Requires "vmray_rest_api" -TODO: - # Deal with archive submissions +The expansion module vmray_submit and import module vmray_import are a two step +process to import data from VMRay. +You can automate this by setting the PyMISP example script 'vmray_automation' +as a cron job ''' import json import base64 +from distutils.util import strtobool import io import zipfile @@ -20,7 +23,7 @@ from ._vmray.vmray_rest_api import VMRayRESTAPI misperrors = {'error': 'Error'} mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']} -moduleinfo = {'version': '0.2', 'author': 'Koen Van Impe', +moduleinfo = {'version': '0.3', 'author': 'Koen Van Impe', 'description': 'Submit a sample to VMRay', 'module-type': ['expansion']} moduleconfig = ['apikey', 'url', 'shareable', 'do_not_reanalyze', 'do_not_include_vmrayjobids'] @@ -40,12 +43,12 @@ def handler(q=False): data = request.get("data") if 'malware-sample' in request: # malicious samples are encrypted with zip (password infected) and then base64 encoded - sample_filename = request.get("malware-sample").split("|",1)[0] + sample_filename = request.get("malware-sample").split("|", 1)[0] data = base64.b64decode(data) fl = io.BytesIO(data) zf = zipfile.ZipFile(fl) sample_hashname = zf.namelist()[0] - data = zf.read(sample_hashname,b"infected") + data = zf.read(sample_hashname, b"infected") zf.close() elif 'attachment' in request: # All attachments get base64 encoded @@ -55,7 +58,7 @@ def handler(q=False): else: misperrors['error'] = "No malware sample or attachment supplied" return misperrors - except: + except Exception: misperrors['error'] = "Unable to process submited sample data" return misperrors @@ -69,25 +72,13 @@ def handler(q=False): do_not_reanalyze = request["config"].get("do_not_reanalyze") do_not_include_vmrayjobids = request["config"].get("do_not_include_vmrayjobids") - # Do we want the sample to be shared? - if shareable == "True": - shareable = True - else: - shareable = False - - # Always reanalyze the sample? - if do_not_reanalyze == "True": - do_not_reanalyze = True - else: - do_not_reanalyze = False - reanalyze = not do_not_reanalyze - - # Include the references to VMRay job IDs - if do_not_include_vmrayjobids == "True": - do_not_include_vmrayjobids = True - else: - do_not_include_vmrayjobids = False - include_vmrayjobids = not do_not_include_vmrayjobids + try: + shareable = bool(strtobool(shareable)) # Do we want the sample to be shared? + reanalyze = not bool(strtobool(do_not_reanalyze)) # Always reanalyze the sample? + include_vmrayjobids = not bool(strtobool(do_not_include_vmrayjobids)) # Include the references to VMRay job IDs + except ValueError: + misperrors["error"] = "Error while processing settings. Please double-check your values." + return misperrors if data and sample_filename: args = {} @@ -97,12 +88,12 @@ def handler(q=False): try: vmraydata = vmraySubmit(api, args) - if vmraydata["errors"]: + if vmraydata["errors"] and "Submission not stored" not in vmraydata["errors"][0]["error_msg"]: misperrors['error'] = "VMRay: %s" % vmraydata["errors"][0]["error_msg"] return misperrors else: return vmrayProcess(vmraydata) - except: + except Exception: misperrors['error'] = "Problem when calling API." return misperrors else: @@ -123,22 +114,20 @@ def vmrayProcess(vmraydata): ''' Process the JSON file returned by vmray''' if vmraydata: try: - submissions = vmraydata["submissions"][0] + sample = vmraydata["samples"][0] jobs = vmraydata["jobs"] # Result received? - if submissions and jobs: + if sample: r = {'results': []} - r["results"].append({"types": "md5", "values": submissions["submission_sample_md5"]}) - r["results"].append({"types": "sha1", "values": submissions["submission_sample_sha1"]}) - r["results"].append({"types": "sha256", "values": submissions["submission_sample_sha256"]}) - r["results"].append({"types": "text", "values": "VMRay Sample ID: %s" % submissions["submission_sample_id"]}) - r["results"].append({"types": "text", "values": "VMRay Submission ID: %s" % submissions["submission_id"]}) - r["results"].append({"types": "text", "values": "VMRay Submission Sample IP: %s" % submissions["submission_ip_ip"]}) - r["results"].append({"types": "link", "values": submissions["submission_webif_url"]}) + r['results'].append({'types': 'md5', 'values': sample['sample_md5hash']}) + r['results'].append({'types': 'sha1', 'values': sample['sample_sha1hash']}) + r['results'].append({'types': 'sha256', 'values': sample['sample_sha256hash']}) + r['results'].append({'types': 'text', 'values': 'VMRay Sample ID: %s' % sample['sample_id'], 'tags': 'workflow:state="incomplete"'}) + r['results'].append({'types': 'link', 'values': sample['sample_webif_url']}) # Include data from different jobs - if include_vmrayjobids: + if include_vmrayjobids and len(jobs) > 0: for job in jobs: job_id = job["job_id"] job_vm_name = job["job_vm_name"] @@ -148,7 +137,7 @@ def vmrayProcess(vmraydata): else: misperrors['error'] = "No valid results returned." return misperrors - except: + except Exception: misperrors['error'] = "No valid submission data returned." return misperrors else: diff --git a/misp_modules/modules/expansion/vulndb.py b/misp_modules/modules/expansion/vulndb.py index 6476199..db6c461 100644 --- a/misp_modules/modules/expansion/vulndb.py +++ b/misp_modules/modules/expansion/vulndb.py @@ -24,8 +24,8 @@ log.addHandler(ch) misperrors = {'error': 'Error'} mispattributes = { - 'input': ['vulnerability'], - 'output': ['text', 'link', 'cpe']} + 'input': ['vulnerability'], + 'output': ['text', 'link', 'cpe']} moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe', 'description': 'Query VulnDB - RiskBasedSecurity.com', 'module-type': ['expansion', 'hover']} @@ -61,7 +61,7 @@ def handler(q=False): add_dates = True add_ext_references = True - if request["config"].get("discard_dates") is not None and request["config"].get("discard_dates").lower() == "true": + if request["config"].get("discard_dates") is not None and request["config"].get("discard_dates").lower() == "true": add_dates = False if request["config"].get("discard_external_references") is not None and request["config"].get("discard_external_references").lower() == "true": add_ext_references = False @@ -80,7 +80,7 @@ def handler(q=False): find_by_cve_url = "%s/api/v1/vulnerabilities/%s/find_by_cve_id%s" % (VULNDB_URL, vulnerability, cpu_vulndb) log.debug(find_by_cve_url) - + try: consumer = oauth.Consumer(key=apikey, secret=apisecret) @@ -116,7 +116,7 @@ def handler(q=False): if t_description: values_text.append(t_description) if manual_notes: - values_text.append("Notes: " + manual_notes) + values_text.append("Notes: " + manual_notes) if keywords: values_text.append("Keywords: " + keywords) if solution: @@ -130,22 +130,22 @@ def handler(q=False): values_text.append("Solution date: " + solution_date) disclosure_date = results.get('disclosure_date', '') or '' if disclosure_date: - values_text.append("Disclosure date: " + disclosure_date) + values_text.append("Disclosure date: " + disclosure_date) discovery_date = results.get('discovery_date', '') or '' if discovery_date: - values_text.append("Discovery date: " + discovery_date) + values_text.append("Discovery date: " + discovery_date) exploit_publish_date = results.get('exploit_publish_date', '') or '' if exploit_publish_date: - values_text.append("Exploit published date: " + exploit_publish_date) + values_text.append("Exploit published date: " + exploit_publish_date) vendor_informed_date = results.get('vendor_informed_date', '') or '' if vendor_informed_date: - values_text.append("Vendor informed date: " + vendor_informed_date) + values_text.append("Vendor informed date: " + vendor_informed_date) vendor_ack_date = results.get('vendor_ack_date', '') or '' if vendor_ack_date: - values_text.append("Vendor acknowledgement date: " + vendor_ack_date) + values_text.append("Vendor acknowledgement date: " + vendor_ack_date) third_party_solution_date = results.get('third_party_solution_date', '') or '' if third_party_solution_date: - values_text.append("Third party solution date: " + third_party_solution_date) + values_text.append("Third party solution date: " + third_party_solution_date) # External references if add_ext_references: @@ -159,7 +159,7 @@ def handler(q=False): elif reference_type == "News Article": values_links.append(reference["value"]) elif reference_type == "Generic Informational URL": - values_links.append(reference["value"]) + values_links.append(reference["value"]) elif reference_type == "Vendor Specific Advisory URL": values_links.append(reference["value"]) elif reference_type == "Vendor URL": @@ -183,7 +183,7 @@ def handler(q=False): values_links.append(reference_link) elif reference_type == "Exploit Database": reference_link = "https://www.exploit-db.com/exploits/%s" % reference["value"] - values_links.append(reference_link) + values_links.append(reference_link) elif reference_type == "Generic Informational URL": values_links.append(reference["value"]) elif reference_type == "Generic Informational URL": @@ -260,17 +260,17 @@ def handler(q=False): values_text.append(vulnerability_classification) # Finished processing the VulnDB reply; set the result for MISP - output['results'] += [{'types': 'text', 'values': values_text }] - output['results'] += [{'types': 'link', 'values': values_links }] + output['results'] += [{'types': 'text', 'values': values_text}] + output['results'] += [{'types': 'link', 'values': values_links}] if add_cpe: - output['results'] += [{'types': 'cpe', 'values': values_cpe }] + output['results'] += [{'types': 'cpe', 'values': values_cpe}] return output else: misperrors["error"] = "No information retrieved from VulnDB." return misperrors - except: + except Exception: misperrors["error"] = "Error while fetching information from VulnDB, wrong API keys?" - return misperrors + return misperrors def introspection(): diff --git a/misp_modules/modules/expansion/vulners.py b/misp_modules/modules/expansion/vulners.py new file mode 100644 index 0000000..c2ec7de --- /dev/null +++ b/misp_modules/modules/expansion/vulners.py @@ -0,0 +1,67 @@ +import json +import vulners + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['vulnerability'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Igor Ivanov', 'description': 'An expansion hover module to expand information about CVE id using Vulners API.', 'module-type': ['hover']} + +# Get API key from https://vulners.com/userinfo +moduleconfig = ["apikey"] + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('vulnerability'): + misperrors['error'] = 'Vulnerability id missing' + return misperrors + + ai_summary = '' + exploit_summary = '' + vuln_summary = '' + + if not request.get('config') or not request['config'].get('apikey'): + return {'error': "A Vulners api key is required for this module."} + + key = request['config']['apikey'] + vulners_api = vulners.Vulners(api_key=key) + vulnerability = request.get('vulnerability') + vulners_document = vulners_api.document(vulnerability) + + # Get AI scoring from the document if it's already calculated + # There is no need to call AI Scoring method + if 'score' in vulners_document.get('enchantments', {}): + vulners_ai_score = vulners_document['enchantments']['score']['value'] + else: + vulners_ai_score = None + + vulners_exploits = vulners_api.searchExploit(vulnerability) + + if vulners_document: + vuln_summary += vulners_document.get('description') + else: + vuln_summary += 'Non existing CVE' + + if vulners_ai_score: + ai_summary += 'Vulners AI Score is ' + str(vulners_ai_score[0]) + " " + + if vulners_exploits: + exploit_summary += " || " + str(len(vulners_exploits)) + " Public exploits available:\n " + for exploit in vulners_exploits: + exploit_summary += exploit['title'] + " " + exploit['href'] + "\n " + exploit_summary += "|| Vulnerability Description: " + vuln_summary + + summary = ai_summary + exploit_summary + vuln_summary + + r = {'results': [{'types': mispattributes['output'], 'values': summary}]} + return r + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/whois.py b/misp_modules/modules/expansion/whois.py index 4aec40c..22c4850 100755 --- a/misp_modules/modules/expansion/whois.py +++ b/misp_modules/modules/expansion/whois.py @@ -29,8 +29,8 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('apikey') and request['config'].et('url')): - misperrors['error'] = 'EUPI authentication is missing' + if not request.get('config') or (not request['config'].get('server') and not request['config'].get('port')): + misperrors['error'] = 'Whois local instance address is missing' return misperrors uwhois = Uwhois(request['config']['server'], int(request['config']['port'])) diff --git a/misp_modules/modules/expansion/wiki.py b/misp_modules/modules/expansion/wiki.py index 85279b1..90dd547 100755 --- a/misp_modules/modules/expansion/wiki.py +++ b/misp_modules/modules/expansion/wiki.py @@ -1,5 +1,4 @@ import json -import requests from SPARQLWrapper import SPARQLWrapper, JSON misperrors = {'error': 'Error'} @@ -17,23 +16,22 @@ def handler(q=False): if not request.get('text'): misperrors['error'] = 'Query text missing' return misperrors - + sparql = SPARQLWrapper(wiki_api_url) query_string = \ - "SELECT ?item \n" \ - "WHERE { \n" \ - "?item rdfs:label\"" + request.get('text') + "\" @en \n" \ - "}\n"; + "SELECT ?item \n" \ + "WHERE { \n" \ + "?item rdfs:label\"" + request.get('text') + "\" @en \n" \ + "}\n" sparql.setQuery(query_string) sparql.setReturnFormat(JSON) results = sparql.query().convert() - summary = '' try: - result = results["results"]["bindings"][0] - summary = result["item"]["value"] + result = results["results"]["bindings"] + summary = result[0]["item"]["value"] if result else 'No additional data found on Wikidata' except Exception as e: - misperrors['error'] = 'wikidata API not accessible' + e + misperrors['error'] = 'wikidata API not accessible {}'.format(e) return misperrors['error'] r = {'results': [{'types': mispattributes['output'], 'values': summary}]} @@ -47,4 +45,3 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo - diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py index d027f99..7999ce2 100644 --- a/misp_modules/modules/expansion/xforceexchange.py +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -1,101 +1,174 @@ -import requests -import json -import sys - -BASEurl = "https://api.xforce.ibmcloud.com/" - -extensions = {"ip1": "ipr/%s", - "ip2": "ipr/malware/%s", - "url": "url/%s", - "hash": "malware/%s", - "vuln": "/vulnerabilities/search/%s", - "dns": "resolve/%s"} - -sys.path.append('./') - -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst', 'vulnerability', 'md5', 'sha1', 'sha256'], - 'output': ['ip-src', 'ip-dst', 'text', 'domain']} - -# possible module-types: 'expansion', 'hover' or both -moduleinfo = {'version': '1', 'author': 'Joerg Stephan (@johest)', - 'description': 'IBM X-Force Exchange expansion module', - 'module-type': ['expansion', 'hover']} - -# config fields that your code expects from the site admin -moduleconfig = ["apikey", "event_limit"] -limit = 5000 #Default - - - -def MyHeader(key=False): - global limit - if key is False: - return None - - return {"Authorization": "Basic %s " % key, - "Accept": "application/json", - 'User-Agent': 'Mozilla 5.0'} - -def handler(q=False): - global limit - if q is False: - return False - - q = json.loads(q) - - key = q["config"]["apikey"] - limit = int(q["config"].get("event_limit", 5)) - - r = {"results": []} - - if "ip-src" in q: - r["results"] += apicall("dns", q["ip-src"], key) - if "ip-dst" in q: - r["results"] += apicall("dns", q["ip-dst"], key) - if "md5" in q: - r["results"] += apicall("hash", q["md5"], key) - if "sha1" in q: - r["results"] += apicall("hash", q["sha1"], key) - if "sha256" in q: - r["results"] += apicall("hash", q["sha256"], key) - if 'vulnerability' in q: - r["results"] += apicall("vuln", q["vulnerability"], key) - if "domain" in q: - r["results"] += apicall("dns", q["domain"], key) - - uniq = [] - for res in r["results"]: - if res not in uniq: - uniq.append(res) - r["results"] = uniq - return r - -def apicall(indicator_type, indicator, key=False): - try: - myURL = BASEurl + (extensions[str(indicator_type)])%indicator - jsondata = requests.get(myURL, headers=MyHeader(key)).json() - except: - jsondata = None - redata = [] - #print(jsondata) - if not jsondata is None: - if indicator_type is "hash": - if "malware" in jsondata: - lopointer = jsondata["malware"] - redata.append({"type": "text", "values": lopointer["risk"]}) - if indicator_type is "dns": - if "records" in str(jsondata): - lopointer = jsondata["Passive"]["records"] - for dataset in lopointer: - redata.append({"type":"domain", "values": dataset["value"]}) - - return redata - -def introspection(): - return mispattributes - - -def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo +import requests +import json +import sys +from collections import defaultdict +from pymisp import MISPAttribute, MISPEvent, MISPObject +from requests.auth import HTTPBasicAuth + +sys.path.append('./') + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['ip-src', 'ip-dst', 'vulnerability', 'md5', 'sha1', 'sha256', 'domain', 'hostname', 'url'], + 'output': ['ip-src', 'ip-dst', 'text', 'domain'], + 'format': 'misp_standard'} + +# possible module-types: 'expansion', 'hover' or both +moduleinfo = {'version': '2', 'author': 'Joerg Stephan (@johest)', + 'description': 'IBM X-Force Exchange expansion module', + 'module-type': ['expansion', 'hover']} + +# config fields that your code expects from the site admin +moduleconfig = ["apikey", "apipassword"] + + +class XforceExchange(): + def __init__(self, attribute, apikey, apipassword): + self.base_url = "https://api.xforce.ibmcloud.com" + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self._apikey = apikey + self._apipassword = apipassword + self.result = {} + self.objects = defaultdict(dict) + self.status_mapping = {403: "Access denied, please check if your authentication is valid and if you did not reach the limit of queries.", + 404: "No result found for your query."} + + def parse(self): + mapping = {'url': '_parse_url', 'vulnerability': '_parse_vulnerability'} + mapping.update(dict.fromkeys(('md5', 'sha1', 'sha256'), '_parse_hash')) + mapping.update(dict.fromkeys(('domain', 'hostname'), '_parse_dns')) + mapping.update(dict.fromkeys(('ip-src', 'ip-dst'), '_parse_ip')) + to_call = mapping[self.attribute.type] + getattr(self, to_call)(self.attribute.value) + + def get_result(self): + if not self.misp_event.objects: + if 'error' not in self.result: + self.result['error'] = "No additional data found on Xforce Exchange." + return self.result + self.misp_event.add_attribute(**self.attribute) + event = json.loads(self.misp_event.to_json()) + result = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': result} + + def _api_call(self, url): + try: + result = requests.get(url, auth=HTTPBasicAuth(self._apikey, self._apipassword)) + except Exception as e: + self.result['error'] = e + return + status_code = result.status_code + if status_code != 200: + try: + self.result['error'] = self.status_mapping[status_code] + except KeyError: + self.result['error'] = 'An error with the API has occurred.' + return + return result.json() + + def _create_file(self, malware, relationship): + file_object = MISPObject('file') + for key, relation in zip(('filepath', 'md5'), ('filename', 'md5')): + file_object.add_attribute(relation, malware[key]) + file_object.add_reference(self.attribute.uuid, relationship) + return file_object + + def _create_url(self, malware): + url_object = MISPObject('url') + for key, relation in zip(('uri', 'domain'), ('url', 'domain')): + url_object.add_attribute(relation, malware[key]) + attributes = tuple(f'{attribute.object_relation}_{attribute.value}' for attribute in url_object.attributes) + if attributes in self.objects['url']: + del url_object + return self.objects['url'][attributes] + url_uuid = url_object.uuid + self.misp_event.add_object(**url_object) + self.objects['url'][attributes] = url_uuid + return url_uuid + + def _fetch_types(self, value): + if self.attribute.type in ('ip-src', 'ip-dst'): + return 'ip', 'domain', self.attribute.value + return 'domain', 'ip', value + + def _handle_file(self, malware, relationship): + file_object = self._create_file(malware, relationship) + attributes = tuple(f'{attribute.object_relation}_{attribute.value}' for attribute in file_object.attributes) + if attributes in self.objects['file']: + self.objects['file'][attributes].add_reference(self._create_url(malware), 'dropped-by') + del file_object + return + file_object.add_reference(self._create_url(malware), 'dropped-by') + self.objects['file'][attributes] = file_object + self.misp_event.add_object(**file_object) + + def _parse_dns(self, value): + dns_result = self._api_call(f'{self.base_url}/resolve/{value}') + if dns_result.get('Passive') and dns_result['Passive'].get('records'): + itype, ftype, value = self._fetch_types(dns_result['Passive']['query']) + misp_object = MISPObject('domain-ip') + misp_object.add_attribute(itype, value) + for record in dns_result['Passive']['records']: + misp_object.add_attribute(ftype, record['value']) + misp_object.add_reference(self.attribute.uuid, 'related-to') + self.misp_event.add_object(**misp_object) + + def _parse_hash(self, value): + malware_result = self._api_call(f'{self.base_url}/malware/{value}') + if malware_result and malware_result.get('malware'): + malware_report = malware_result['malware'] + for malware in malware_report.get('origins', {}).get('CnCServers', {}).get('rows', []): + self._handle_file(malware, 'related-to') + + def _parse_ip(self, value): + self._parse_dns(value) + self._parse_malware(value, 'ipr') + + def _parse_malware(self, value, feature): + malware_result = self._api_call(f'{self.base_url}/{feature}/malware/{value}') + if malware_result and malware_result.get('malware'): + for malware in malware_result['malware']: + self._handle_file(malware, 'associated-with') + + def _parse_url(self, value): + self._parse_dns(value) + self._parse_malware(value, 'url') + + def _parse_vulnerability(self, value): + vulnerability_result = self._api_call(f'{self.base_url}/vulnerabilities/search/{value}') + if vulnerability_result: + for vulnerability in vulnerability_result: + misp_object = MISPObject('vulnerability') + for code in vulnerability['stdcode']: + misp_object.add_attribute('id', code) + for feature, relation in zip(('title', 'description', 'temporal_score'), + ('summary', 'description', 'cvss-score')): + misp_object.add_attribute(relation, vulnerability[feature]) + for reference in vulnerability['references']: + misp_object.add_attribute('references', reference['link_target']) + misp_object.add_reference(self.attribute.uuid, 'related-to') + self.misp_event.add_object(**misp_object) + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('config') or not (request['config'].get('apikey') and request['config'].get('apipassword')): + misperrors['error'] = 'An API authentication is required (key and password).' + return misperrors + key = request["config"]["apikey"] + password = request['config']['apipassword'] + parser = XforceExchange(request['attribute'], key, password) + parser.parse() + return parser.get_result() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/xlsx_enrich.py b/misp_modules/modules/expansion/xlsx_enrich.py new file mode 100644 index 0000000..6e0ee73 --- /dev/null +++ b/misp_modules/modules/expansion/xlsx_enrich.py @@ -0,0 +1,53 @@ +import json +import binascii +import np +import pandas +import io + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': '.xlsx to freetext-import IOC extractor', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + xlsx_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + xls_content = "" + xls_file = io.BytesIO(xlsx_array) + pandas.set_option('display.max_colwidth', -1) + try: + xls = pandas.read_excel(xls_file) + xls_content = xls.to_string(max_rows=None) + print(xls_content) + return {'results': [{'types': ['freetext'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}, + {'types': ['text'], 'values': xls_content, 'comment': ".xlsx-to-text from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file as .xlsx. Error was: " + str(e) + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/yara_query.py b/misp_modules/modules/expansion/yara_query.py new file mode 100644 index 0000000..3a75acc --- /dev/null +++ b/misp_modules/modules/expansion/yara_query.py @@ -0,0 +1,54 @@ +import json +import re +try: + import yara +except (OSError, ImportError): + print("yara is missing, use 'pip3 install -I -r REQUIREMENTS' from the root of this repository to install it.") + +misperrors = {'error': 'Error'} +moduleinfo = {'version': '1', 'author': 'Christian STUDER', + 'description': 'Yara export for hashes.', + 'module-type': ['expansion', 'hover'], + 'require_standard_format': True} +moduleconfig = [] +mispattributes = {'input': ['md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256', 'imphash'], 'output': ['yara']} + + +def get_hash_condition(hashtype, hashvalue): + hashvalue = hashvalue.lower() + required_module, params = ('pe', '()') if hashtype == 'imphash' else ('hash', '(0, filesize)') + return '{}.{}{} == "{}"'.format(required_module, hashtype, params, hashvalue), required_module + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + del request['module'] + if 'event_id' in request: + del request['event_id'] + uuid = request.pop('attribute_uuid') if 'attribute_uuid' in request else None + attribute_type, value = list(request.items())[0] + if 'filename' in attribute_type: + _, attribute_type = attribute_type.split('|') + _, value = value.split('|') + condition, required_module = get_hash_condition(attribute_type, value) + import_section = 'import "{}"'.format(required_module) + rule_start = '%s\r\nrule %s_%s {' % (import_section, attribute_type.upper(), re.sub(r'\W+', '_', uuid)) if uuid else '%s\r\nrule %s {' % (import_section, attribute_type.upper()) + condition = '\tcondition:\r\n\t\t{}'.format(condition) + rule = '\r\n'.join([rule_start, condition, '}']) + try: + yara.compile(source=rule) + except Exception as e: + misperrors['error'] = 'Syntax error: {}'.format(e) + return misperrors + return {'results': [{'types': mispattributes['output'], 'values': rule}]} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/yara_syntax_validator.py b/misp_modules/modules/expansion/yara_syntax_validator.py index 4953c41..cad533f 100644 --- a/misp_modules/modules/expansion/yara_syntax_validator.py +++ b/misp_modules/modules/expansion/yara_syntax_validator.py @@ -1,9 +1,8 @@ import json -import requests try: import yara -except: - print("yara is missing, use 'pip3 install yara' to install it.") +except (OSError, ImportError): + print("yara is missing, use 'pip3 install -I -r REQUIREMENTS' from the root of this repository to install it.") misperrors = {'error': 'Error'} mispattributes = {'input': ['yara'], 'output': ['text']} @@ -20,7 +19,7 @@ def handler(q=False): return misperrors try: - rules = yara.compile(source=request.get('yara')) + yara.compile(source=request.get('yara')) summary = ("Syntax valid") except Exception as e: summary = ("Syntax error: " + str(e)) diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 0034f5d..1b0e1d0 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1 +1,2 @@ -__all__ = ['testexport','cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export'] +__all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', + 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph'] diff --git a/misp_modules/modules/export_mod/cef_export.py b/misp_modules/modules/export_mod/cef_export.py index 3f2ff61..0aa82f0 100755 --- a/misp_modules/modules/export_mod/cef_export.py +++ b/misp_modules/modules/export_mod/cef_export.py @@ -12,30 +12,32 @@ moduleinfo = {'version': '1', 'author': 'Hannah Ward', # config fields that your code expects from the site admin moduleconfig = ["Default_Severity", "Device_Vendor", "Device_Product", "Device_Version"] -cefmapping = {"ip-src":"src", "ip-dst":"dst", "hostname":"dhost", "domain":"dhost", - "md5":"fileHash", "sha1":"fileHash", "sha256":"fileHash", - "url":"request"} +cefmapping = {"ip-src": "src", "ip-dst": "dst", "hostname": "dhost", "domain": "dhost", + "md5": "fileHash", "sha1": "fileHash", "sha256": "fileHash", + "url": "request"} -mispattributes = {'input':list(cefmapping.keys())} +mispattributes = {'input': list(cefmapping.keys())} outputFileExtension = "cef" responseType = "application/txt" + def handler(q=False): if q is False: return False request = json.loads(q) if "config" in request: - config = request["config"] + config = request["config"] else: - config = {"Default_Severity":1, "Device_Vendor":"MISP", "Device_Product":"MISP", "Device_Version":1} + config = {"Default_Severity": 1, "Device_Vendor": "MISP", + "Device_Product": "MISP", "Device_Version": 1} data = request["data"] response = "" for ev in data: - event = ev["Attribute"] - for attr in event: - if attr["type"] in cefmapping: - response += "{} host CEF:0|{}|{}|{}|{}|{}|{}|{}={}\n".format( + event = ev["Attribute"] + for attr in event: + if attr["type"] in cefmapping: + response += "{} host CEF:0|{}|{}|{}|{}|{}|{}|{}={}\n".format( datetime.datetime.fromtimestamp(int(attr["timestamp"])).strftime("%b %d %H:%M:%S"), config["Device_Vendor"], config["Device_Product"], @@ -45,37 +47,37 @@ def handler(q=False): config["Default_Severity"], cefmapping[attr["type"]], attr["value"], - ) - - r = {"response":[], "data":str(base64.b64encode(bytes(response, 'utf-8')), 'utf-8')} + ) + + r = {"response": [], "data": str(base64.b64encode(bytes(response, 'utf-8')), 'utf-8')} return r def introspection(): - modulesetup = {} - try: + modulesetup = {} + try: responseType modulesetup['responseType'] = responseType - except NameError: - pass - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + def version(): moduleinfo['config'] = moduleconfig return moduleinfo - diff --git a/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py new file mode 100644 index 0000000..ab79692 --- /dev/null +++ b/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py @@ -0,0 +1,140 @@ +###################################################### +# # +# Author: Stanislav Klevtsov, Ukraine; Feb 2019. # +# # +# # +# Script was tested on the following configuration: # +# MISP v2.4.90 # +# Cisco Firesight Manager Console v6.2.3 (bld 84) # +# # +###################################################### + +import json +import base64 +from urllib.parse import quote + +misperrors = {'error': 'Error'} + +moduleinfo = {'version': '1', 'author': 'Stanislav Klevtsov', + 'description': 'Export malicious network activity attributes of the MISP event to Cisco firesight manager block rules', + 'module-type': ['export']} + + +moduleconfig = ['fmc_ip_addr', 'fmc_login', 'fmc_pass', 'domain_id', 'acpolicy_id'] + +fsmapping = {"ip-dst": "dst", "url": "request"} + +mispattributes = {'input': list(fsmapping.keys())} + +# options: event, attribute, event-collection, attribute-collection +inputSource = ['event'] + +outputFileExtension = 'sh' +responseType = 'application/txt' + +# .sh file templates +SH_FILE_HEADER = """#!/bin/sh\n\n""" + +BLOCK_JSON_TMPL = """ +BLOCK_RULE='{{ "action": "BLOCK", "enabled": true, "type": "AccessRule", "name": "{rule_name}", "destinationNetworks": {{ "literals": [ {dst_networks} ] }}, "urls": {{ "literals": [ {urls} ] }}, "newComments": [ "{event_info_comment}" ] }}'\n +""" + +BLOCK_DST_JSON_TMPL = """{{ "type": "Host", "value": "{ipdst}" }} """ +BLOCK_URL_JSON_TMPL = """{{ "type": "Url", "url": "{url}" }} """ + +CURL_ADD_RULE_TMPL = """ +curl -X POST -v -k -H 'Content-Type: application/json' -H \"Authorization: Basic $LOGINPASS_BASE64\" -H \"X-auth-access-token: $ACC_TOKEN\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_config/v1/domain/$DOMAIN_ID/policy/accesspolicies/$ACPOLICY_ID/accessrules\" --data \"$BLOCK_RULE\" """ + + +def handler(q=False): + if q is False: + return False + + r = {'results': []} + request = json.loads(q) + + if "config" in request: + config = request["config"] + + # check if config is empty + if not config['fmc_ip_addr']: + config['fmc_ip_addr'] = "0.0.0.0" + if not config['fmc_login']: + config['fmc_login'] = "login" + if not config['fmc_pass']: + config['fmc_pass'] = "password" + if not config['domain_id']: + config['domain_id'] = "SET_FIRESIGHT_DOMAIN_ID" + if not config['acpolicy_id']: + config['acpolicy_id'] = "SET_FIRESIGHT_ACPOLICY_ID" + + data = request["data"] + output = "" + ipdst = [] + urls = [] + + # populate the ACL rule with attributes + for ev in data: + + event = ev["Attribute"] + event_id = ev["Event"]["id"] + event_info = ev["Event"]["info"] + + for index, attr in enumerate(event): + if attr["to_ids"] is True: + if attr["type"] in fsmapping: + if attr["type"] == "ip-dst": + ipdst.append(BLOCK_DST_JSON_TMPL.format(ipdst=attr["value"])) + else: + urls.append(BLOCK_URL_JSON_TMPL.format(url=quote(attr["value"], safe='@/:;?&=-_.,+!*'))) + + # building the .sh file + output += SH_FILE_HEADER + output += "FIRESIGHT_IP_ADDR='{}'\n".format(config['fmc_ip_addr']) + + output += "LOGINPASS_BASE64=`echo -n '{}:{}' | base64`\n".format(config['fmc_login'], config['fmc_pass']) + output += "DOMAIN_ID='{}'\n".format(config['domain_id']) + output += "ACPOLICY_ID='{}'\n\n".format(config['acpolicy_id']) + + output += "ACC_TOKEN=`curl -X POST -v -k -sD - -o /dev/null -H \"Authorization: Basic $LOGINPASS_BASE64\" -i \"https://$FIRESIGHT_IP_ADDR/api/fmc_platform/v1/auth/generatetoken\" | grep -i x-auth-acc | sed 's/.*:\\ //g' | tr -d '[:space:]' | tr -d '\\n'`\n" + + output += BLOCK_JSON_TMPL.format(rule_name="misp_event_{}".format(event_id), + dst_networks=', '.join(ipdst), + urls=', '.join(urls), + event_info_comment=event_info) + "\n" + + output += CURL_ADD_RULE_TMPL + # END building the .sh file + + r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')} + return r + + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/goamlexport.py b/misp_modules/modules/export_mod/goamlexport.py index c277640..b9ce086 100644 --- a/misp_modules/modules/export_mod/goamlexport.py +++ b/misp_modules/modules/export_mod/goamlexport.py @@ -1,4 +1,5 @@ -import json, datetime, base64 +import json +import base64 from pymisp import MISPEvent from collections import defaultdict, Counter @@ -26,7 +27,7 @@ goAMLmapping = {'bank-account': {'bank-account': 't_account', 'institution-name' 'person': {'person': 't_person', 'text': 'comments', 'first-name': 'first_name', 'middle-name': 'middle_name', 'last-name': 'last_name', 'title': 'title', 'mothers-name': 'mothers_name', 'alias': 'alias', 'date-of-birth': 'birthdate', - 'place-of-birth': 'birth_place', 'gender': 'gender','nationality': 'nationality1', + 'place-of-birth': 'birth_place', 'gender': 'gender', 'nationality': 'nationality1', 'passport-number': 'passport_number', 'passport-country': 'passport_country', 'social-security-number': 'ssn', 'identity-card-number': 'id_number'}, 'geolocation': {'geolocation': 'location', 'city': 'city', 'region': 'state', @@ -47,6 +48,7 @@ referencesMapping = {'bank-account': {'aml_type': '{}_account', 'bracket': 't_{} 'legal-entity': {'transaction': {'aml_type': '{}_entity', 'bracket': 't_{}'}, 'bank-account': {'aml_type': 't_entity'}}, 'geolocation': {'aml_type': 'address', 'bracket': 'addresses'}} + class GoAmlGeneration(object): def __init__(self, config): self.config = config @@ -67,7 +69,7 @@ class GoAmlGeneration(object): try: report_code.append(obj.get_attributes_by_relation('report-code')[0].value.split(' ')[0]) currency_code.append(obj.get_attributes_by_relation('currency-code')[0].value) - except: + except IndexError: print('report_code or currency_code error') self.uuids, self.report_codes, self.currency_codes = uuids, report_code, currency_code @@ -87,9 +89,12 @@ class GoAmlGeneration(object): person_to_parse = [person_uuid for person_uuid in self.uuids.get('person') if person_uuid not in self.parsed_uuids.get('person')] if len(person_to_parse) == 1: self.itterate('person', 'reporting_person', person_to_parse[0], 'header') - location_to_parse = [location_uuid for location_uuid in self.uuids.get('geolocation') if location_uuid not in self.parsed_uuids.get('geolocation')] - if len(location_to_parse) == 1: - self.itterate('geolocation', 'location', location_to_parse[0], 'header') + try: + location_to_parse = [location_uuid for location_uuid in self.uuids.get('geolocation') if location_uuid not in self.parsed_uuids.get('geolocation')] + if len(location_to_parse) == 1: + self.itterate('geolocation', 'location', location_to_parse[0], 'header') + except TypeError: + pass self.xml['data'] += "" def itterate(self, object_type, aml_type, uuid, xml_part): @@ -182,6 +187,7 @@ class GoAmlGeneration(object): self.itterate(next_object_type, next_aml_type, uuid, xml_part) self.xml[xml_part] += "".format(bracket) + def handler(q=False): if q is False: return False @@ -208,6 +214,7 @@ def handler(q=False): exp_doc = "{}{}".format(export_doc.xml.get('header'), export_doc.xml.get('data')) return {'response': [], 'data': str(base64.b64encode(bytes(exp_doc, 'utf-8')), 'utf-8')} + def introspection(): modulesetup = {} try: @@ -232,6 +239,7 @@ def introspection(): pass return modulesetup + def version(): moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/export_mod/liteexport.py b/misp_modules/modules/export_mod/liteexport.py index 5d47489..870f52a 100755 --- a/misp_modules/modules/export_mod/liteexport.py +++ b/misp_modules/modules/export_mod/liteexport.py @@ -3,10 +3,10 @@ import base64 misperrors = {'error': 'Error'} -moduleinfo = {'version': '1', - 'author': 'TM', - 'description': 'export lite', - 'module-type': ['export']} +moduleinfo = {'version': '1', + 'author': 'TM', + 'description': 'export lite', + 'module-type': ['export']} moduleconfig = ["indent_json_export"] @@ -14,76 +14,75 @@ mispattributes = {} outputFileExtension = "json" responseType = "application/json" + def handler(q=False): - if q is False: - return False + if q is False: + return False - request = json.loads(q) + request = json.loads(q) - config = {} - if "config" in request: - config = request["config"] - else: - config = {"indent_json_export" : None} + config = {} + if "config" in request: + config = request["config"] + else: + config = {"indent_json_export": None} - if config['indent_json_export'] is not None: - try: - config['indent_json_export'] = int(config['indent_json_export']) - except: - config['indent_json_export'] = None + if config['indent_json_export'] is not None: + try: + config['indent_json_export'] = int(config['indent_json_export']) + except Exception: + config['indent_json_export'] = None - if 'data' not in request: - return False + if 'data' not in request: + return False - #~ Misp json structur - liteEvent = {'Event':{}} + # ~ Misp json structur + liteEvent = {'Event': {}} - for evt in request['data']: - rawEvent = evt['Event'] - liteEvent['Event']['info'] = rawEvent['info'] - liteEvent['Event']['Attribute'] = [] - - attrs = evt['Attribute'] - for attr in attrs: - if 'Internal reference' not in attr['category']: - liteAttr = {} - liteAttr['category'] = attr['category'] - liteAttr['type'] = attr['type'] - liteAttr['value'] = attr['value'] - liteEvent['Event']['Attribute'].append(liteAttr) + for evt in request['data']: + rawEvent = evt['Event'] + liteEvent['Event']['info'] = rawEvent['info'] + liteEvent['Event']['Attribute'] = [] + + attrs = evt['Attribute'] + for attr in attrs: + if 'Internal reference' not in attr['category']: + liteAttr = {} + liteAttr['category'] = attr['category'] + liteAttr['type'] = attr['type'] + liteAttr['value'] = attr['value'] + liteEvent['Event']['Attribute'].append(liteAttr) + + return {'response': [], + 'data': str(base64.b64encode(bytes( + json.dumps(liteEvent, indent=config['indent_json_export']), 'utf-8')), 'utf-8')} - return {'response' : [], - 'data' : str(base64.b64encode( - bytes( - json.dumps(liteEvent, indent=config['indent_json_export']), - 'utf-8')), - 'utf-8') - } def introspection(): - modulesetup = {} - try: - responseType - modulesetup['responseType'] = responseType - except NameError: - pass - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/mass_eql_export.py b/misp_modules/modules/export_mod/mass_eql_export.py new file mode 100644 index 0000000..f42874d --- /dev/null +++ b/misp_modules/modules/export_mod/mass_eql_export.py @@ -0,0 +1,99 @@ +""" +Export module for converting MISP events into Endgame EQL queries +""" +import base64 +import io +import json +import logging + +misperrors = {"error": "Error"} + +moduleinfo = { + "version": "0.1", + "author": "92 COS DOM", + "description": "Export MISP event in Event Query Language", + "module-type": ["export"] +} + +# Map of MISP fields => Endgame fields +fieldmap = { + "ip-src": "source_address", + "ip-dst": "destination_address", + "filename": "file_name" +} + +# Describe what events have what fields +event_types = { + "source_address": "network", + "destination_address": "network", + "file_name": "file" +} + +# combine all the MISP fields from fieldmap into one big list +mispattributes = { + "input": list(fieldmap.keys()) +} + + +def handler(q=False): + """ + Convert a MISP query into a CSV file matching the ThreatConnect Structured Import file format. + Input + q: Query dictionary + """ + if q is False or not q: + return False + + # Check if we were given a configuration + request = json.loads(q) + config = request.get("config", {"Default_Source": ""}) + logging.info("Setting config to: %s", config) + + response = io.StringIO() + + # start parsing MISP data + queryDict = {} + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute["type"] in mispattributes["input"]: + logging.debug("Adding %s to EQL query", attribute["value"]) + event_type = event_types[fieldmap[attribute["type"]]] + if event_type not in queryDict.keys(): + queryDict[event_type] = {} + queryDict[event_type][attribute["value"]] = fieldmap[attribute["type"]] + i = 0 + for query in queryDict.keys(): + response.write("{} where\n".format(query)) + for value in queryDict[query].keys(): + if i != 0: + response.write(" or\n") + response.write("\t{} == \"{}\"".format(queryDict[query][value], value)) + i += 1 + + return {"response": [], "data": str(base64.b64encode(bytes(response.getvalue(), 'utf-8')), 'utf-8')} + + +def introspection(): + """ + Relay the supported attributes to MISP. + No Input + Output + Dictionary of supported MISP attributes + """ + modulesetup = { + "responseType": "application/txt", + "outputFileExtension": "txt", + "userConfig": {}, + "inputSource": [] + } + return modulesetup + + +def version(): + """ + Relay module version and associated metadata to MISP. + No Input + Output + moduleinfo: metadata output containing all potential configuration values + """ + return moduleinfo diff --git a/misp_modules/modules/export_mod/nexthinkexport.py b/misp_modules/modules/export_mod/nexthinkexport.py new file mode 100755 index 0000000..c87b3fb --- /dev/null +++ b/misp_modules/modules/export_mod/nexthinkexport.py @@ -0,0 +1,121 @@ +""" +Export module for coverting MISP events into Nexthink NXQL queries. +Source: https://github.com/HacknowledgeCH/misp-modules/blob/master/misp_modules/modules/export_mod/nexthinkexport.py +Config['Period'] : allows to define period over witch to look for IOC from now (15m, 1d, 2w, 30d, ...), see Nexthink data model documentation +""" + +import base64 +import json + +misperrors = {"error": "Error"} + +types_to_use = ['sha1', 'sha256', 'md5', 'domain'] + +userConfig = { + +} + +moduleconfig = ["Period"] +inputSource = ['event'] + +outputFileExtension = 'nxql' +responseType = 'application/txt' + +moduleinfo = {'version': '1.0', 'author': 'Julien Bachmann, Hacknowledge', + 'description': 'Nexthink NXQL query export module', + 'module-type': ['export']} + + +def handle_sha1(value, period): + query = '''select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) +(from (binary user device execution) +(where binary (eq sha1 (sha1 %s))) +(between now-%s now)) +(limit 1000) + ''' % (value, period) + return query.replace('\n', ' ') + + +def handle_sha256(value, period): + query = '''select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) +(from (binary user device execution) +(where binary (eq sha256 (sha256 %s))) +(between now-%s now)) +(limit 1000) + ''' % (value, period) + return query.replace('\n', ' ') + + +def handle_md5(value, period): + query = '''select ((binary (executable_name version)) (user (name)) (device (name last_ip_address)) (execution (binary_path start_time))) +(from (binary user device execution) +(where binary (eq hash (md5 %s))) +(between now-%s now)) +(limit 1000) + ''' % (value, period) + return query.replace('\n', ' ') + + +def handle_domain(value, period): + query = '''select ((device name) (device (name last_ip_address)) (user name)(user department) (binary executable_name)(binary application_name)(binary description)(binary application_category)(binary (executable_name version)) (binary #"Suspicious binary")(binary first_seen)(binary last_seen)(binary threat_level)(binary hash) (binary paths) +(destination name)(domain name) (domain domain_category)(domain hosting_country)(domain protocol)(domain threat_level) (port port_number)(web_request incoming_traffic)(web_request outgoing_traffic)) +(from (web_request device user binary executable destination domain port) +(where domain (eq name(string %s))) +(between now-%s now)) +(limit 1000) + ''' % (value, period) + return query.replace('\n', ' ') + + +handlers = { + 'sha1': handle_sha1, + 'sha256': handle_sha256, + 'md5': handle_md5, + 'domain': handle_domain +} + + +def handler(q=False): + if q is False: + return False + r = {'results': []} + request = json.loads(q) + config = request.get("config", {"Period": ""}) + output = '' + + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute['type'] in types_to_use: + output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n' + r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')} + return r + + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/osqueryexport.py b/misp_modules/modules/export_mod/osqueryexport.py new file mode 100755 index 0000000..6368875 --- /dev/null +++ b/misp_modules/modules/export_mod/osqueryexport.py @@ -0,0 +1,115 @@ +""" +Export module for coverting MISP events into OSQuery queries. +Source: https://github.com/0xmilkmix/misp-modules/blob/master/misp_modules/modules/export_mod/osqueryexport.py +""" + +import base64 +import json +import re + +misperrors = {"error": "Error"} + +types_to_use = ['regkey', 'regkey|value', 'mutex', 'windows-service-displayname', 'windows-scheduled-task', 'yara'] + +userConfig = { + +} + +moduleconfig = [] +inputSource = ['event'] + +outputFileExtension = 'conf' +responseType = 'application/txt' + + +moduleinfo = {'version': '1.0', 'author': 'Julien Bachmann, Hacknowledge', + 'description': 'OSQuery query export module', + 'module-type': ['export']} + + +def handle_regkey(value): + rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'} + rep = dict((re.escape(k), v) for k, v in rep.items()) + pattern = re.compile("|".join(rep.keys())) + value = pattern.sub(lambda m: rep[re.escape(m.group(0))], value) + return 'SELECT * FROM registry WHERE path LIKE \'%s\';' % value + + +def handle_regkeyvalue(value): + key, value = value.split('|') + rep = {'HKCU': 'HKEY_USERS\\%', 'HKLM': 'HKEY_LOCAL_MACHINE'} + rep = dict((re.escape(k), v) for k, v in rep.items()) + pattern = re.compile("|".join(rep.keys())) + key = pattern.sub(lambda m: rep[re.escape(m.group(0))], key) + return 'SELECT * FROM registry WHERE path LIKE \'%s\' AND data LIKE \'%s\';' % (key, value) + + +def handle_mutex(value): + return 'SELECT * FROM winbaseobj WHERE object_name LIKE \'%s\';' % value + + +def handle_service(value): + return 'SELECT * FROM services WHERE display_name LIKE \'%s\' OR name like \'%s\';' % (value, value) + + +def handle_yara(value): + return 'not implemented yet, not sure it\'s easily feasible w/o dropping the sig on the hosts first' + + +def handle_scheduledtask(value): + return 'SELECT * FROM scheduled_tasks WHERE name LIKE \'%s\';' % value + + +handlers = { + 'regkey': handle_regkey, + 'regkey|value': handle_regkeyvalue, + 'mutex': handle_mutex, + 'windows-service-displayname': handle_service, + 'windows-scheduled-task': handle_scheduledtask, + 'yara': handle_yara +} + + +def handler(q=False): + if q is False: + return False + r = {'results': []} + request = json.loads(q) + output = '' + + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute['type'] in types_to_use: + output = output + handlers[attribute['type']](attribute['value']) + '\n' + r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')} + return r + + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 2aeaec7..44b3bc9 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -1,67 +1,29 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from datetime import date import json -import shlex -import subprocess -import base64 from pymisp import MISPEvent - +from pymisp.tools import reportlab_generator misperrors = {'error': 'Error'} -moduleinfo = {'version': '1', - 'author': 'Raphaël Vinot', +moduleinfo = {'version': '2', + 'author': 'Vincent Falconieri (prev. Raphaël Vinot)', 'description': 'Simple export to PDF', 'module-type': ['export'], 'require_standard_format': True} -moduleconfig = [] - +# config fields that your code expects from the site admin +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"] mispattributes = {} + outputFileExtension = "pdf" responseType = "application/pdf" types_to_attach = ['ip-dst', 'url', 'domain'] objects_to_attach = ['domain-ip'] -headers = """ -:toc: right -:toclevels: 1 -:toc-title: Daily Report -:icons: font -:sectanchors: -:sectlinks: -= Daily report by {org_name} -{date} - -:icons: font - -""" - -event_level_tags = """ -IMPORTANT: This event is classified TLP:{value}. - -{expanded} - -""" - -attributes = """ -=== Indicator(s) of compromise - -{list_attributes} - -""" - -title = """ -== ({internal_id}) {title} - -{summary} - -""" - class ReportGenerator(): def __init__(self): @@ -79,60 +41,6 @@ class ReportGenerator(): self.misp_event = MISPEvent() self.misp_event.load(event) - def attributes(self): - if not self.misp_event.attributes: - return '' - list_attributes = [] - for attribute in self.misp_event.attributes: - if attribute.type in types_to_attach: - list_attributes.append("* {}".format(attribute.value)) - for obj in self.misp_event.Object: - if obj.name in objects_to_attach: - for attribute in obj.Attribute: - if attribute.type in types_to_attach: - list_attributes.append("* {}".format(attribute.value)) - return attributes.format(list_attributes="\n".join(list_attributes)) - - def _get_tag_info(self, machinetag): - return self.taxonomies.revert_machinetag(machinetag) - - def report_headers(self): - content = {'org_name': 'name', - 'date': date.today().isoformat()} - self.report += headers.format(**content) - - def event_level_tags(self): - if not self.misp_event.Tag: - return '' - for tag in self.misp_event.Tag: - # Only look for TLP for now - if tag['name'].startswith('tlp'): - tax, predicate = self._get_tag_info(tag['name']) - return self.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) - - def title(self): - internal_id = '' - summary = '' - # Get internal refs for report - if not hasattr(self.misp_event, 'Object'): - return '' - for obj in self.misp_event.Object: - if obj.name != 'report': - continue - for a in obj.Attribute: - if a.object_relation == 'case-number': - internal_id = a.value - if a.object_relation == 'summary': - summary = a.value - - return title.format(internal_id=internal_id, title=self.misp_event.info, - summary=summary) - - def asciidoc(self, lang='en'): - self.report += self.title() - self.report += self.event_level_tags() - self.report += self.attributes() - def handler(q=False): if q is False: @@ -143,45 +51,48 @@ def handler(q=False): if 'data' not in request: return False - for evt in request['data']: - report = ReportGenerator() - report.report_headers() - report.from_event(evt) - report.asciidoc() + config = {} - command_line = 'asciidoctor-pdf -' - args = shlex.split(command_line) - with subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) as process: - cmd_out, cmd_err = process.communicate(input=report.report.encode('utf-8')) - return {'response': [], 'data': str(base64.b64encode(cmd_out), 'utf-8')} + # Construct config object for reportlab_generator + for config_item in moduleconfig: + if (request.get('config')) and (request['config'].get(config_item) is not None): + config[config_item] = request['config'].get(config_item) + + for evt in request['data']: + misp_event = MISPEvent() + misp_event.load(evt) + + pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event, config)) + + return {'response': [], 'data': str(pdf, 'utf-8')} def introspection(): - modulesetup = {} - try: - responseType - modulesetup['responseType'] = responseType - except NameError: - pass + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - outputFileExtension - modulesetup['outputFileExtension'] = outputFileExtension - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup def version(): - moduleinfo['config'] = moduleconfig - return moduleinfo + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/export_mod/testexport.py b/misp_modules/modules/export_mod/testexport.py index ed93228..1fc7ff7 100755 --- a/misp_modules/modules/export_mod/testexport.py +++ b/misp_modules/modules/export_mod/testexport.py @@ -1,13 +1,12 @@ import json import base64 -import csv misperrors = {'error': 'Error'} userConfig = { -}; +} moduleconfig = [] @@ -28,9 +27,9 @@ def handler(q=False): if q is False: return False r = {'results': []} - result = json.loads(q) - output = ''; # Insert your magic here! - r = {"data":base64.b64encode(output.encode('utf-8')).decode('utf-8')} + result = json.loads(q) # noqa + output = '' # Insert your magic here! + r = {"data": base64.b64encode(output.encode('utf-8')).decode('utf-8')} return r diff --git a/misp_modules/modules/export_mod/threatStream_misp_export.py b/misp_modules/modules/export_mod/threatStream_misp_export.py index 3fd88c8..a9f7f06 100755 --- a/misp_modules/modules/export_mod/threatStream_misp_export.py +++ b/misp_modules/modules/export_mod/threatStream_misp_export.py @@ -49,9 +49,7 @@ def handler(q=False): if q is False or not q: return False - request = json.loads(q) - response = io.StringIO() writer = csv.DictWriter(response, fieldnames=["value", "itype", "tags"]) diff --git a/misp_modules/modules/export_mod/vt_graph.py b/misp_modules/modules/export_mod/vt_graph.py new file mode 100644 index 0000000..70c1952 --- /dev/null +++ b/misp_modules/modules/export_mod/vt_graph.py @@ -0,0 +1,113 @@ +'''Export MISP event to VirusTotal Graph.''' + + +import base64 +import json +from vt_graph_parser.importers.pymisp_response import from_pymisp_response + + +misperrors = { + 'error': 'Error' +} +moduleinfo = { + 'version': '0.1', + 'author': 'VirusTotal', + 'description': 'Send event to VirusTotal Graph', + 'module-type': ['export'] +} +mispattributes = { + 'input': [ + 'hostname', + 'domain', + 'ip-src', + 'ip-dst', + 'md5', + 'sha1', + 'sha256', + 'url', + 'filename|md5', + 'filename' + ] +} +moduleconfig = [ + 'vt_api_key', + 'fetch_information', + 'private', + 'fetch_vt_enterprise', + 'expand_one_level', + 'user_editors', + 'user_viewers', + 'group_editors', + 'group_viewers' +] + + +def handler(q=False): + """Expansion handler. + + Args: + q (bool, optional): module data. Defaults to False. + + Returns: + [str]: VirusTotal graph links + """ + if not q: + return False + request = json.loads(q) + + if not request.get('config') or not request['config'].get('vt_api_key'): + misperrors['error'] = 'A VirusTotal api key is required for this module.' + return misperrors + + config = request['config'] + + api_key = config.get('vt_api_key') + fetch_information = config.get('fetch_information') or False + private = config.get('private') or False + fetch_vt_enterprise = config.get('fetch_vt_enterprise') or False + expand_one_level = config.get('expand_one_level') or False + + user_editors = config.get('user_editors') + if user_editors: + user_editors = user_editors.split(',') + user_viewers = config.get('user_viewers') + if user_viewers: + user_viewers = user_viewers.split(',') + group_editors = config.get('group_editors') + if group_editors: + group_editors = group_editors.split(',') + group_viewers = config.get('group_viewers') + if group_viewers: + group_viewers = group_viewers.split(',') + + graphs = from_pymisp_response( + request, api_key, fetch_information=fetch_information, + private=private, fetch_vt_enterprise=fetch_vt_enterprise, + user_editors=user_editors, user_viewers=user_viewers, + group_editors=group_editors, group_viewers=group_viewers, + expand_node_one_level=expand_one_level) + links = [] + + for graph in graphs: + graph.save_graph() + links.append(graph.get_ui_link()) + + # This file will contains one VirusTotal graph link for each exported event + file_data = str(base64.b64encode( + bytes('\n'.join(links), 'utf-8')), 'utf-8') + return {'response': [], 'data': file_data} + + +def introspection(): + modulesetup = { + 'responseType': 'application/txt', + 'outputFileExtension': 'txt', + 'userConfig': {}, + 'inputSource': [] + } + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index 0a732e2..fbad911 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,3 +1,18 @@ -from . import _vmray +from . import _vmray # noqa +import os +import sys +sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) -__all__ = ['vmray_import', 'testimport', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport'] +__all__ = [ + 'vmray_import', + 'lastline_import', + 'ocr', + 'cuckooimport', + 'goamlimport', + 'email_import', + 'mispjson', + 'openiocimport', + 'threatanalyzer_import', + 'csvimport', + 'joe_import', +] diff --git a/misp_modules/modules/import_mod/csvimport.py b/misp_modules/modules/import_mod/csvimport.py index 5ccf287..34eed8c 100644 --- a/misp_modules/modules/import_mod/csvimport.py +++ b/misp_modules/modules/import_mod/csvimport.py @@ -1,156 +1,315 @@ # -*- coding: utf-8 -*- -import json, os, base64 -import pymisp +from pymisp import MISPEvent, MISPObject +from pymisp import __path__ as pymisp_path +import csv +import io +import json +import os +import base64 misperrors = {'error': 'Error'} -moduleinfo = {'version': '0.1', 'author': 'Christian Studer', +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', 'description': 'Import Attributes from a csv file.', 'module-type': ['import']} moduleconfig = [] -inputSource = ['file'] -userConfig = {'header': { - 'type': 'String', - 'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'}, - 'has_header':{ - 'type': 'Boolean', - 'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, in the file (which will be skipped atm).' - }} +userConfig = { + 'header': { + 'type': 'String', + 'message': 'Define the header of the csv file, with types (included in MISP attribute types or attribute fields) separated by commas.\nFor fields that do not match these types or that you want to skip, please use space or simply nothing between commas.\nFor instance: ip-src,domain, ,timestamp'}, + 'has_header': { + 'type': 'Boolean', + 'message': 'Tick this box ONLY if there is a header line, NOT COMMENTED, and all the fields of this header are respecting the recommendations above.'}, + 'special_delimiter': { + 'type': 'String', + 'message': 'IF THE DELIMITERS ARE NOT COMMAS, please specify which ones are used (for instance: ";", "|", "/", "\t" for tabs, etc).' + } +} +mispattributes = {'userConfig': userConfig, 'inputSource': ['file'], 'format': 'misp_standard'} + +misp_standard_csv_header = ['uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'date', + 'object_relation', 'attribute_tag', 'object_uuid', 'object_name', 'object_meta_category'] +misp_context_additional_fields = ['event_info', 'event_member_org', 'event_source_org', 'event_distribution', + 'event_threat_level_id', 'event_analysis', 'event_date', 'event_tag'] +misp_extended_csv_header = misp_standard_csv_header + misp_context_additional_fields -duplicatedFields = {'mispType': {'mispComment': 'comment'}, - 'attrField': {'attrComment': 'comment'}} -attributesFields = ['type', 'value', 'category', 'to_ids', 'comment', 'distribution'] -delimiters = [',', ';', '|', '/', '\t', ' '] class CsvParser(): - def __init__(self, header, has_header): + def __init__(self, header, has_header, delimiter, data, from_misp, MISPtypes, categories): + self.misp_event = MISPEvent() self.header = header - self.fields_number = len(header) self.has_header = has_header - self.attributes = [] + self.delimiter = delimiter + self.data = data + self.from_misp = from_misp + self.MISPtypes = MISPtypes + self.categories = categories + self.fields_number = len(self.header) + self.__score_mapping = {0: self.__create_standard_attribute, + 1: self.__create_attribute_with_ids, + 2: self.__create_attribute_with_tags, + 3: self.__create_attribute_with_ids_and_tags, + 4: self.__create_attribute_check_category, + 5: self.__create_attribute_check_category_and_ids, + 6: self.__create_attribute_check_category_and_tags, + 7: self.__create_attribute_check_category_with_ids_and_tags} - def parse_data(self, data): - return_data = [] - if self.fields_number == 1: - for line in data: - l = line.split('#')[0].strip() - if l: - return_data.append(l) - self.delimiter = None - else: - self.delimiter_count = dict([(d, 0) for d in delimiters]) - for line in data: - l = line.split('#')[0].strip() - if l: - self.parse_delimiter(l) - return_data.append(l) - # find which delimiter is used - self.delimiter = self.find_delimiter() - self.data = return_data[1:] if self.has_header else return_data - - def parse_delimiter(self, line): - for d in delimiters: - if line.count(d) >= (self.fields_number - 1): - self.delimiter_count[d] += 1 - - def find_delimiter(self): - _, delimiter = max((n, v) for v, n in self.delimiter_count.items()) - return delimiter - - def buildAttributes(self): - # if there is only 1 field of data - if self.delimiter is None: - mispType = self.header[0] - for data in self.data: - d = data.strip() - if d: - self.attributes.append({'types': mispType, 'values': d}) - else: - # split fields that should be recognized as misp attribute types from the others - list2pop, misp, head = self.findMispTypes() - # for each line of data - for data in self.data: - datamisp = [] - datasplit = data.split(self.delimiter) - # in case there is an empty line or an error - if len(datasplit) != self.fields_number: - continue - # pop from the line data that matches with a misp type, using the list of indexes - for l in list2pop: - datamisp.append(datasplit.pop(l).strip()) - # for each misp type, we create an attribute - for m, dm in zip(misp, datamisp): - attribute = {'types': m, 'values': dm} - for h, ds in zip(head, datasplit): - if h: - attribute[h] = ds.strip() - self.attributes.append(attribute) - - def findMispTypes(self): - descFilename = os.path.join(pymisp.__path__[0], 'data/describeTypes.json') - with open(descFilename, 'r') as f: - MispTypes = json.loads(f.read())['result'].get('types') - list2pop = [] - misp = [] - head = [] - for h in reversed(self.header): - n = self.header.index(h) - # fields that are misp attribute types - if h in MispTypes: - list2pop.append(n) - misp.append(h) - # handle confusions between misp attribute types and attribute fields - elif h in duplicatedFields['mispType']: - # fields that should be considered as misp attribute types - list2pop.append(n) - misp.append(duplicatedFields['mispType'].get(h)) - elif h in duplicatedFields['attrField']: - # fields that should be considered as attribute fields - head.append(duplicatedFields['attrField'].get(h)) - # or, it could be an attribute field - elif h in attributesFields: - head.append(h) - # otherwise, it is not defined + def parse_csv(self): + if self.from_misp: + if self.header == misp_standard_csv_header: + self.__parse_misp_csv() else: - head.append('') - # return list of indexes of the misp types, list of the misp types, remaining fields that will be attribute fields - return list2pop, misp, list(reversed(head)) + attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:10] + object_fields = ['object_id'] + misp_standard_csv_header[10:] + attribute_indexes = [] + object_indexes = [] + for i in range(len(self.header)): + if self.header[i] in attribute_fields: + attribute_indexes.append(i) + elif self.header[i] in object_fields: + object_indexes.append(i) + if object_indexes: + if not any(field in self.header for field in ('object_uuid', 'object_id')) or 'object_name' not in self.header: + for line in self.data: + for index in object_indexes: + if line[index].strip(): + return {'error': 'It is not possible to import MISP objects from your csv file if you do not specify any object identifier and object name to separate each object from each other.'} + if 'object_relation' not in self.header: + return {'error': 'In order to import MISP objects, an object relation for each attribute contained in an object is required.'} + self.__build_misp_event(attribute_indexes, object_indexes) + else: + attribute_fields = attribute_fields = misp_standard_csv_header[:1] + misp_standard_csv_header[2:9] + attribute_indexes = [] + types_indexes = [] + for i in range(len(self.header)): + if self.header[i] in attribute_fields: + attribute_indexes.append(i) + elif self.header[i] in self.MISPtypes: + types_indexes.append(i) + self.__parse_external_csv(attribute_indexes, types_indexes) + self.__finalize_results() + return {'success': 1} + + ################################################################################ + # Parsing csv data with MISP fields, # + # but a custom header # + ################################################################################ + + def __build_misp_event(self, attribute_indexes, object_indexes): + score = self.__get_score() + if object_indexes: + objects = {} + id_name = 'object_id' if 'object_id' in self.header else 'object_uuid' + object_id_index = self.header.index(id_name) + name_index = self.header.index('object_name') + for line in self.data: + attribute = self.__score_mapping[score](line, attribute_indexes) + object_id = line[object_id_index] + if object_id: + if object_id not in objects: + misp_object = MISPObject(line[name_index]) + if id_name == 'object_uuid': + misp_object.uuid = object_id + objects[object_id] = misp_object + objects[object_id].add_attribute(**attribute) + else: + self.event.add_attribute(**attribute) + for misp_object in objects.values(): + self.misp_event.add_object(**misp_object) + else: + for line in self.data: + attribute = self.__score_mapping[score](line, attribute_indexes) + self.misp_event.add_attribute(**attribute) + + ################################################################################ + # Parsing csv data containing fields that are not # + # MISP attributes or objects standard fields # + # (but should be MISP attribute types!!) # + ################################################################################ + + def __parse_external_csv(self, attribute_indexes, types_indexes): + score = self.__get_score() + if attribute_indexes: + for line in self.data: + try: + base_attribute = self.__score_mapping[score](line, attribute_indexes) + except IndexError: + continue + for index in types_indexes: + attribute = {'type': self.header[index], 'value': line[index]} + attribute.update(base_attribute) + self.misp_event.add_attribute(**attribute) + else: + for line in self.data: + for index in types_indexes: + self.misp_event.add_attribute(**{'type': self.header[index], 'value': line[index]}) + + ################################################################################ + # Parsing standard MISP csv format # + ################################################################################ + + def __parse_misp_csv(self): + objects = {} + attribute_fields = self.header[:1] + self.header[2:8] + for line in self.data: + a_uuid, _, category, _type, value, comment, ids, timestamp, relation, tag, o_uuid, name, _ = line[:self.fields_number] + attribute = {t: v.strip('"') for t, v in zip(attribute_fields, (a_uuid, category, _type, value, comment, ids, timestamp))} + attribute['to_ids'] = True if attribute['to_ids'] == '1' else False + if tag: + attribute['Tag'] = [{'name': t.strip()} for t in tag.split(',')] + if relation: + if o_uuid not in objects: + objects[o_uuid] = MISPObject(name) + objects[o_uuid].add_attribute(relation, **attribute) + else: + self.misp_event.add_attribute(**attribute) + for uuid, misp_object in objects.items(): + misp_object.uuid = uuid + self.misp_event.add_object(**misp_object) + + ################################################################################ + # Utility functions # + ################################################################################ + + def __create_attribute_check_category(self, line, indexes): + attribute = self.__create_standard_attribute(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_and_ids(self, line, indexes): + attribute = self.__create_attribute_with_ids(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_and_tags(self, line, indexes): + attribute = self.__create_attribute_with_tags(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_check_category_with_ids_and_tags(self, line, indexes): + attribute = self.__create_attribute_with_ids_and_tags(line, indexes) + self.__check_category(attribute) + return attribute + + def __create_attribute_with_ids(self, line, indexes): + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_ids(attribute) + return attribute + + def __create_attribute_with_ids_and_tags(self, line, indexes): + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_ids(attribute) + self.__deal_with_tags(attribute) + return attribute + + def __create_attribute_with_tags(self, line, indexes): + attribute = self.__create_standard_attribute(line, indexes) + self.__deal_with_tags(attribute) + return attribute + + def __create_standard_attribute(self, line, indexes): + return {self.header[index]: line[index] for index in indexes if line[index]} + + def __check_category(self, attribute): + category = attribute['category'] + if category in self.categories: + return + if category.capitalize() in self.categories: + attribute['category'] = category.capitalize() + return + del attribute['category'] + + @staticmethod + def __deal_with_ids(attribute): + attribute['to_ids'] = True if attribute['to_ids'] == '1' else False + + @staticmethod + def __deal_with_tags(attribute): + attribute['Tag'] = [{'name': tag.strip()} for tag in attribute['Tag'].split(',')] + + def __get_score(self): + score = 1 if 'to_ids' in self.header else 0 + if 'attribute_tag' in self.header: + score += 2 + if 'category' in self.header: + score += 4 + return score + + def __finalize_results(self): + event = json.loads(self.misp_event.to_json()) + self.results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + + +def __any_mandatory_misp_field(header): + return any(field in header for field in ('type', 'value')) + + +def __special_parsing(data, delimiter): + return list(tuple(part.strip() for part in line[0].split(delimiter)) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#')) + + +def __standard_parsing(data): + return list(tuple(part.strip() for part in line) for line in csv.reader(io.TextIOWrapper(io.BytesIO(data.encode()), encoding='utf-8')) if line and not line[0].startswith('#')) + def handler(q=False): if q is False: return False request = json.loads(q) if request.get('data'): - data = base64.b64decode(request['data']).decode('utf-8') + try: + data = base64.b64decode(request['data']).decode('utf-8') + except UnicodeDecodeError: + misperrors['error'] = "Input is not valid UTF-8" + return misperrors else: misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not request['config'].get('header'): - misperrors['error'] = "Configuration error" - return misperrors - header = request['config'].get('header').split(',') - header = [c.strip() for c in header] has_header = request['config'].get('has_header') has_header = True if has_header == '1' else False - csv_parser = CsvParser(header, has_header) - csv_parser.parse_data(data.split('\n')) + header = request['config']['header'].split(',') if request['config'].get('header').strip() else [] + delimiter = request['config']['special_delimiter'] if request['config'].get('special_delimiter').strip() else ',' + data = __standard_parsing(data) if delimiter == ',' else __special_parsing(data, delimiter) + if not header: + if has_header: + header = data.pop(0) + else: + misperrors['error'] = "Configuration error. Provide a header or use the one within the csv file and tick the checkbox 'Has_header'." + return misperrors + else: + header = [h.strip() for h in header] + if has_header: + del data[0] + if header == misp_standard_csv_header or header == misp_extended_csv_header: + header = misp_standard_csv_header + descFilename = os.path.join(pymisp_path[0], 'data/describeTypes.json') + with open(descFilename, 'r') as f: + description = json.loads(f.read())['result'] + MISPtypes = description['types'] + for h in header: + if not any((h in MISPtypes, h in misp_extended_csv_header, h in ('', ' ', '_', 'object_id'))): + misperrors['error'] = 'Wrong header field: {}. Please use a header value that can be recognized by MISP (or alternatively skip it using a whitespace).'.format(h) + return misperrors + from_misp = all((h in misp_extended_csv_header or h in ('', ' ', '_', 'object_id') for h in header)) + if from_misp: + if not __any_mandatory_misp_field(header): + misperrors['error'] = 'Please make sure the data you try to import can be identified with a type/value combinaison.' + return misperrors + else: + if __any_mandatory_misp_field(header): + wrong_types = tuple(wrong_type for wrong_type in ('type', 'value') if wrong_type in header) + misperrors['error'] = 'Error with the following header: {}. It contains the following field(s): {}, which is(are) already provided by the usage of at least on MISP attribute type in the header.'.format(header, 'and'.join(wrong_types)) + return misperrors + csv_parser = CsvParser(header, has_header, delimiter, data, from_misp, MISPtypes, description['categories']) # build the attributes - csv_parser.buildAttributes() - r = {'results': csv_parser.attributes} - return r + result = csv_parser.parse_csv() + if 'error' in result: + return result + return {'results': csv_parser.results} + def introspection(): - modulesetup = {} - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + return mispattributes + def version(): moduleinfo['config'] = moduleconfig diff --git a/misp_modules/modules/import_mod/cuckooimport.py b/misp_modules/modules/import_mod/cuckooimport.py index 193477f..3ed52bd 100755 --- a/misp_modules/modules/import_mod/cuckooimport.py +++ b/misp_modules/modules/import_mod/cuckooimport.py @@ -1,196 +1,744 @@ import json -import logging -import sys -import base64 +import base64 +import io +import logging +import posixpath +import stat +import tarfile +import zipfile +from pymisp import MISPEvent, MISPObject, MISPAttribute +from pymisp.tools import make_binary_objects +from collections import OrderedDict + +log = logging.getLogger(__name__) misperrors = {'error': 'Error'} -userConfig = {} -inputSource = ['file'] -moduleinfo = {'version': '0.1', 'author': 'Victor van der Stoep', - 'description': 'Cuckoo JSON import', - 'module-type': ['import']} +moduleinfo = { + 'version': '1.1', + 'author': 'Pierre-Jean Grenier', + 'description': "Import a Cuckoo archive (zipfile or bzip2 tarball), " + "either downloaded manually or exported from the " + "API (/tasks/report/{task_id}/all).", + 'module-type': ['import'], +} moduleconfig = [] +mispattributes = { + 'inputSource': ['file'], + 'output': ['MISP objects', 'malware-sample'], + 'format': 'misp_standard', +} + +# Attributes for which we can set the "Artifacts dropped" +# category if we want to +ARTIFACTS_DROPPED = ( + "filename", + "md5", + "sha1", + "sha256", + "sha512", + "malware-sample", + "mimetype", + "ssdeep", +) + +# Same for the category "Payload delivery" +PAYLOAD_DELIVERY = ARTIFACTS_DROPPED + + +class PrettyDict(OrderedDict): + """ + This class is just intended for a pretty print + of its keys and values. + """ + MAX_SIZE = 30 + + def __str__(self): + tmp = [] + for k, v in self.items(): + v = str(v) + if len(v) > self.MAX_SIZE: + k += ',cut' + v = v[:self.MAX_SIZE] + v.replace('\n', ' ') + tmp.append((k, v)) + return "; ".join(f"({k}) {v}" for k, v in tmp) + + +def search_objects(event, name, attributes=[]): + """ + Search for objects in event, which name is `name` and + contain at least the attributes given. + Return a generator. + @ param attributes: a list of (object_relation, value) + """ + match = filter( + lambda obj: all( + obj.name == name + and (obj_relation, str(attr_value)) in map( + lambda attr: (attr.object_relation, str(attr.value)), + obj.attributes + ) + for obj_relation, attr_value in attributes + ), event.objects + ) + return match + + +def find_process_by_pid(event, pid): + """ + Find a 'process' MISPObject by its PID. If multiple objects are found, + only return the first one. + @ param pid: integer or str + """ + generator = search_objects( + event, + "process", + (('pid', pid),) + ) + return next(generator, None) + + +class CuckooParser(): + # This dict is used to generate the userConfig and link the different + # options to the corresponding method of the parser. This way, we avoid + # redundancy and make future changes easier (instead of for instance + # defining all the options in userConfig directly, and then making a + # switch when running the parser). + # Careful about the order here, as we create references between + # MISPObjects/MISPAttributes at the same time we generate them. + # Hence when we create object B, which we want to reference to + # object A, we should already have created object A. + # TODO create references only after all parsing is done + options = { + "Sandbox info": { + "method": lambda self: self.add_sandbox_info(), + "userConfig": { + 'type': 'Boolean', + 'message': "Add info related to the sandbox", + 'checked': 'true', + }, + }, + "Upload sample": { + "method": lambda self: self.add_sample(), + "userConfig": { + 'type': 'Boolean', + 'message': "Upload the sample", + 'checked': 'true', + }, + }, + "Processes": { + "method": lambda self: self.add_process_tree(), + "userConfig": { + 'type': 'Boolean', + 'message': "Add info related to the processes", + 'checked': 'true', + }, + }, + "DNS": { + "method": lambda self: self.add_dns(), + "userConfig": { + 'type': 'Boolean', + 'message': "Add DNS queries/answers", + 'checked': 'true', + }, + }, + "TCP": { + "method": lambda self: self.add_network("tcp"), + "userConfig": { + 'type': 'Boolean', + 'message': "Add TCP connections", + 'checked': 'true', + }, + }, + "UDP": { + "method": lambda self: self.add_network("udp"), + "userConfig": { + 'type': 'Boolean', + 'message': "Add UDP connections", + 'checked': 'true', + }, + }, + "HTTP": { + "method": lambda self: self.add_http(), + "userConfig": { + 'type': 'Boolean', + 'message': "Add HTTP requests", + 'checked': 'true', + }, + }, + "Signatures": { + "method": lambda self: self.add_signatures(), + "userConfig": { + 'type': 'Boolean', + 'message': "Add Cuckoo's triggered signatures", + 'checked': 'true', + }, + }, + "Screenshots": { + "method": lambda self: self.add_screenshots(), + "userConfig": { + 'type': 'Boolean', + 'message': "Upload the screenshots", + 'checked': 'true', + }, + }, + "Dropped files": { + "method": lambda self: self.add_dropped_files(), + "userConfig": { + 'type': 'Boolean', + 'message': "Upload the dropped files", + 'checked': 'true', + }, + }, + "Dropped buffers": { + "method": lambda self: self.add_dropped_buffers(), + "userConfig": { + 'type': 'Boolean', + 'message': "Upload the dropped buffers", + 'checked': 'true', + }, + }, + } + + def __init__(self, config): + self.event = MISPEvent() + self.files = None + self.malware_binary = None + self.report = None + self.config = { + # if an option is missing (we receive None as a value), + # fall back to the default specified in the options + key: int( + on if on is not None + else self.options[key]["userConfig"]["checked"] == 'true' + ) + for key, on in config.items() + } + + def get_file(self, relative_filepath): + """Return an io.BufferedIOBase for the corresponding relative_filepath + in the Cuckoo archive. If not found, return an empty io.BufferedReader + to avoid fatal errors.""" + blackhole = io.BufferedReader(open('/dev/null', 'rb')) + res = self.files.get(relative_filepath, blackhole) + if res == blackhole: + log.debug(f"Did not find file {relative_filepath}, " + f"returned an empty file instead") + return res + + def read_archive(self, archive_encoded): + """Read the archive exported from Cuckoo and initialize the class""" + # archive_encoded is base 64 encoded content + # we extract the info about each file but do not retrieve + # it automatically, as it may take too much space in memory + buf_io = io.BytesIO(base64.b64decode(archive_encoded)) + if zipfile.is_zipfile(buf_io): + # the archive was probably downloaded from the WebUI + buf_io.seek(0) # don't forget this not to read an empty buffer + z = zipfile.ZipFile(buf_io, 'r') + self.files = { + info.filename: z.open(info) + for info in z.filelist + # only extract the regular files and dirs, we don't + # want any symbolic link + if stat.S_ISREG(info.external_attr >> 16) + or stat.S_ISDIR(info.external_attr >> 16) + } + else: + # the archive was probably downloaded from the API + buf_io.seek(0) # don't forget this not to read an empty buffer + f = tarfile.open(fileobj=buf_io, mode='r:bz2') + self.files = { + info.name: f.extractfile(info) + for info in f.getmembers() + # only extract the regular files and dirs, we don't + # want any symbolic link + if info.isreg() or info.isdir() + } + + # We want to keep the order of the keys of sub-dicts in the report, + # eg. the signatures have marks with unknown keys such as + # {'marks': [ + # {"suspicious_features": "Connection to IP address", + # "suspicious_request": "OPTIONS http://85.20.18.18/doc"} + # ]} + # To render those marks properly, we can only hope the developpers + # thought about the order in which they put the keys, and keep this + # order so that the signature makes sense to the reader. + # We use PrettyDict, a customization of OrderedDict to do so. + # It will be instanced iteratively when parsing the json (ie. subdicts + # will also be instanced as PrettyDict) + self.report = json.load( + self.get_file("reports/report.json"), + object_pairs_hook=PrettyDict, + ) + + def read_malware(self): + self.malware_binary = self.get_file("binary").read() + if not self.malware_binary: + log.warn("No malware binary found") + + def add_sandbox_info(self): + info = self.report.get("info", {}) + if not info: + log.warning("The 'info' field was not found " + "in the report, skipping") + return False + + o = MISPObject(name='sandbox-report') + o.add_attribute('score', info['score']) + o.add_attribute('sandbox-type', 'on-premise') + o.add_attribute('on-premise-sandbox', 'cuckoo') + o.add_attribute('raw-report', + f'started on:{info["machine"]["started_on"]} ' + f'duration:{info["duration"]}s ' + f'vm:{info["machine"]["name"]}/' + f'{info["machine"]["label"]}') + self.event.add_object(o) + + def add_sample(self): + """Add the sample/target of the analysis""" + target = self.report.get("target", {}) + category = target.get("category", "") + if not category: + log.warning("Could not find info about the sample " + "in the report, skipping") + return False + + if category == "file": + log.debug("Sample is a file, uploading it") + self.read_malware() + file_o, bin_type_o, bin_section_li = make_binary_objects( + pseudofile=io.BytesIO(self.malware_binary), + filename=target["file"]["name"], + ) + + file_o.comment = "Submitted sample" + # fix categories + for obj in filter(None, (file_o, bin_type_o, *bin_section_li,)): + for attr in obj.attributes: + if attr.type in PAYLOAD_DELIVERY: + attr.category = "Payload delivery" + self.event.add_object(obj) + + elif category == "url": + log.debug("Sample is a URL") + o = MISPObject(name='url') + o.add_attribute('url', target['url']) + o.add_attribute('text', "Submitted URL") + self.event.add_object(o) + + def add_http(self): + """Add the HTTP requests""" + network = self.report.get("network", []) + http = network.get("http", []) + if not http: + log.info("No HTTP connection found in the report, skipping") + return False + + for request in http: + o = MISPObject(name='http-request') + o.add_attribute('host', request['host']) + o.add_attribute('method', request['method']) + o.add_attribute('uri', request['uri']) + o.add_attribute('user-agent', request['user-agent']) + o.add_attribute('text', f"count:{request['count']} " + f"port:{request['port']}") + self.event.add_object(o) + + def add_network(self, proto=None): + """ + Add UDP/TCP traffic + proto must be one of "tcp", "udp" + """ + network = self.report.get("network", []) + li_conn = network.get(proto, []) + if not li_conn: + log.info(f"No {proto} connection found in the report, skipping") + return False + + from_to = [] + # sort by time to get the "first packet seen" right + li_conn.sort(key=lambda x: x["time"]) + for conn in li_conn: + src = conn['src'] + dst = conn['dst'] + sport = conn['sport'] + dport = conn['dport'] + if (src, sport, dst, dport) in from_to: + continue + + from_to.append((src, sport, dst, dport)) + + o = MISPObject(name='network-connection') + o.add_attribute('ip-src', src) + o.add_attribute('ip-dst', dst) + o.add_attribute('src-port', sport) + o.add_attribute('dst-port', dport) + o.add_attribute('layer3-protocol', "IP") + o.add_attribute('layer4-protocol', proto.upper()) + o.add_attribute('first-packet-seen', conn['time']) + self.event.add_object(o) + + def add_dns(self): + """Add DNS records""" + network = self.report.get("network", []) + dns = network.get("dns", []) + if not dns: + log.info("No DNS connection found in the report, skipping") + return False + + for record in dns: + o = MISPObject(name='dns-record') + o.add_attribute('text', f"request type:{record['type']}") + o.add_attribute('queried-domain', record['request']) + for answer in record.get("answers", []): + if answer["type"] in ("A", "AAAA"): + o.add_attribute('a-record', answer['data']) + # TODO implement MX/NS + + self.event.add_object(o) + + def _get_marks_str(self, marks): + marks_strings = [] + for m in marks: + m_type = m.pop("type") # temporarily remove the type + + if m_type == "generic": + marks_strings.append(str(m)) + + elif m_type == "ioc": + marks_strings.append(m['ioc']) + + elif m_type == "call": + call = m["call"] + arguments = call.get("arguments", {}) + flags = call.get("flags", {}) + info = "" + for details in (arguments, flags): + info += f" {details}" + marks_strings.append(f"Call API '{call['api']}'%s" % info) + + else: + logging.debug(f"Unknown mark type '{m_type}', skipping") + + m["type"] = m_type # restore key 'type' + # TODO implemented marks 'config' and 'volatility' + return marks_strings + + def _add_ttp(self, attribute, ttp_short, ttp_num): + """ + Internal wrapper to add the TTP tag from the MITRE galaxy. + @ params + - attribute: MISPAttribute + - ttp_short: short description of the TTP + (eg. "Credential Dumping") + - ttp_num: formatted as "T"+int + (eg. T1003) + """ + attribute.add_tag(f'misp-galaxy:mitre-attack-pattern=' + f'"{ttp_short} - {ttp_num}"') + + def add_signatures(self): + """Add the Cuckoo signatures, with as many details as possible + regarding the marks""" + signatures = self.report.get("signatures", []) + if not signatures: + log.info("No signature found in the report") + return False + + o = MISPObject(name='sb-signature') + o.add_attribute('software', "Cuckoo") + + for sign in signatures: + marks = sign["marks"] + marks_strings = self._get_marks_str(marks) + summary = sign['description'] + if marks_strings: + summary += "\n---\n" + + marks_strings = set(marks_strings) + description = summary + "\n".join(marks_strings) + + a = MISPAttribute() + a.from_dict(type='text', value=description) + for ttp_num, desc in sign.get("ttp", {}).items(): + ttp_short = desc["short"] + self._add_ttp(a, ttp_short, ttp_num) + + # this signature was triggered by the processes with the following + # PIDs, we can create references + triggered_by_pids = filter( + None, + (m.get("pid", None) for m in marks) + ) + # remove redundancy + triggered_by_pids = set(triggered_by_pids) + for pid in triggered_by_pids: + process_o = find_process_by_pid(self.event, pid) + if process_o: + process_o.add_reference(a, "triggers") + + o.add_attribute('signature', **a) + + self.event.add_object(o) + + def _handle_process(self, proc, accu): + """ + This is an internal recursive function to handle one process + from a process tree and then iterate on its children. + List the objects to be added, based on the tree, into the `accu` list. + The `accu` list uses a DFS-like order. + """ + o = MISPObject(name='process') + accu.append(o) + o.add_attribute('pid', proc['pid']) + o.add_attribute('command-line', proc['command_line']) + o.add_attribute('name', proc['process_name']) + o.add_attribute('parent-pid', proc['ppid']) + for child in proc.get('children', []): + pos_child = len(accu) + o.add_attribute('child-pid', child['pid']) + self._handle_process(child, accu) + child_obj = accu[pos_child] + child_obj.add_reference(o, 'child-of') + + return o + + def add_process_tree(self): + """Add process tree from the report, as separated process objects""" + behavior = self.report.get("behavior", {}) + tree = behavior.get("processtree", []) + if not tree: + log.warning("No process tree found in the report, skipping") + return False + + for proc in tree: + objs = [] + self._handle_process(proc, objs) + for o in objs: + self.event.add_object(o) + + def get_relpath(self, path): + """ + Transform an absolute or relative path into a path relative to the + correct cuckoo analysis directory, without knowing the cuckoo + working directory. + Return an empty string if the path given does not refer to a + file from the analysis directory. + """ + head, tail = posixpath.split(path) + if not tail: + return "" + prev = self.get_relpath(head) + longer = posixpath.join(prev, tail) + if longer in self.files: + return longer + elif tail in self.files: + return tail + else: + return "" + + def add_screenshots(self): + """Add the screenshots taken by Cuckoo in a sandbox-report object""" + screenshots = self.report.get('screenshots', []) + if not screenshots: + log.info("No screenshot found in the report, skipping") + return False + + o = MISPObject(name='sandbox-report') + o.add_attribute('sandbox-type', 'on-premise') + o.add_attribute('on-premise-sandbox', "cuckoo") + for shot in screenshots: + # The path given by Cuckoo is an absolute path, but we need a path + # relative to the analysis folder. + path = self.get_relpath(shot['path']) + img = self.get_file(path) + # .decode('utf-8') in order to avoid the b'' format + img_data = base64.b64encode(img.read()).decode('utf-8') + filename = posixpath.basename(path) + + o.add_attribute( + "sandbox-file", value=filename, + data=img_data, type='attachment', + category="External analysis", + ) + + self.event.add_object(o) + + def _get_dropped_objs(self, path, filename=None, comment=None): + """ + Internal wrapper to get dropped files/buffers as file objects + @ params + - path: relative to the cuckoo analysis directory + - filename: if not specified, deduced from the path + """ + if not filename: + filename = posixpath.basename(path) + + dropped_file = self.get_file(path) + dropped_binary = io.BytesIO(dropped_file.read()) + # create ad hoc objects + file_o, bin_type_o, bin_section_li = make_binary_objects( + pseudofile=dropped_binary, filename=filename, + ) + + if comment: + file_o.comment = comment + # fix categories + for obj in filter(None, (file_o, bin_type_o, *bin_section_li,)): + for attr in obj.attributes: + if attr.type in ARTIFACTS_DROPPED: + attr.category = "Artifacts dropped" + + return file_o, bin_type_o, bin_section_li + + def _add_yara(self, obj, yara_dict): + """Internal wrapper to add Yara matches to an MISPObject""" + for yara in yara_dict: + description = yara.get("meta", {}).get("description", "") + name = yara.get("name", "") + obj.add_attribute( + "text", + f"Yara match\n(name) {name}\n(description) {description}", + comment="Yara match" + ) + + def add_dropped_files(self): + """Upload the dropped files as file objects""" + dropped = self.report.get("dropped", []) + if not dropped: + log.info("No dropped file found, skipping") + return False + + for d in dropped: + # Cuckoo logs three things that are of interest for us: + # - 'filename' which is not the original name of the file + # but is formatted as follow: + # 8 first bytes of SHA265 + _ + original name in lower case + # - 'filepath' which is the original filepath on the VM, + # where the file was dropped + # - 'path' which is the local path of the stored file, + # in the cuckoo archive + filename = d.get("name", "") + original_path = d.get("filepath", "") + sha256 = d.get("sha256", "") + if original_path and sha256: + log.debug(f"Will now try to restore original filename from " + f"path {original_path}") + try: + s = filename.split("_") + if not s: + raise Exception("unexpected filename read " + "in the report") + sha256_first_8_bytes = s[0] + original_name = s[1] + # check our assumptions are valid, if so we can safely + # restore the filename, if not the format may have changed + # so we'll keep the filename of the report + if sha256.startswith(sha256_first_8_bytes) and \ + original_path.lower().endswith(original_name) and \ + filename not in original_path.lower(): + # we can restore the original case of the filename + position = original_path.lower().rindex(original_name) + filename = original_path[position:] + log.debug(f"Successfully restored original filename: " + f"{filename}") + else: + raise Exception("our assumptions were wrong, " + "filename format may have changed") + except Exception as e: + log.debug(f"Cannot restore filename: {e}") + + if not filename: + filename = "NO NAME FOUND IN THE REPORT" + log.warning(f'No filename found for dropped file! ' + f'Will use "{filename}"') + + file_o, bin_type_o, bin_section_o = self._get_dropped_objs( + self.get_relpath(d['path']), + filename=filename, + comment="Dropped file" + ) + + self._add_yara(file_o, d.get("yara", [])) + + file_o.add_attribute("fullpath", original_path, + category="Artifacts dropped") + + # why is this a list? for when various programs drop the same file? + for pid in d.get("pids", []): + # if we have an object for the process that dropped the file, + # we can link the two (we just take the first result from + # the search) + process_o = find_process_by_pid(self.event, pid) + if process_o: + file_o.add_reference(process_o, "dropped-by") + + self.event.add_object(file_o) + + def add_dropped_buffers(self): + """"Upload the dropped buffers as file objects""" + buffer = self.report.get("buffer", []) + if not buffer: + log.info("No dropped buffer found, skipping") + return False + + for i, buf in enumerate(buffer): + file_o, bin_type_o, bin_section_o = self._get_dropped_objs( + self.get_relpath(buf['path']), + filename=f"buffer {i}", + comment="Dropped buffer" + ) + self._add_yara(file_o, buf.get("yara", [])) + self.event.add_object(file_o) + + def parse(self): + """Run the parsing""" + for name, active in self.config.items(): + if active: + self.options[name]["method"](self) + + def get_misp_event(self): + log.debug("Running MISP expansions") + self.event.run_expansions() + return self.event + + def handler(q=False): - # Just in case we have no data + # In case there's no data if q is False: return False - - # The return value - r = {'results': []} - # Load up that JSON - q = json.loads(q) - data = base64.b64decode(q.get("data")).decode('utf-8') - - # If something really weird happened - if not data: - return json.dumps({"success": 0}) - - data = json.loads(data) - - # Get characteristics of file - targetFile = data['target']['file'] - - # Process the inital binary - processBinary(r, targetFile, initial = True) - - # Get binary information for dropped files - if(data.get('dropped')): - for droppedFile in data['dropped']: - processBinary(r, droppedFile, dropped = True) - - # Add malscore to results - r["results"].append({ - "values": "Malscore: {} ".format(data['malscore']), - "types": "comment", - "categories": "Payload delivery", - "comment": "Cuckoo analysis: MalScore" - }) - - # Add virustotal data, if exists - if(data.get('virustotal')): - processVT(r, data['virustotal']) - - # Add network information, should be improved - processNetwork(r, data['network']) - - # Add behavioral information - processSummary(r, data['behavior']['summary']) - - # Return - return r + q = json.loads(q) + data = q['data'] -def processSummary(r, summary): - r["results"].append({ - "values": summary['mutexes'], - "types": "mutex", - "categories": "Artifacts dropped", - "comment": "Cuckoo analysis: Observed mutexes" - }) - -def processVT(r, virustotal): - category = "Antivirus detection" - comment = "VirusTotal analysis" - - if(virustotal.get('permalink')): - r["results"].append({ - "values": virustotal['permalink'], - "types": "link", - "categories": category, - "comments": comment + " - Permalink" - }) - - if(virustotal.get('total')): - r["results"].append({ - "values": "VirusTotal detection rate {}/{}".format( - virustotal['positives'], - virustotal['total'] - ), - "types": "comment", - "categories": category, - "comment": comment - }) - else: - r["results"].append({ - "values": "Sample not detected on VirusTotal", - "types": "comment", - "categories": category, - "comment": comment - }) - + parser = CuckooParser(q['config']) + parser.read_archive(data) + parser.parse() + event = parser.get_misp_event() -def processNetwork(r, network): - category = "Network activity" - - for host in network['hosts']: - r["results"].append({ - "values": host['ip'], - "types": "ip-dst", - "categories": category, - "comment": "Cuckoo analysis: Observed network traffic" - }) - + event = json.loads(event.to_json()) + results = { + key: event[key] + for key in ('Attribute', 'Object') + if (key in event and event[key]) + } + return {'results': results} -def processBinary(r, target, initial = False, dropped = False): - if(initial): - comment = "Cuckoo analysis: Initial file" - category = "Payload delivery" - elif(dropped): - category = "Artifacts dropped" - comment = "Cuckoo analysis: Dropped file" - - r["results"].append({ - "values": target['name'], - "types": "filename", - "categories": category, - "comment": comment - }) - - r["results"].append({ - "values": target['md5'], - "types": "md5", - "categories": category, - "comment": comment - }) - - r["results"].append({ - "values": target['sha1'], - "types": "sha1", - "categories": category, - "comment": comment - }) - - r["results"].append({ - "values": target['sha256'], - "types": "sha256", - "categories": category, - "comment": comment - }) - - r["results"].append({ - "values": target['sha512'], - "types": "sha512", - "categories": category, - "comment": comment - }) - - # todo : add file size? - - if(target.get('guest_paths')): - r["results"].append({ - "values": target['guest_paths'], - "types": "filename", - "categories": "Payload installation", - "comment": comment + " - Path" - }) - def introspection(): - modulesetup = {} - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + userConfig = { + key: o["userConfig"] + for key, o in CuckooParser.options.items() + } + mispattributes['userConfig'] = userConfig + return mispattributes def version(): moduleinfo['config'] = moduleconfig return moduleinfo - -if __name__ == '__main__': - x = open('test.json', 'r') - q = [] - q['data'] = x.read() - q = base64.base64encode(q) - - handler(q) diff --git a/misp_modules/modules/import_mod/email_import.py b/misp_modules/modules/import_mod/email_import.py index fa7d5dc..114f8c9 100644 --- a/misp_modules/modules/import_mod/email_import.py +++ b/misp_modules/modules/import_mod/email_import.py @@ -3,24 +3,25 @@ import json import base64 -import io import zipfile -import codecs import re -from email import message_from_bytes -from email.utils import parseaddr -from email.iterators import typed_subpart_iterator -from email.parser import Parser from html.parser import HTMLParser -from email.header import decode_header +from pymisp.tools import EMailObject, make_binary_objects +try: + from pymisp.tools import URLObject +except ImportError: + raise ImportError('Unable to import URLObject, pyfaup missing') +from io import BytesIO +from pathlib import Path + misperrors = {'error': 'Error'} -userConfig = {} -inputSource = ['file'] +mispattributes = {'inputSource': ['file'], 'output': ['MISP objects'], + 'format': 'misp_standard'} -moduleinfo = {'version': '0.1', - 'author': 'Seamus Tuohy', +moduleinfo = {'version': '0.2', + 'author': 'Seamus Tuohy, Raphaël Vinot', 'description': 'Email import module for MISP', 'module-type': ['import']} @@ -35,93 +36,13 @@ moduleconfig = ["unzip_attachments", def handler(q=False): if q is False: return False - results = [] # Decode and parse email request = json.loads(q) # request data is always base 64 byte encoded data = base64.b64decode(request["data"]) - # Double decode to force headers to be re-parsed with proper encoding - message = Parser().parsestr(message_from_bytes(data).as_string()) - # Decode any encoded headers to get at proper string - for key, val in message.items(): - replacement = get_decoded_header(key, val) - if replacement is not None: - message.replace_header(key, replacement) - - # Extract all header information - all_headers = "" - for k, v in message.items(): - all_headers += "{0}: {1}\n".format(k.strip(), v.strip()) - results.append({"values": all_headers, "type": 'email-header'}) - - # E-Mail MIME Boundry - if message.get_boundary(): - results.append({"values": message.get_boundary(), "type": 'email-mime-boundary'}) - - # E-Mail Reply To - if message.get('In-Reply-To'): - results.append({"values": message.get('In-Reply-To').strip(), "type": 'email-reply-to'}) - - # X-Mailer - if message.get('X-Mailer'): - results.append({"values": message.get('X-Mailer'), "type": 'email-x-mailer'}) - - # Thread Index - if message.get('Thread-Index'): - results.append({"values": message.get('Thread-Index'), "type": 'email-thread-index'}) - - # Email Message ID - if message.get('Message-ID'): - results.append({"values": message.get('Message-ID'), "type": 'email-message-id'}) - - # Subject - if message.get('Subject'): - results.append({"values": message.get('Subject'), "type": 'email-subject'}) - - # Source - from_addr = message.get('From') - if from_addr: - results.append({"values": parseaddr(from_addr)[1], "type": 'email-src', "comment": "From: {0}".format(from_addr)}) - results.append({"values": parseaddr(from_addr)[0], "type": 'email-src-display-name', "comment": "From: {0}".format(from_addr)}) - - # Return Path - return_path = message.get('Return-Path') - if return_path: - # E-Mail Source - results.append({"values": parseaddr(return_path)[1], "type": 'email-src', "comment": "Return Path: {0}".format(return_path)}) - # E-Mail Source Name - results.append({"values": parseaddr(return_path)[0], "type": 'email-src-display-name', "comment": "Return Path: {0}".format(return_path)}) - - # Destinations - # Split and sort destination header values - recipient_headers = ['To', 'Cc', 'Bcc'] - - for hdr_val in recipient_headers: - if message.get(hdr_val): - addrs = message.get(hdr_val).split(',') - for addr in addrs: - # Parse and add destination header values - parsed_addr = parseaddr(addr) - results.append({"values": parsed_addr[1], "type": "email-dst", "comment": "{0}: {1}".format(hdr_val, addr)}) - results.append({"values": parsed_addr[0], "type": "email-dst-display-name", "comment": "{0}: {1}".format(hdr_val, addr)}) - - # Get E-Mail Targets - # Get the addresses that received the email. - # As pulled from the Received header - received = message.get_all('Received') - if received: - email_targets = set() - for rec in received: - try: - email_check = re.search("for\s(.*@.*);", rec).group(1) - email_check = email_check.strip(' <>') - email_targets.add(parseaddr(email_check)[1]) - except (AttributeError): - continue - for tar in email_targets: - results.append({"values": tar, "type": "target-email", "comment": "Extracted from email 'Received' header"}) + email_object = EMailObject(pseudofile=BytesIO(data), attach_original_mail=True, standalone=False) # Check if we were given a configuration config = request.get("config", {}) @@ -137,66 +58,82 @@ def handler(q=False): zip_pass_crack = config.get("guess_zip_attachment_passwords", None) if (zip_pass_crack is not None and zip_pass_crack.lower() in acceptable_config_yes): zip_pass_crack = True - password_list = None # Only want to collect password list once + password_list = get_zip_passwords(email_object.email) # Do we extract URL's from the email. extract_urls = config.get("extract_urls", None) if (extract_urls is not None and extract_urls.lower() in acceptable_config_yes): extract_urls = True + file_objects = [] # All possible file objects # Get Attachments # Get file names of attachments - for part in message.walk(): - filename = part.get_filename() - if filename is not None: - results.append({"values": filename, "type": 'email-attachment'}) - attachment_data = part.get_payload(decode=True) - # Base attachment data is default - attachment_files = [{"values": filename, "data": base64.b64encode(attachment_data).decode()}] - if unzip is True: # Attempt to unzip the attachment and return its files - zipped_files = ["doc", "docx", "dot", "dotx", "xls", - "xlsx", "xlm", "xla", "xlc", "xlt", - "xltx", "xlw", "ppt", "pptx", "pps", - "ppsx", "pot", "potx", "potx", "sldx", - "odt", "ods", "odp", "odg", "odf", - "fodt", "fods", "fodp", "fodg", "ott", - "uot"] + for attachment_name, attachment in email_object.attachments: + # Create file objects for the attachments + if not attachment_name: + attachment_name = 'NameMissing.txt' - zipped_filetype = False - for ext in zipped_files: - if filename.endswith(ext) is True: - zipped_filetype = True - if zipped_filetype == False: - try: - attachment_files += get_zipped_contents(filename, attachment_data) - except RuntimeError: # File is encrypted with a password - if zip_pass_crack is True: - if password_list is None: - password_list = get_zip_passwords(message) - password = test_zip_passwords(attachment_data, password_list) - if password is None: # Inform the analyst that we could not crack password - attachment_files[0]['comment'] = "Encrypted Zip: Password could not be cracked from message" - else: - attachment_files[0]['comment'] = """Original Zipped Attachment with Password {0}""".format(password) - attachment_files += get_zipped_contents(filename, attachment_data, password=password) - except zipfile.BadZipFile: # Attachment is not a zipfile - pass - for attch_item in attachment_files: - attch_item["type"] = 'malware-sample' - results.append(attch_item) - else: # Check email body part for urls - if (extract_urls is True and part.get_content_type() == 'text/html'): + temp_filename = Path(attachment_name) + zipped_files = ["doc", "docx", "dot", "dotx", "xls", "xlsx", "xlm", "xla", + "xlc", "xlt", "xltx", "xlw", "ppt", "pptx", "pps", "ppsx", + "pot", "potx", "potx", "sldx", "odt", "ods", "odp", "odg", + "odf", "fodt", "fods", "fodp", "fodg", "ott", "uot"] + # Attempt to unzip the attachment and return its files + if unzip and temp_filename.suffix[1:] not in zipped_files: + try: + unzip_attachement(attachment_name, attachment, email_object, file_objects) + except RuntimeError: # File is encrypted with a password + if zip_pass_crack is True: + password = test_zip_passwords(attachment, password_list) + if password: + unzip_attachement(attachment_name, attachment, email_object, file_objects, password) + else: # Inform the analyst that we could not crack password + f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + f_object.comment = "Encrypted Zip: Password could not be cracked from message" + file_objects.append(f_object) + file_objects.append(main_object) + file_objects += sections + email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') + except zipfile.BadZipFile: # Attachment is not a zipfile + # Just straight add the file + f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + file_objects.append(f_object) + file_objects.append(main_object) + file_objects += sections + email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') + else: + # Just straight add the file + f_object, main_object, sections = make_binary_objects(pseudofile=attachment, filename=attachment_name, standalone=False) + file_objects.append(f_object) + file_objects.append(main_object) + file_objects += sections + email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') + + mail_body = email_object.email.get_body(preferencelist=('html', 'plain')) + if extract_urls: + if mail_body: + charset = mail_body.get_content_charset() + if mail_body.get_content_type() == 'text/html': url_parser = HTMLURLParser() - charset = get_charset(part, get_charset(message)) - url_parser.feed(part.get_payload(decode=True).decode(charset)) + url_parser.feed(mail_body.get_payload(decode=True).decode(charset, errors='ignore')) urls = url_parser.urls - for url in urls: - results.append({"values": url, "type": "url"}) - r = {'results': results} + else: + urls = re.findall(r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+', mail_body.get_payload(decode=True).decode(charset, errors='ignore')) + for url in urls: + if not url: + continue + url_object = URLObject(url, standalone=False) + file_objects.append(url_object) + email_object.add_reference(url_object.uuid, 'includes', 'URL in email body') + + objects = [email_object.to_json()] + if file_objects: + objects += [o.to_json() for o in file_objects if o] + r = {'results': {'Object': [json.loads(o) for o in objects]}} return r -def get_zipped_contents(filename, data, password=None): +def unzip_attachement(filename, data, email_object, file_objects, password=None): """Extract the contents of a zipfile. Args: @@ -210,17 +147,23 @@ def get_zipped_contents(filename, data, password=None): "comment":"string here"} """ - with zipfile.ZipFile(io.BytesIO(data), "r") as zf: - unzipped_files = [] + with zipfile.ZipFile(data, "r") as zf: if password is not None: + comment = f'Extracted from {filename} with password "{password}"' password = str.encode(password) # Byte encoded password required + else: + comment = f'Extracted from {filename}' for zip_file_name in zf.namelist(): # Get all files in the zip file with zf.open(zip_file_name, mode='r', pwd=password) as fp: - file_data = fp.read() - unzipped_files.append({"values": zip_file_name, - "data": base64.b64encode(file_data).decode(), # Any password works when not encrypted - "comment": "Extracted from {0}".format(filename)}) - return unzipped_files + file_data = BytesIO(fp.read()) + f_object, main_object, sections = make_binary_objects(pseudofile=file_data, + filename=zip_file_name, + standalone=False) + f_object.comment = comment + file_objects.append(f_object) + file_objects.append(main_object) + file_objects += sections + email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') def test_zip_passwords(data, test_passwords): @@ -234,7 +177,7 @@ def test_zip_passwords(data, test_passwords): Returns a byte string containing a found password and None if password is not found. """ - with zipfile.ZipFile(io.BytesIO(data), "r") as zf: + with zipfile.ZipFile(data, "r") as zf: firstfile = zf.namelist()[0] for pw_test in test_passwords: byte_pwd = str.encode(pw_test) @@ -268,33 +211,31 @@ def get_zip_passwords(message): # Not checking for multi-part message because by having an # encrypted zip file it must be multi-part. - text_parts = [part for part in typed_subpart_iterator(message, 'text', 'plain')] - html_parts = [part for part in typed_subpart_iterator(message, 'text', 'html')] body = [] - # Get full message character set once - # Language example reference (using python2) - # http://ginstrom.com/scribbles/2007/11/19/parsing-multilingual-email-with-python/ - message_charset = get_charset(message) - for part in text_parts: - charset = get_charset(part, message_charset) - body.append(part.get_payload(decode=True).decode(charset)) - for part in html_parts: - charset = get_charset(part, message_charset) - html_part = part.get_payload(decode=True).decode(charset) - html_parser = HTMLTextParser() - html_parser.feed(html_part) - for text in html_parser.text_data: - body.append(text) + for part in message.walk(): + charset = part.get_content_charset() + if not charset: + charset = "utf-8" + if part.get_content_type() == 'text/plain': + body.append(part.get_payload(decode=True).decode(charset, errors='ignore')) + elif part.get_content_type() == 'text/html': + html_parser = HTMLTextParser() + payload = part.get_payload(decode=True) + if payload: + html_parser.feed(payload.decode(charset, errors='ignore')) + for text in html_parser.text_data: + body.append(text) raw_text = "\n".join(body).strip() # Add subject to text corpus to parse - subject = " " + message.get('Subject') - raw_text += subject + if "Subject" in message: + subject = " " + message.get('Subject') + raw_text += subject # Grab any strings that are marked off by special chars marking_chars = [["\'", "\'"], ['"', '"'], ['[', ']'], ['(', ')']] for char_set in marking_chars: - regex = re.compile("""\{0}([^\{1}]*)\{1}""".format(char_set[0], char_set[1])) + regex = re.compile(r"""\{0}([^\{1}]*)\{1}""".format(char_set[0], char_set[1])) marked_off = re.findall(regex, raw_text) possible_passwords += marked_off @@ -334,69 +275,19 @@ class HTMLURLParser(HTMLParser): def handle_starttag(self, tag, attrs): if tag == 'a': self.urls.append(dict(attrs).get('href')) - - -def get_charset(message, default="ascii"): - """Get a message objects charset - - Args: - message (email.message): Email message object to parse. - default (string): String containing default charset to return. - """ - if message.get_content_charset(): - return message.get_content_charset() - if message.get_charset(): - return message.get_charset() - return default - - -def get_decoded_header(header, value): - subject, encoding = decode_header(value)[0] - subject = subject.strip() # extra whitespace will mess up encoding - if isinstance(subject, bytes): - # Remove Byte Order Mark (BOM) from UTF strings - if encoding == 'utf-8': - return re.sub(codecs.BOM_UTF8, b"", subject).decode(encoding) - if encoding == 'utf-16': - return re.sub(codecs.BOM_UTF16, b"", subject).decode(encoding) - elif encoding == 'utf-32': - return re.sub(codecs.BOM_UTF32, b"", subject).decode(encoding) - # Try various UTF decodings for any unknown 8bit encodings - elif encoding == 'unknown-8bit': - for enc in [('utf-8', codecs.BOM_UTF8), - ('utf-32', codecs.BOM_UTF32), # 32 before 16 so it raises errors - ('utf-16', codecs.BOM_UTF16)]: - try: - return re.sub(enc[1], b"", subject).decode(enc[0]) - except UnicodeDecodeError: - continue - # If none of those encoding work return it in RFC2047 format - return str(subject) - # Provide RFC2047 format string if encoding is a unknown encoding - # Better to have the analyst decode themselves than to provide a mangled string - elif encoding is None: - return str(subject) - else: - return subject.decode(encoding) + if tag == 'img': + self.urls.append(dict(attrs).get('src')) def introspection(): - modulesetup = {} - try: - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + return mispattributes def version(): moduleinfo['config'] = moduleconfig return moduleinfo + if __name__ == '__main__': with open('tests/test_no_attach.eml', 'r') as email_file: handler(q=email_file.read()) diff --git a/misp_modules/modules/import_mod/goamlimport.py b/misp_modules/modules/import_mod/goamlimport.py index ecb0a2d..79b4cfe 100644 --- a/misp_modules/modules/import_mod/goamlimport.py +++ b/misp_modules/modules/import_mod/goamlimport.py @@ -1,6 +1,7 @@ -import json, datetime, time, base64 +import json +import time +import base64 import xml.etree.ElementTree as ET -from collections import defaultdict from pymisp import MISPEvent, MISPObject misperrors = {'error': 'Error'} @@ -8,15 +9,16 @@ moduleinfo = {'version': 1, 'author': 'Christian Studer', 'description': 'Import from GoAML', 'module-type': ['import']} moduleconfig = [] -mispattributes = {'inputSource': ['file'], 'output': ['MISP objects']} +mispattributes = {'inputSource': ['file'], 'output': ['MISP objects'], + 'format': 'misp_standard'} t_from_objects = {'nodes': ['from_person', 'from_account', 'from_entity'], - 'leaves': ['from_funds_code', 'from_country']} + 'leaves': ['from_funds_code', 'from_country']} t_to_objects = {'nodes': ['to_person', 'to_account', 'to_entity'], - 'leaves': ['to_funds_code', 'to_country']} + 'leaves': ['to_funds_code', 'to_country']} t_person_objects = {'nodes': ['addresses'], - 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', - 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} + 'leaves': ['first_name', 'middle_name', 'last_name', 'gender', 'title', 'mothers_name', 'birthdate', + 'passport_number', 'passport_country', 'id_number', 'birth_place', 'alias', 'nationality1']} t_account_objects = {'nodes': ['signatory'], 'leaves': ['institution_name', 'institution_code', 'swift', 'branch', 'non_banking_insitution', 'account', 'currency_code', 'account_name', 'iban', 'client_number', 'opened', 'closed', @@ -51,7 +53,7 @@ t_account_mapping = {'misp_name': 'bank-account', 'institution_name': 'instituti t_person_mapping = {'misp_name': 'person', 'comments': 'text', 'first_name': 'first-name', 'middle_name': 'middle-name', 'last_name': 'last-name', 'title': 'title', 'mothers_name': 'mothers-name', 'alias': 'alias', - 'birthdate': 'date-of-birth', 'birth_place': 'place-of-birth', 'gender': 'gender','nationality1': 'nationality', + 'birthdate': 'date-of-birth', 'birth_place': 'place-of-birth', 'gender': 'gender', 'nationality1': 'nationality', 'passport_number': 'passport-number', 'passport_country': 'passport-country', 'ssn': 'social-security-number', 'id_number': 'identity-card-number'} @@ -73,6 +75,7 @@ goAMLmapping = {'from_account': t_account_mapping, 'to_account': t_account_mappi nodes_to_ignore = ['addresses', 'signatory'] relationship_to_keep = ['signatory', 't_from', 't_from_my_client', 't_to', 't_to_my_client', 'address'] + class GoAmlParser(): def __init__(self): self.misp_event = MISPEvent() @@ -145,6 +148,7 @@ class GoAmlParser(): to_country_attribute = {'object_relation': 'to-country', 'value': to_country} misp_object.add_attribute(**to_country_attribute) + def handler(q=False): if q is False: return False @@ -157,16 +161,18 @@ def handler(q=False): aml_parser = GoAmlParser() try: aml_parser.read_xml(data) - except: + except Exception: misperrors['error'] = "Impossible to read XML data" return misperrors aml_parser.parse_xml() - r = {'results': [obj.to_json() for obj in aml_parser.misp_event.objects]} + r = {'results': {'Object': [obj.to_json() for obj in aml_parser.misp_event.objects]}} return r + def introspection(): return mispattributes + def version(): moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/import_mod/joe_import.py b/misp_modules/modules/import_mod/joe_import.py new file mode 100644 index 0000000..0753167 --- /dev/null +++ b/misp_modules/modules/import_mod/joe_import.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +import base64 +import json +from joe_parser import JoeParser + +misperrors = {'error': 'Error'} +userConfig = { + "Import PE": { + "type": "Boolean", + "message": "Import PE Information", + }, + "Mitre Att&ck": { + "type": "Boolean", + "message": "Import Mitre Att&ck techniques", + }, +} + +inputSource = ['file'] + +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', + 'description': 'Import for Joe Sandbox JSON reports', + 'module-type': ['import']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + config = { + "import_pe": bool(int(q["config"]["Import PE"])), + "mitre_attack": bool(int(q["config"]["Mitre Att&ck"])), + } + + data = base64.b64decode(q.get('data')).decode('utf-8') + if not data: + return json.dumps({'success': 0}) + + joe_parser = JoeParser(config) + joe_parser.parse_data(json.loads(data)['analysis']) + joe_parser.finalize_results() + return {'results': joe_parser.results} + + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + modulesetup['format'] = 'misp_standard' + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/import_mod/lastline_import.py b/misp_modules/modules/import_mod/lastline_import.py new file mode 100644 index 0000000..37f6249 --- /dev/null +++ b/misp_modules/modules/import_mod/lastline_import.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +Module (type "import") to import a Lastline report from an analysis link. +""" +import json + +import lastline_api + + +misperrors = { + "error": "Error", +} + +userConfig = { + "analysis_link": { + "type": "String", + "errorMessage": "Expected analysis link", + "message": "The link to a Lastline analysis" + }, +} + +inputSource = [] + +moduleinfo = { + "version": "0.1", + "author": "Stefano Ortolani", + "description": "Import a Lastline report from an analysis link.", + "module-type": ["import"] +} + +moduleconfig = [ + "username", + "password", + "verify_ssl", +] + + +def introspection(): + modulesetup = {} + try: + userConfig + modulesetup["userConfig"] = userConfig + except NameError: + pass + try: + inputSource + modulesetup["inputSource"] = inputSource + except NameError: + pass + modulesetup["format"] = "misp_standard" + return modulesetup + + +def version(): + moduleinfo["config"] = moduleconfig + return moduleinfo + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + # Parse the init parameters + try: + config = request["config"] + auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) + analysis_link = request["config"]["analysis_link"] + # The API url changes based on the analysis link host name + api_url = lastline_api.get_portal_url_from_task_link(analysis_link) + except Exception as e: + misperrors["error"] = "Error parsing configuration: {}".format(e) + return misperrors + + # Parse the call parameters + try: + task_uuid = lastline_api.get_uuid_from_task_link(analysis_link) + except (KeyError, ValueError) as e: + misperrors["error"] = "Error processing input parameters: {}".format(e) + return misperrors + + # Make the API calls + try: + api_client = lastline_api.PortalClient(api_url, auth_data, verify_ssl=config.get('verify_ssl', True).lower() in ("true")) + response = api_client.get_progress(task_uuid) + if response.get("completed") != 1: + raise ValueError("Analysis is not finished yet.") + + response = api_client.get_result(task_uuid) + if not response: + raise ValueError("Analysis report is empty.") + + except Exception as e: + misperrors["error"] = "Error issuing the API call: {}".format(e) + return misperrors + + # Parse and return + result_parser = lastline_api.LastlineResultBaseParser() + result_parser.parse(analysis_link, response) + + event = result_parser.misp_event + event_dictionary = json.loads(event.to_json()) + + return { + "results": { + key: event_dictionary[key] + for key in ("Attribute", "Object", "Tag") + if (key in event and event[key]) + } + } + + +if __name__ == "__main__": + """Test importing information from a Lastline analysis link.""" + import argparse + import configparser + + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config-file", dest="config_file") + parser.add_argument("-s", "--section-name", dest="section_name") + args = parser.parse_args() + c = configparser.ConfigParser() + c.read(args.config_file) + a = lastline_api.LastlineAbstractClient.get_login_params_from_conf(c, args.section_name) + + j = json.dumps( + { + "config": { + **a, + "analysis_link": ( + "https://user.lastline.com/portal#/analyst/task/" + "1fcbcb8f7fb400100772d6a7b62f501b/overview" + ) + } + } + ) + print(json.dumps(handler(j), indent=4, sort_keys=True)) + + j = json.dumps( + { + "config": { + **a, + "analysis_link": ( + "https://user.lastline.com/portal#/analyst/task/" + "f3c0ae115d51001017ff8da768fa6049/overview" + ) + } + } + ) + print(json.dumps(handler(j), indent=4, sort_keys=True)) diff --git a/misp_modules/modules/import_mod/mispjson.py b/misp_modules/modules/import_mod/mispjson.py index f9d52ec..a9c2226 100755 --- a/misp_modules/modules/import_mod/mispjson.py +++ b/misp_modules/modules/import_mod/mispjson.py @@ -2,7 +2,7 @@ import json import base64 misperrors = {'error': 'Error'} -userConfig = { }; +userConfig = {} inputSource = ['file'] @@ -19,23 +19,24 @@ def handler(q=False): r = {'results': []} request = json.loads(q) try: - mfile = base64.b64decode(request["data"]).decode('utf-8') - misp = json.loads(mfile) - event = misp['response'][0]['Event'] - for a in event["Attribute"]: - tmp = {} - tmp["values"] = a["value"] - tmp["categories"] = a["category"] - tmp["types"] = a["type"] - tmp["to_ids"] = a["to_ids"] - tmp["comment"] = a["comment"] - if a.get("data"): - tmp["data"] = a["data"] - r['results'].append(tmp) - except: - pass + mfile = base64.b64decode(request["data"]).decode('utf-8') + misp = json.loads(mfile) + event = misp['response'][0]['Event'] + for a in event["Attribute"]: + tmp = {} + tmp["values"] = a["value"] + tmp["categories"] = a["category"] + tmp["types"] = a["type"] + tmp["to_ids"] = a["to_ids"] + tmp["comment"] = a["comment"] + if a.get("data"): + tmp["data"] = a["data"] + r['results'].append(tmp) + except Exception: + pass return r + def introspection(): modulesetup = {} try: @@ -55,6 +56,7 @@ def version(): moduleinfo['config'] = moduleconfig return moduleinfo + if __name__ == '__main__': x = open('test.json', 'r') r = handler(q=x.read()) diff --git a/misp_modules/modules/import_mod/ocr.py b/misp_modules/modules/import_mod/ocr.py index aafe653..fef0fd1 100755 --- a/misp_modules/modules/import_mod/ocr.py +++ b/misp_modules/modules/import_mod/ocr.py @@ -1,16 +1,24 @@ +import sys import json import base64 - -from PIL import Image - -from pytesseract import image_to_string from io import BytesIO + +import logging + +log = logging.getLogger('ocr') +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) + misperrors = {'error': 'Error'} -userConfig = { }; +userConfig = {} inputSource = ['file'] -moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy', +moduleinfo = {'version': '0.2', 'author': 'Alexandre Dulaunoy', 'description': 'Optical Character Recognition (OCR) module for MISP', 'module-type': ['import']} @@ -18,14 +26,60 @@ moduleconfig = [] def handler(q=False): + # try to import modules and return errors if module not found + try: + from PIL import Image + except ImportError: + misperrors['error'] = "Please pip(3) install pillow" + return misperrors + + try: + # Official ImageMagick module + from wand.image import Image as WImage + except ImportError: + misperrors['error'] = "Please pip(3) install wand" + return misperrors + + try: + from pytesseract import image_to_string + except ImportError: + misperrors['error'] = "Please pip(3) install pytesseract" + return misperrors + if q is False: return False r = {'results': []} request = json.loads(q) - image = base64.b64decode(request["data"]) + document = base64.b64decode(request["data"]) + document = WImage(blob=document) + if document.format == 'PDF': + with document as pdf: + # Get number of pages + pages = len(pdf.sequence) + log.debug("PDF with {} page(s) detected".format(pages)) + # Create new image object where the height will be the number of pages. With huge PDFs this will overflow, break, consume silly memory etc… + img = WImage(width=pdf.width, height=pdf.height * pages) + # Cycle through pages and stitch it together to one big file + for p in range(pages): + log.debug("Stitching page {}".format(p + 1)) + image = img.composite(pdf.sequence[p], top=pdf.height * p, left=0) + # Create a png blob + image = img.make_blob('png') + log.debug("Final image size is {}x{}".format(pdf.width, pdf.height * (p + 1))) + else: + image = document + image_file = BytesIO(image) image_file.seek(0) - ocrized = image_to_string(Image.open(image_file)) + + try: + im = Image.open(image_file) + except IOError: + misperrors['error'] = "Corrupt or not an image file." + return misperrors + + ocrized = image_to_string(im) + freetext = {} freetext['values'] = ocrized freetext['types'] = ['freetext'] @@ -52,6 +106,7 @@ def version(): moduleinfo['config'] = moduleconfig return moduleinfo + if __name__ == '__main__': x = open('test.json', 'r') handler(q=x.read()) diff --git a/misp_modules/modules/import_mod/openiocimport.py b/misp_modules/modules/import_mod/openiocimport.py index c237bdc..074a464 100755 --- a/misp_modules/modules/import_mod/openiocimport.py +++ b/misp_modules/modules/import_mod/openiocimport.py @@ -63,7 +63,7 @@ def handler(q=False): "comment": getattr(attrib, 'comment', '')} # add tag if q.get('config') and q['config'].get('default tag') is not None: - toAppend["tags"] = q['config']['default tag'].split(",") + toAppend["tags"] = q['config']['default tag'].split(",") r["results"].append(toAppend) return r diff --git a/misp_modules/modules/import_mod/testimport.py b/misp_modules/modules/import_mod/testimport.py index 02369c5..891b3a6 100755 --- a/misp_modules/modules/import_mod/testimport.py +++ b/misp_modules/modules/import_mod/testimport.py @@ -1,28 +1,27 @@ import json import base64 -import csv misperrors = {'error': 'Error'} userConfig = { - 'number1': { - 'type': 'Integer', - 'regex': '/^[0-4]$/i', - 'errorMessage': 'Expected a number in range [0-4]', - 'message': 'Column number used for value' - }, - 'some_string': { - 'type': 'String', - 'message': 'A text field' - }, - 'boolean_field': { - 'type': 'Boolean', - 'message': 'Boolean field test' - }, - 'comment': { - 'type': 'Integer', - 'message': 'Column number used for comment' - } - }; + 'number1': { + 'type': 'Integer', + 'regex': '/^[0-4]$/i', + 'errorMessage': 'Expected a number in range [0-4]', + 'message': 'Column number used for value' + }, + 'some_string': { + 'type': 'String', + 'message': 'A text field' + }, + 'boolean_field': { + 'type': 'Boolean', + 'message': 'Boolean field test' + }, + 'comment': { + 'type': 'Integer', + 'message': 'Column number used for comment' + } +} inputSource = ['file', 'paste'] @@ -39,8 +38,8 @@ def handler(q=False): r = {'results': []} request = json.loads(q) request["data"] = base64.b64decode(request["data"]) - fields = ["value", "category", "type", "comment"] - r = {"results":[{"values":["192.168.56.1"], "types":["ip-src"], "categories":["Network activity"]}]} + # fields = ["value", "category", "type", "comment"] + r = {"results": [{"values": ["192.168.56.1"], "types":["ip-src"], "categories": ["Network activity"]}]} return r diff --git a/misp_modules/modules/import_mod/threatanalyzer_import.py b/misp_modules/modules/import_mod/threatanalyzer_import.py index 757f849..cbb9fef 100755 --- a/misp_modules/modules/import_mod/threatanalyzer_import.py +++ b/misp_modules/modules/import_mod/threatanalyzer_import.py @@ -15,7 +15,7 @@ misperrors = {'error': 'Error'} userConfig = {} inputSource = ['file'] -moduleinfo = {'version': '0.7', 'author': 'Christophe Vandeplas', +moduleinfo = {'version': '0.10', 'author': 'Christophe Vandeplas', 'description': 'Import for ThreatAnalyzer archive.zip/analysis.json files', 'module-type': ['import']} @@ -45,16 +45,21 @@ def handler(q=False): if re.match(r"Analysis/proc_\d+/modified_files/mapping\.log", zip_file_name): with zf.open(zip_file_name, mode='r', pwd=None) as fp: file_data = fp.read() - for line in file_data.decode().split('\n'): - if line: + for line in file_data.decode("utf-8", 'ignore').split('\n'): + if not line: + continue + if line.count('|') == 3: l_fname, l_size, l_md5, l_created = line.split('|') - l_fname = cleanup_filepath(l_fname) - if l_fname: - if l_size == 0: - pass # FIXME create an attribute for the filename/path - else: - # file is a non empty sample, upload the sample later - modified_files_mapping[l_md5] = l_fname + if line.count('|') == 4: + l_fname, l_size, l_md5, l_sha256, l_created = line.split('|') + l_fname = cleanup_filepath(l_fname) + if l_fname: + if l_size == 0: + results.append({'values': l_fname, 'type': 'filename', 'to_ids': True, + 'categories': ['Artifacts dropped', 'Payload delivery'], 'comment': ''}) + else: + # file is a non empty sample, upload the sample later + modified_files_mapping[l_md5] = l_fname # now really process the data for zip_file_name in zf.namelist(): # Get all files in the zip file @@ -84,8 +89,8 @@ def handler(q=False): results.append({ 'values': sample_filename, 'data': base64.b64encode(file_data).decode(), - 'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''}) - except Exception as e: + 'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''}) + except Exception: # no 'sample' in archive, might be an url analysis, just ignore pass @@ -94,7 +99,7 @@ def handler(q=False): results = process_analysis_json(json.loads(data.decode('utf-8'))) except ValueError: log.warning('MISP modules {0} failed: uploaded file is not a zip or json file.'.format(request['module'])) - return {'error': 'Uploaded file is not a zip or json file.'.format(request['module'])} + return {'error': 'Uploaded file is not a zip or json file.'} pass # keep only unique entries based on the value field results = list({v['values']: v for v in results}.values()) @@ -109,7 +114,22 @@ def process_analysis_json(analysis_json): for process in analysis_json['analysis']['processes']['process']: # print_json(process) if 'connection_section' in process and 'connection' in process['connection_section']: + # compensate for absurd behavior of the data format: if one entry = immediately the dict, if multiple entries = list containing dicts + # this will always create a list, even with only one item + if isinstance(process['connection_section']['connection'], dict): + process['connection_section']['connection'] = [process['connection_section']['connection']] + + # iterate over each entry for connection_section_connection in process['connection_section']['connection']: + # compensate for absurd behavior of the data format: if one entry = immediately the dict, if multiple entries = list containing dicts + # this will always create a list, even with only one item + for subsection in ['http_command', 'http_header']: + if isinstance(connection_section_connection[subsection], dict): + connection_section_connection[subsection] = [connection_section_connection[subsection]] + + if 'name_to_ip' in connection_section_connection: # TA 6.1 data format + connection_section_connection['@remote_ip'] = connection_section_connection['name_to_ip']['@result_addresses'] + connection_section_connection['@remote_hostname'] = connection_section_connection['name_to_ip']['@request_name'] connection_section_connection['@remote_ip'] = cleanup_ip(connection_section_connection['@remote_ip']) connection_section_connection['@remote_hostname'] = cleanup_hostname(connection_section_connection['@remote_hostname']) @@ -120,7 +140,7 @@ def process_analysis_json(analysis_json): # connection_section_connection['@remote_hostname'], # connection_section_connection['@remote_ip']) # ) - yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''}) + yield({'values': val, 'type': 'domain|ip', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) elif connection_section_connection['@remote_ip']: # print("connection_section_connection ip-dst: {} IDS:yes".format( # connection_section_connection['@remote_ip']) @@ -134,31 +154,31 @@ def process_analysis_json(analysis_json): if 'http_command' in connection_section_connection: for http_command in connection_section_connection['http_command']: # print('connection_section_connection HTTP COMMAND: {}\t{}'.format( - # http_command['@method'], # comment - # http_command['@url']) # url + # connection_section_connection['http_command']['@method'], # comment + # connection_section_connection['http_command']['@url']) # url # ) val = cleanup_url(http_command['@url']) if val: - yield({'values': val, 'type': 'url', 'categories': 'Network activity', 'to_ids': True, 'comment': http_command['@method']}) + yield({'values': val, 'type': 'url', 'categories': ['Network activity'], 'to_ids': True, 'comment': http_command['@method']}) if 'http_header' in connection_section_connection: for http_header in connection_section_connection['http_header']: if 'User-Agent:' in http_header['@header']: val = http_header['@header'][len('User-Agent: '):] - yield({'values': val, 'type': 'user-agent', 'categories': 'Network activity', 'to_ids': False, 'comment': ''}) + yield({'values': val, 'type': 'user-agent', 'categories': ['Network activity'], 'to_ids': False, 'comment': ''}) elif 'Host:' in http_header['@header']: val = http_header['@header'][len('Host: '):] if ':' in val: try: val_port = int(val.split(':')[1]) - except ValueError as e: + except ValueError: val_port = False val_hostname = cleanup_hostname(val.split(':')[0]) val_ip = cleanup_ip(val.split(':')[0]) if val_hostname and val_port: val_combined = '{}|{}'.format(val_hostname, val_port) # print({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) - yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) + yield({'values': val_combined, 'type': 'hostname|port', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) elif val_ip and val_port: val_combined = '{}|{}'.format(val_ip, val_port) # print({'values': val_combined, 'type': 'ip-dst|port', 'to_ids': True, 'comment': ''}) @@ -203,7 +223,7 @@ def process_analysis_json(analysis_json): # networkoperation_section_dns_request_by_name['@request_name'], # networkoperation_section_dns_request_by_name['@result_addresses']) # ) - yield({'values': val, 'type': 'domain|ip', 'categories': 'Network activity', 'to_ids': True, 'comment': ''}) + yield({'values': val, 'type': 'domain|ip', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) elif networkoperation_section_dns_request_by_name['@request_name']: # print("networkoperation_section_dns_request_by_name hostname: {} IDS:yes".format( # networkoperation_section_dns_request_by_name['@request_name']) @@ -227,14 +247,14 @@ def process_analysis_json(analysis_json): # networkpacket_section_connect_to_computer['@remote_port']) # ) val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_ip']) - yield({'values': val_combined, 'type': 'hostname|ip', 'to_ids': True, 'comment': ''}) + yield({'values': val_combined, 'type': 'domain|ip', 'to_ids': True, 'comment': ''}) elif networkpacket_section_connect_to_computer['@remote_hostname']: # print("networkpacket_section_connect_to_computer hostname: {} IDS:yes COMMENT:port {}".format( # networkpacket_section_connect_to_computer['@remote_hostname'], # networkpacket_section_connect_to_computer['@remote_port']) # ) val_combined = "{}|{}".format(networkpacket_section_connect_to_computer['@remote_hostname'], networkpacket_section_connect_to_computer['@remote_port']) - yield({'values': val_combined, 'type': 'hostname|port', 'to_ids': True, 'comment': ''}) + yield({'values': val_combined, 'type': 'hostname|port', 'categories': ['Network activity'], 'to_ids': True, 'comment': ''}) elif networkpacket_section_connect_to_computer['@remote_ip']: # print("networkpacket_section_connect_to_computer ip-dst: {} IDS:yes COMMENT:port {}".format( # networkpacket_section_connect_to_computer['@remote_ip'], @@ -305,7 +325,7 @@ def process_analysis_json(analysis_json): for stored_created_file in process['stored_files']['stored_created_file']: stored_created_file['@filename'] = cleanup_filepath(stored_created_file['@filename']) if stored_created_file['@filename']: - if stored_created_file['@filesize'] is not '0': + if stored_created_file['@filesize'] != '0': val = '{}|{}'.format(stored_created_file['@filename'], stored_created_file['@md5']) # print("stored_created_file filename|md5: {}|{} IDS:yes".format( # stored_created_file['@filename'], # filename @@ -326,7 +346,7 @@ def process_analysis_json(analysis_json): for stored_modified_file in process['stored_files']['stored_modified_file']: stored_modified_file['@filename'] = cleanup_filepath(stored_modified_file['@filename']) if stored_modified_file['@filename']: - if stored_modified_file['@filesize'] is not '0': + if stored_modified_file['@filesize'] != '0': val = '{}|{}'.format(stored_modified_file['@filename'], stored_modified_file['@md5']) # print("stored_modified_file MODIFY FILE: {}\t{}".format( # stored_modified_file['@filename'], # filename @@ -442,9 +462,9 @@ def cleanup_filepath(item): '\\AppData\\Roaming\\Adobe\\Acrobat\\9.0\\UserCache.bin', '\\AppData\\Roaming\\Macromedia\\Flash Player\\macromedia.com\\support\\flashplayer\\sys\\settings.sol', - '\\AppData\\Roaming\Adobe\\Flash Player\\NativeCache\\', + '\\AppData\\Roaming\\Adobe\\Flash Player\\NativeCache\\', 'C:\\Windows\\AppCompat\\Programs\\', - 'C:\~' # caused by temp file created by MS Office when opening malicious doc/xls/... + 'C:\\~' # caused by temp file created by MS Office when opening malicious doc/xls/... } if list_in_string(noise_substrings, item): return None diff --git a/misp_modules/modules/import_mod/vmray_import.py b/misp_modules/modules/import_mod/vmray_import.py index 88caa8f..824c970 100644 --- a/misp_modules/modules/import_mod/vmray_import.py +++ b/misp_modules/modules/import_mod/vmray_import.py @@ -8,68 +8,62 @@ This version supports import from different analyze jobs, starting from one samp Requires "vmray_rest_api" -TODO: - # Import one job (analyze_id) - # Import STIX package (XML version) +The expansion module vmray_submit and import module vmray_import are a two step +process to import data from VMRay. +You can automate this by setting the PyMISP example script 'vmray_automation' +as a cron job ''' import json -import re from ._vmray.vmray_rest_api import VMRayRESTAPI misperrors = {'error': 'Error'} inputSource = [] -moduleinfo = {'version': '0.1', 'author': 'Koen Van Impe', - 'description': 'Import VMRay (VTI) results', +moduleinfo = {'version': '0.2', 'author': 'Koen Van Impe', + 'description': 'Import VMRay results', 'module-type': ['import']} -userConfig = {'include_textdescr': {'type': 'Boolean', - 'message': 'Include textual description' - }, - 'include_analysisid': {'type': 'Boolean', - 'message': 'Include VMRay analysis_id text' +userConfig = {'include_analysisid': {'type': 'Boolean', + 'message': 'Include link to VMRay analysis' }, - 'only_network_info': {'type': 'Boolean', - 'message': 'Only include network (src-ip, hostname, domain, ...) information' - }, + 'include_analysisdetails': {'type': 'Boolean', + 'message': 'Include (textual) analysis details' + }, + 'include_vtidetails': {'type': 'Boolean', + 'message': 'Include VMRay Threat Identifier (VTI) rules' + }, + 'include_imphash_ssdeep': {'type': 'Boolean', + 'message': 'Include imphash and ssdeep' + }, + 'include_extracted_files': {'type': 'Boolean', + 'message': 'Include extracted files section' + }, + 'sample_id': {'type': 'Integer', 'errorMessage': 'Expected a sample ID', 'message': 'The VMRay sample_id' } } -moduleconfig = ['apikey', 'url'] - -include_textdescr = False -include_analysisid = False -only_network_info = False +moduleconfig = ['apikey', 'url', 'wait_period'] def handler(q=False): - global include_textdescr - global include_analysisid - global only_network_info + global include_analysisid, include_imphash_ssdeep, include_extracted_files, include_analysisdetails, include_vtidetails, include_static_to_ids if q is False: return False request = json.loads(q) - include_textdescr = request["config"].get("include_textdescr") - include_analysisid = request["config"].get("include_analysisid") - only_network_info = request["config"].get("only_network_info") - if include_textdescr == "1": - include_textdescr = True - else: - include_textdescr = False - if include_analysisid == "1": - include_analysisid = True - else: - include_analysisid = False - if only_network_info == "1": - only_network_info = True - else: - only_network_info = False + include_analysisid = bool(int(request["config"].get("include_analysisid"))) + include_imphash_ssdeep = bool(int(request["config"].get("include_imphash_ssdeep"))) + include_extracted_files = bool(int(request["config"].get("include_extracted_files"))) + include_analysisdetails = bool(int(request["config"].get("include_extracted_files"))) + include_vtidetails = bool(int(request["config"].get("include_vtidetails"))) + include_static_to_ids = True + + # print("include_analysisid: %s include_imphash_ssdeep: %s include_extracted_files: %s include_analysisdetails: %s include_vtidetails: %s" % ( include_analysisid, include_imphash_ssdeep, include_extracted_files, include_analysisdetails, include_vtidetails)) sample_id = int(request["config"].get("sample_id")) @@ -81,44 +75,67 @@ def handler(q=False): try: api = VMRayRESTAPI(request["config"].get("url"), request["config"].get("apikey"), False) vmray_results = {'results': []} + # Get all information on the sample, returns a set of finished analyze jobs data = vmrayGetInfoAnalysis(api, sample_id) if data["data"]: - vti_patterns_found = False for analysis in data["data"]: - analysis_id = analysis["analysis_id"] - + analysis_id = int(analysis["analysis_id"]) if analysis_id > 0: # Get the details for an analyze job analysis_data = vmrayDownloadAnalysis(api, analysis_id) if analysis_data: - if "analysis_vti_patterns" in analysis_data: - p = vmrayVtiPatterns(analysis_data["analysis_vti_patterns"]) - else: - p = vmrayVtiPatterns(analysis_data["vti_patterns"]) - if p and len(p["results"]) > 0: - vti_patterns_found = True - vmray_results = {'results': vmray_results["results"] + p["results"]} + if include_analysisdetails and "analysis_details" in analysis_data: + analysis_details = vmrayAnalysisDetails(analysis_data["analysis_details"], analysis_id) + if analysis_details and len(analysis_details["results"]) > 0: + vmray_results = {'results': vmray_results["results"] + analysis_details["results"]} + + if "classifications" in analysis_data: + classifications = vmrayClassifications(analysis_data["classifications"], analysis_id) + if classifications and len(classifications["results"]) > 0: + vmray_results = {'results': vmray_results["results"] + classifications["results"]} + + if include_extracted_files and "extracted_files" in analysis_data: + extracted_files = vmrayExtractedfiles(analysis_data["extracted_files"]) + if extracted_files and len(extracted_files["results"]) > 0: + vmray_results = {'results': vmray_results["results"] + extracted_files["results"]} + + if include_vtidetails and "vti" in analysis_data: + vti = vmrayVti(analysis_data["vti"]) + if vti and len(vti["results"]) > 0: + vmray_results = {'results': vmray_results["results"] + vti["results"]} + + if "artifacts" in analysis_data: + artifacts = vmrayArtifacts(analysis_data["artifacts"]) + if artifacts and len(artifacts["results"]) > 0: + vmray_results = {'results': vmray_results["results"] + artifacts["results"]} + if include_analysisid: a_id = {'results': []} - url1 = "https://cloud.vmray.com/user/analysis/view?from_sample_id=%u" % sample_id + url1 = request["config"].get("url") + "/user/analysis/view?from_sample_id=%u" % sample_id url2 = "&id=%u" % analysis_id url3 = "&sub=%2Freport%2Foverview.html" - a_id["results"].append({ "values": url1 + url2 + url3, "types": "link" }) - vmray_results = {'results': vmray_results["results"] + a_id["results"] } + a_id["results"].append({"values": url1 + url2 + url3, "types": "link"}) + vmray_results = {'results': vmray_results["results"] + a_id["results"]} + # Clean up (remove doubles) - if vti_patterns_found: + if len(vmray_results["results"]) > 0: vmray_results = vmrayCleanup(vmray_results) return vmray_results else: misperrors['error'] = "No vti_results returned or jobs not finished" return misperrors else: + if "result" in data: + if data["result"] == "ok": + return vmray_results + + # Fallback misperrors['error'] = "Unable to fetch sample id %u" % (sample_id) return misperrors - except: - misperrors['error'] = "Unable to access VMRay API" + except Exception as e: # noqa + misperrors['error'] = "Unable to access VMRay API : %s" % (e) return misperrors else: misperrors['error'] = "Not a valid sample id" @@ -158,80 +175,212 @@ def vmrayGetInfoAnalysis(api, sample_id): def vmrayDownloadAnalysis(api, analysis_id): ''' Get the details from an analysis''' if analysis_id: - data = api.call("GET", "/rest/analysis/%u/archive/additional/vti_result.json" % (analysis_id), raw_data=True) - return json.loads(data.read().decode()) + try: + data = api.call("GET", "/rest/analysis/%u/archive/logs/summary.json" % (analysis_id), raw_data=True) + return json.loads(data.read().decode()) + except Exception as e: # noqa + misperrors['error'] = "Unable to download summary.json for analysis %s" % (analysis_id) + return misperrors else: return False -def vmrayVtiPatterns(vti_patterns): - ''' Match the VTI patterns to MISP data''' +def vmrayVti(vti): + '''VMRay Threat Identifier (VTI) rules that matched for this analysis''' - if vti_patterns: + if vti: + r = {'results': []} + for rule in vti: + if rule == "vti_rule_matches": + vti_rule = vti["vti_rule_matches"] + for el in vti_rule: + if "operation_desc" in el: + comment = "" + types = ["text"] + values = el["operation_desc"] + r['results'].append({'types': types, 'values': values, 'comment': comment}) + + return r + + else: + return False + + +def vmrayExtractedfiles(extracted_files): + ''' Information about files which were extracted during the analysis, such as files that were created, modified, or embedded by the malware''' + + if extracted_files: + r = {'results': []} + + for file in extracted_files: + if "file_type" and "norm_filename" in file: + comment = "%s - %s" % (file["file_type"], file["norm_filename"]) + else: + comment = "" + + if "norm_filename" in file: + attr_filename_c = file["norm_filename"].rsplit("\\", 1) + if len(attr_filename_c) > 1: + attr_filename = attr_filename_c[len(attr_filename_c) - 1] + else: + attr_filename = "vmray_sample" + else: + attr_filename = "vmray_sample" + + if "md5_hash" in file and file["md5_hash"] is not None: + r['results'].append({'types': ["filename|md5"], 'values': '{}|{}'.format(attr_filename, file["md5_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if include_imphash_ssdeep and "imp_hash" in file and file["imp_hash"] is not None: + r['results'].append({'types': ["filename|imphash"], 'values': '{}|{}'.format(attr_filename, file["imp_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if "sha1_hash" in file and file["sha1_hash"] is not None: + r['results'].append({'types': ["filename|sha1"], 'values': '{}|{}'.format(attr_filename, file["sha1_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if "sha256_hash" in file and file["sha256_hash"] is not None: + r['results'].append({'types': ["filename|sha256"], 'values': '{}|{}'.format(attr_filename, file["sha256_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if include_imphash_ssdeep and "ssdeep_hash" in file and file["ssdeep_hash"] is not None: + r['results'].append({'types': ["filename|ssdeep"], 'values': '{}|{}'.format(attr_filename, file["ssdeep_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + + return r + + else: + return False + + +def vmrayClassifications(classification, analysis_id): + ''' List the classifications, tag them on a "text" attribute ''' + + if classification: + r = {'results': []} + types = ["text"] + comment = "" + values = "Classification : %s " % (", ".join(str(x) for x in classification)) + r['results'].append({'types': types, 'values': values, 'comment': comment}) + + return r + + else: + return False + + +def vmrayAnalysisDetails(details, analysis_id): + ''' General information about the analysis information ''' + + if details: + r = {'results': []} + types = ["text"] + comment = "" + if "execution_successful" in details: + values = "Analysis %s : execution_successful : %s " % (analysis_id, str(details["execution_successful"])) + r['results'].append({'types': types, 'values': values, 'comment': comment}) + if "termination_reason" in details: + values = "Analysis %s : termination_reason : %s " % (analysis_id, str(details["termination_reason"])) + r['results'].append({'types': types, 'values': values, 'comment': comment}) + if "result_str" in details: + values = "Analysis %s : result : %s " % (analysis_id, details["result_str"]) + r['results'].append({'types': types, 'values': values, 'comment': comment}) + + return r + + else: + return False + + +def vmrayArtifacts(patterns): + ''' IOCs that were seen during the analysis ''' + + if patterns: r = {'results': []} y = {'results': []} - for pattern in vti_patterns: - content = False - if pattern["category"] == "_network" and pattern["operation"] == "_download_data": - content = vmrayGeneric(pattern, "url", 1) - elif pattern["category"] == "_network" and pattern["operation"] == "_connect": - content = vmrayConnect(pattern) - elif pattern["category"] == "_network" and pattern["operation"] == "_install_server": - content = vmrayGeneric(pattern) + for pattern in patterns: + if pattern == "domains": + for el in patterns[pattern]: + values = el["domain"] + types = ["domain", "hostname"] + if "sources" in el: + sources = el["sources"] + comment = "Found in: " + ", ".join(str(x) for x in sources) + else: + comment = "" + r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids}) + if pattern == "files": + for el in patterns[pattern]: + filename_values = el["filename"] + attr_filename_c = filename_values.rsplit("\\", 1) + if len(attr_filename_c) > 1: + attr_filename = attr_filename_c[len(attr_filename_c) - 1] + else: + attr_filename = "" + filename_types = ["filename"] + filename_operations = el["operations"] + comment = "File operations: " + ", ".join(str(x) for x in filename_operations) + r['results'].append({'types': filename_types, 'values': filename_values, 'comment': comment}) - elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_alloc_wx_page": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_install_ipc_endpoint": - content = vmrayGeneric(pattern, "mutex", 1) - elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_crashed_process": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_read_from_remote_process": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_process" and pattern["operation"] == "_create_process_with_hidden_window": - content = vmrayGeneric(pattern) + # Run through all hashes + if "hashes" in el: + for hash in el["hashes"]: + if "md5_hash" in hash and hash["md5_hash"] is not None: + r['results'].append({'types': ["filename|md5"], 'values': '{}|{}'.format(attr_filename, hash["md5_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if include_imphash_ssdeep and "imp_hash" in hash and hash["imp_hash"] is not None: + r['results'].append({'types': ["filename|imphash"], 'values': '{}|{}'.format(attr_filename, hash["imp_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if "sha1_hash" in hash and hash["sha1_hash"] is not None: + r['results'].append({'types': ["filename|sha1"], 'values': '{}|{}'.format(attr_filename, hash["sha1_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if "sha256_hash" in hash and hash["sha256_hash"] is not None: + r['results'].append({'types': ["filename|sha256"], 'values': '{}|{}'.format(attr_filename, hash["sha256_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if include_imphash_ssdeep and "ssdeep_hash" in hash and hash["ssdeep_hash"] is not None: + r['results'].append({'types': ["filename|ssdeep"], 'values': '{}|{}'.format(attr_filename, hash["ssdeep_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) + if pattern == "ips": + for el in patterns[pattern]: + values = el["ip_address"] + types = ["ip-dst"] + if "sources" in el: + sources = el["sources"] + comment = "Found in: " + ", ".join(str(x) for x in sources) + else: + comment = "" - elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_delay_execution": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_anti_analysis" and pattern["operation"] == "_dynamic_api_usage": - content = vmrayGeneric(pattern) + r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids}) + if pattern == "mutexes": + for el in patterns[pattern]: + values = el["mutex_name"] + types = ["mutex"] + if "operations" in el: + sources = el["operations"] + comment = "Operations: " + ", ".join(str(x) for x in sources) + else: + comment = "" - elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_drop_pe_file": - content = vmrayGeneric(pattern, "filename", 1) - elif only_network_info is False and pattern["category"] == "_static" and pattern["operation"] == "_execute_dropped_pe_file": - content = vmrayGeneric(pattern, "filename", 1) + r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids}) + if pattern == "registry": + for el in patterns[pattern]: + values = el["reg_key_name"] + types = ["regkey"] + include_static_to_ids_tmp = include_static_to_ids + if "operations" in el: + sources = el["operations"] + if sources == ["access"]: + include_static_to_ids_tmp = False + comment = "Operations: " + ", ".join(str(x) for x in sources) + else: + comment = "" - elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory_system": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_memory_non_system": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_injection" and pattern["operation"] == "_modify_control_flow_non_system": - content = vmrayGeneric(pattern) - elif only_network_info is False and pattern["category"] == "_file_system" and pattern["operation"] == "_create_many_files": - content = vmrayGeneric(pattern) + r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids_tmp}) + if pattern == "urls": + for el in patterns[pattern]: + values = el["url"] + types = ["url"] + if "operations" in el: + sources = el["operations"] + comment = "Operations: " + ", ".join(str(x) for x in sources) + else: + comment = "" - elif only_network_info is False and pattern["category"] == "_hide_tracks" and pattern["operation"] == "_hide_data_in_registry": - content = vmrayGeneric(pattern, "regkey", 1) + r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids}) - elif only_network_info is False and pattern["category"] == "_persistence" and pattern["operation"] == "_install_startup_script": - content = vmrayGeneric(pattern, "regkey", 1) - elif only_network_info is False and pattern["category"] == "_os" and pattern["operation"] == "_enable_process_privileges": - content = vmrayGeneric(pattern) - - if content: - r["results"].append(content["attributes"]) - r["results"].append(content["text"]) - - # Remove empty results - r["results"] = [x for x in r["results"] if isinstance(x, dict) and len(x["values"]) != 0] + # Remove doubles for el in r["results"]: if el not in y["results"]: y["results"].append(el) return y + else: return False @@ -239,84 +388,7 @@ def vmrayVtiPatterns(vti_patterns): def vmrayCleanup(x): ''' Remove doubles''' y = {'results': []} - for el in x["results"]: if el not in y["results"]: y["results"].append(el) return y - - -def vmraySanitizeInput(s): - ''' Sanitize some input so it gets properly imported in MISP''' - if s: - s = s.replace('"', '') - s = re.sub('\\\\', r'\\', s) - return s - else: - return False - - -def vmrayGeneric(el, attr="", attrpos=1): - ''' Convert a 'generic' VTI pattern to MISP data''' - - r = {"values": []} - f = {"values": []} - - if el: - content = el["technique_desc"] - if content: - if attr: - # Some elements are put between \"\" ; replace them to single - content = content.replace("\"\"","\"") - content_split = content.split("\"") - # Attributes are between open " and close "; so use > - if len(content_split) > attrpos: - content_split[attrpos] = vmraySanitizeInput(content_split[attrpos]) - r["values"].append(content_split[attrpos]) - r["types"] = [attr] - - # Adding the value also as text to get the extra description, - # but this is pretty useless for "url" - if include_textdescr and attr != "url": - f["values"].append(vmraySanitizeInput(content)) - f["types"] = ["text"] - - return {"text": f, "attributes": r} - else: - return False - else: - return False - - -def vmrayConnect(el): - ''' Extension of vmrayGeneric , parse network connect data''' - ipre = re.compile("([0-9]{1,3}.){3}[0-9]{1,3}") - - r = {"values": []} - f = {"values": []} - - if el: - content = el["technique_desc"] - if content: - target = content.split("\"") - # port = (target[1].split(":"))[1] ## FIXME: not used - host = (target[1].split(":"))[0] - if ipre.match(str(host)): - r["values"].append(host) - r["types"] = ["ip-dst"] - else: - r["values"].append(host) - r["types"] = ["domain", "hostname"] - - f["values"].append(vmraySanitizeInput(target[1])) - f["types"] = ["text"] - - if include_textdescr: - f["values"].append(vmraySanitizeInput(content)) - f["types"] = ["text"] - - return {"text": f, "attributes": r} - else: - return False - else: - return False diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..be23ba7 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,97 @@ +# For Help: https://www.mkdocs.org/user-guide/configuration/ +# https://squidfunk.github.io/mkdocs-material/getting-started/ +# Requirements: mkdocs >1.x && mkdocs-material && markdown_include + +# Project information +site_name: MISP Modules Documentation +site_description: MISP Modules Project +site_author: MISP Project +site_url: https://www.misp-project.org/ + +# Repository +repo_name: 'MISP/misp-modules' +repo_url: https://github.com/MISP/misp-modules/ +edit_uri: "" + +use_directory_urls: true + +# Copyright +copyright: "Copyright © 2019 MISP Project" + +# Options +extra: + search: + languages: "en" + social: + - type: globe + link: https://www.misp-project.org/ + - type: github-alt + link: https://github.com/MISP + - type: twitter + link: https://twitter.com/MISPProject + +theme: + name: material + palette: + primary: 'white' + accent: 'blue' + language: en + favicon: img/favicon.ico + logo: img/misp.png + +# Extensions +markdown_extensions: + # - markdown_include.include: + # base_path: docs + # mkdcomments is buggy atm, see: https://github.com/ryneeverett/python-markdown-comments/issues/3 + #- mkdcomments + - toc: + permalink: "#" + baselevel: 2 + separator: "_" + - markdown.extensions.admonition + - markdown.extensions.codehilite: + guess_lang: false + - markdown.extensions.def_list + - markdown.extensions.footnotes + - markdown.extensions.meta + - markdown.extensions.toc: + permalink: true + - pymdownx.arithmatex + - pymdownx.betterem: + smart_enable: all + - pymdownx.caret + - pymdownx.critic + - pymdownx.details + - pymdownx.emoji: + emoji_generator: !!python/name:pymdownx.emoji.to_svg + - pymdownx.inlinehilite + - pymdownx.magiclink + - pymdownx.mark + - pymdownx.smartsymbols + - pymdownx.superfences + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.tilde + +nav: + - Home: 'index.md' + - Modules: + - Expansion Modules: expansion.md + - Export Modules: export_mod.md + - Import Modules: import_mod.md + - Install Guides: install.md + - Contribute: contribute.md + # - 'Ubuntu 18.04': 'INSTALL.ubuntu1804.md' + # - 'Kali Linux': 'INSTALL.kali.md' + # - 'RHEL7/CentOS7': 'INSTALL.rhel7.md' + # - 'RHEL8': 'INSTALL.rhel8.md' + # - Config Guides: + # - 'Elastic Search Logging': 'CONFIG.elasticsearch-logging.md' + # - 'Amazon S3 attachments': 'CONFIG.s3-attachments.md' + # - 'S/MIME': 'CONFIG.SMIME.md' + # - Update MISP: 'UPDATE.md' + # - Upgrading MISP: 'UPGRADE.md' + - About: + # - 'MISP Release Notes': 'Changelog.md' + - 'License': 'license.md' diff --git a/setup.py b/setup.py index f6c3a64..55ed8b7 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ setup( description='MISP modules are autonomous modules that can be used for expansion and other services in MISP', packages=find_packages(), entry_points={'console_scripts': ['misp-modules = misp_modules:main']}, + scripts=['tools/update_misp_modules.sh'], test_suite="tests", classifiers=[ 'License :: OSI Approved :: GNU Affero General Public License v3', @@ -23,18 +24,7 @@ setup( ], install_requires=[ 'tornado', - 'dnspython3', - 'requests', - 'urlarchiver', - 'passivetotal', - 'PyPDNS', - 'pypssl', - 'redis', - 'pyeupi', - 'ipasn-redis', - 'asnhistory', - 'pillow', - 'pytesseract', - 'shodan', + 'psutil', + 'redis>=3' ], ) diff --git a/tests/bodyhashdd.json b/tests/bodyhashdd.json index b6d256c..3bdfa82 100644 --- a/tests/bodyhashdd.json +++ b/tests/bodyhashdd.json @@ -1 +1 @@ -{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCB"} +{"module": "hashdd", "md5": "838DE99E82C5B9753BAC96D82C1A8DCC"} diff --git a/tests/test.py b/tests/test.py index d32bd00..37abcc3 100644 --- a/tests/test.py +++ b/tests/test.py @@ -57,6 +57,7 @@ class TestModules(unittest.TestCase): assert("mrxcls.sys" in values) assert("mdmcpq3.PNF" in values) + @unittest.skip("Need Rewrite") def test_email_headers(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, @@ -105,6 +106,7 @@ class TestModules(unittest.TestCase): self.assertEqual(types['email-reply-to'], 1) self.assertIn("", values) + @unittest.skip("Need Rewrite") def test_email_attachment_basic(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, @@ -129,6 +131,7 @@ class TestModules(unittest.TestCase): attch_data = base64.b64decode(i["data"]) self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + @unittest.skip("Need Rewrite") def test_email_attachment_unpack(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", @@ -159,6 +162,8 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + + @unittest.skip("Need Rewrite") def test_email_dont_unpack_compressed_doc_attachments(self): """Ensures that compressed """ @@ -192,6 +197,7 @@ class TestModules(unittest.TestCase): self.assertEqual(filesum.hexdigest(), '098da5381a90d4a51e6b844c18a0fecf2e364813c2f8b317cfdc51c21f2506a5') + @unittest.skip("Need Rewrite") def test_email_attachment_unpack_with_password(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", @@ -220,6 +226,7 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + @unittest.skip("Need Rewrite") def test_email_attachment_password_in_body(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", @@ -243,6 +250,7 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + @unittest.skip("Need Rewrite") def test_email_attachment_password_in_body_quotes(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", @@ -271,6 +279,7 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + @unittest.skip("Need Rewrite") def test_email_attachment_password_in_html_body(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", @@ -311,6 +320,7 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + @unittest.skip("Need Rewrite") def test_email_body_encoding(self): query = {"module":"email_import"} query["config"] = {"unzip_attachments": None, @@ -331,6 +341,7 @@ class TestModules(unittest.TestCase): self.assertIn('results', response, "No server results found.") + @unittest.skip("Need Rewrite") def test_email_header_proper_encoding(self): query = {"module":"email_import"} query["config"] = {"unzip_attachments": None, @@ -395,6 +406,7 @@ class TestModules(unittest.TestCase): self.assertIn("", values) + @unittest.skip("Need Rewrite") def test_email_header_malformed_encoding(self): query = {"module":"email_import"} query["config"] = {"unzip_attachments": None, @@ -462,6 +474,7 @@ class TestModules(unittest.TestCase): self.assertIn("", values) + @unittest.skip("Need Rewrite") def test_email_header_CJK_encoding(self): query = {"module":"email_import"} query["config"] = {"unzip_attachments": None, @@ -489,6 +502,7 @@ class TestModules(unittest.TestCase): self.assertNotEqual(RFC_format, i['values'], RFC_encoding_error) self.assertEqual(japanese_charset, i['values'], "Subject not properly decoded") + @unittest.skip("Need Rewrite") def test_email_malformed_header_CJK_encoding(self): query = {"module":"email_import"} query["config"] = {"unzip_attachments": None, @@ -519,6 +533,7 @@ class TestModules(unittest.TestCase): self.assertNotEqual(RFC_format, i['values'], RFC_encoding_error) self.assertEqual(japanese_charset, i['values'], "Subject not properly decoded") + @unittest.skip("Need Rewrite") def test_email_malformed_header_emoji_encoding(self): query = {"module":"email_import"} query["config"] = {"unzip_attachments": None, @@ -549,6 +564,7 @@ class TestModules(unittest.TestCase): self.assertNotEqual(RFC_format, i['values'], RFC_encoding_error) self.assertEqual(emoji_string, i['values'], "Subject not properly decoded") + @unittest.skip("Need Rewrite") def test_email_attachment_emoji_filename(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, @@ -576,6 +592,7 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, b'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + @unittest.skip("Need Rewrite") def test_email_attachment_password_in_subject(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": "true", @@ -606,6 +623,7 @@ class TestModules(unittest.TestCase): self.assertEqual(attch_data, 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-') + @unittest.skip("Need Rewrite") def test_email_extract_html_body_urls(self): query = {"module": "email_import"} query["config"] = {"unzip_attachments": None, diff --git a/tests/test_expansions.py b/tests/test_expansions.py new file mode 100644 index 0000000..b853c25 --- /dev/null +++ b/tests/test_expansions.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import unittest +import requests +from urllib.parse import urljoin +from base64 import b64encode +import json +import os + + +class TestExpansions(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.headers = {'Content-Type': 'application/json'} + self.url = "http://127.0.0.1:6666/" + self.dirname = os.path.dirname(os.path.realpath(__file__)) + self.sigma_rule = "title: Antivirus Web Shell Detection\r\ndescription: Detects a highly relevant Antivirus alert that reports a web shell\r\ndate: 2018/09/09\r\nmodified: 2019/10/04\r\nauthor: Florian Roth\r\nreferences:\r\n - https://www.nextron-systems.com/2018/09/08/antivirus-event-analysis-cheat-sheet-v1-4/\r\ntags:\r\n - attack.persistence\r\n - attack.t1100\r\nlogsource:\r\n product: antivirus\r\ndetection:\r\n selection:\r\n Signature: \r\n - \"PHP/Backdoor*\"\r\n - \"JSP/Backdoor*\"\r\n - \"ASP/Backdoor*\"\r\n - \"Backdoor.PHP*\"\r\n - \"Backdoor.JSP*\"\r\n - \"Backdoor.ASP*\"\r\n - \"*Webshell*\"\r\n condition: selection\r\nfields:\r\n - FileName\r\n - User\r\nfalsepositives:\r\n - Unlikely\r\nlevel: critical" + try: + with open(f'{self.dirname}/expansion_configs.json', 'rb') as f: + self.configs = json.loads(f.read().decode()) + except FileNotFoundError: + self.configs = {} + + def misp_modules_post(self, query): + return requests.post(urljoin(self.url, "query"), json=query) + + @staticmethod + def get_attribute(response): + data = response.json() + if not isinstance(data, dict): + print(json.dumps(data, indent=2)) + return data + return data['results']['Attribute'][0]['type'] + + @staticmethod + def get_data(response): + data = response.json() + if not isinstance(data, dict): + print(json.dumps(data, indent=2)) + return data + return data['results'][0]['data'] + + @staticmethod + def get_errors(response): + data = response.json() + if not isinstance(data, dict): + print(json.dumps(data, indent=2)) + return data + return data['error'] + + @staticmethod + def get_object(response): + data = response.json() + if not isinstance(data, dict): + print(json.dumps(data, indent=2)) + return data + return data['results']['Object'][0]['name'] + + @staticmethod + def get_values(response): + data = response.json() + if not isinstance(data, dict): + print(json.dumps(data, indent=2)) + return data + for result in data['results']: + values = result['values'] + if values: + return values[0] if isinstance(values, list) else values + return data['results'][0]['values'] + + def test_apiosintds(self): + query = {'module': 'apiosintds', 'ip-dst': '185.255.79.90'} + response = self.misp_modules_post(query) + try: + self.assertTrue(self.get_values(response).startswith('185.255.79.90 IS listed by OSINT.digitalside.it.')) + except AssertionError: + self.assertTrue(self.get_values(response).startswith('185.255.79.90 IS NOT listed by OSINT.digitalside.it.')) + + def test_apivoid(self): + module_name = "apivoid" + query = {"module": module_name, + "attribute": {"type": "domain", + "value": "circl.lu", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + "config": {}} + if module_name in self.configs: + query['config'] = self.configs[module_name] + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_object(response), 'dns-record') + except Exception: + self.assertTrue(self.get_errors(response).startswith('You do not have enough APIVoid credits')) + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'An API key for APIVoid is required.') + + def test_bgpranking(self): + query = {"module": "bgpranking", "AS": "13335"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET, US') + + def test_btc_steroids(self): + query = {"module": "btc_steroids", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} + response = self.misp_modules_post(query) + try: + self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC / -0.0005355700 BTC)')) + + except Exception: + self.assertEqual(self.get_values(response), 'Not a valid BTC address, or Balance has changed') + + def test_btc_scam_check(self): + query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), '1es14c7qlb5cyhlmuekctxlgc1fv2ti9da fraudolent bitcoin address') + + def test_circl_passivedns(self): + module_name = "circl_passivedns" + query = {"module": module_name, + "attribute": {"type": "domain", + "value": "circl.lu", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + "config": {}} + if module_name in self.configs: + query['config'] = self.configs[module_name] + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_object(response), 'passive-dns') + except Exception: + self.assertTrue(self.get_errors(response).startswith('There is an authentication error')) + else: + response = self.misp_modules_post(query) + self.assertTrue(self.get_errors(response).startswith('CIRCL Passive DNS authentication is missing.')) + + def test_circl_passivessl(self): + module_name = "circl_passivessl" + query = {"module": module_name, + "attribute": {"type": "ip-dst", + "value": "149.13.33.14", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + "config": {}} + if module_name in self.configs: + query['config'] = self.configs[module_name] + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_object(response), 'x509') + except Exception: + self.assertTrue(self.get_errors(response).startswith('There is an authentication error')) + else: + response = self.misp_modules_post(query) + self.assertTrue(self.get_errors(response).startswith('CIRCL Passive SSL authentication is missing.')) + + def test_countrycode(self): + query = {"module": "countrycode", "domain": "www.circl.lu"} + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_values(response), 'Luxembourg') + except Exception: + results = ('http://www.geognos.com/api/en/countries/info/all.json not reachable', 'Unknown', + 'Not able to get the countrycode references from http://www.geognos.com/api/en/countries/info/all.json') + self.assertIn(self.get_values(response), results) + + def test_cve(self): + query = {"module": "cve", "vulnerability": "CVE-2010-4444", "config": {"custom_API": "https://cve.circl.lu/api/cve/"}} + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response).startswith("Unspecified vulnerability in Oracle Sun Java System Access Manager")) + + def test_cve_advanced(self): + query = {"module": "cve_advanced", + "attribute": {"type": "vulnerability", + "value": "CVE-2010-4444", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + "config": {}} + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_object(response), 'vulnerability') + except Exception: + print(self.get_errors(response)) + + def test_dbl_spamhaus(self): + query = {"module": "dbl_spamhaus", "domain": "totalmateria.net"} + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_values(response), 'totalmateria.net - spam domain') + except Exception: + try: + self.assertTrue(self.get_values(response).startswith('None of DNS query names exist:')) + except Exception: + self.assertEqual(self.get_errors(response), 'Not able to reach dbl.spamhaus.org or something went wrong') + + def test_dns(self): + query = {"module": "dns", "hostname": "www.circl.lu", "config": {"nameserver": "8.8.8.8"}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), '149.13.33.14') + + def test_docx(self): + filename = 'test.docx' + with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + encoded = b64encode(f.read()).decode() + query = {"module": "docx_enrich", "attachment": filename, "data": encoded} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), '\nThis is an basic test docx file. ') + + def test_farsight_passivedns(self): + module_name = 'farsight_passivedns' + if module_name in self.configs: + query_types = ('domain', 'ip-src') + query_values = ('google.com', '8.8.8.8') + results = ('mail.casadostemperos.com.br', 'outmail.wphf.at') + for query_type, query_value, result in zip(query_types, query_values, results): + query = {"module": module_name, query_type: query_value, 'config': self.configs[module_name]} + response = self.misp_modules_post(query) + try: + self.assertIn(result, self.get_values(response)) + except Exception: + self.assertTrue(self.get_errors(response).startwith('Something went wrong')) + else: + query = {"module": module_name, "ip-src": "8.8.8.8"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'Farsight DNSDB apikey is missing') + + def test_haveibeenpwned(self): + query = {"module": "hibp", "email-src": "info@circl.lu"} + response = self.misp_modules_post(query) + to_check = self.get_values(response) + if to_check == "haveibeenpwned.com API not accessible (HTTP 401)": + self.skipTest(f"haveibeenpwned blocks travis IPs: {response}") + self.assertEqual(to_check, 'OK (Not Found)', response) + + def test_greynoise(self): + query = {"module": "greynoise", "ip-dst": "1.1.1.1"} + response = self.misp_modules_post(query) + value = self.get_values(response) + if value != 'GreyNoise API not accessible (HTTP 429)': + self.assertTrue(value.startswith('{"ip":"1.1.1.1","status":"ok"')) + + def test_ipasn(self): + query = {"module": "ipasn", + "attribute": {"type": "ip-src", + "value": "149.13.33.14", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_object(response), 'asn') + + def test_macaddess_io(self): + module_name = 'macaddress_io' + query = {"module": module_name, "mac-address": "44:38:39:ff:ef:57"} + if module_name in self.configs: + query["config"] = self.configs[module_name] + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response)['Valid MAC address'], 'True') + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'Authorization required') + + def test_macvendors(self): + query = {"module": "macvendors", "mac-address": "FC-A1-3E-2A-1C-33"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'Samsung Electronics Co.,Ltd') + + def test_ocr(self): + filename = 'misp-logo.png' + with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + encoded = b64encode(f.read()).decode() + query = {"module": "ocr_enrich", "attachment": filename, "data": encoded} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'Threat Sharing') + + def test_ods(self): + filename = 'test.ods' + with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + encoded = b64encode(f.read()).decode() + query = {"module": "ods_enrich", "attachment": filename, "data": encoded} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), '\n column_0\n0 ods test') + + def test_odt(self): + filename = 'test.odt' + with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + encoded = b64encode(f.read()).decode() + query = {"module": "odt_enrich", "attachment": filename, "data": encoded} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'odt test') + + def test_onyphe(self): + module_name = "onyphe" + query = {"module": module_name, "ip-src": "8.8.8.8"} + if module_name in self.configs: + query["config"] = self.configs[module_name] + response = self.misp_modules_post(query) + try: + self.assertTrue(self.get_values(response).startswith('https://pastebin.com/raw/')) + except Exception: + self.assertEqual(self.get_errors(response), 'no more credits') + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'Onyphe authentication is missing') + + def test_onyphe_full(self): + module_name = "onyphe_full" + query = {"module": module_name, "ip-src": "8.8.8.8"} + if module_name in self.configs: + query["config"] = self.configs[module_name] + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_values(response), '37.7510,-97.8220') + except Exception: + self.assertTrue(self.get_errors(response).startswith('Error ')) + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'Onyphe authentication is missing') + + def test_otx(self): + query_types = ('domain', 'ip-src', 'md5') + query_values = ('circl.lu', '8.8.8.8', '616eff3e9a7575ae73821b4668d2801c') + results = (('149.13.33.14', '149.13.33.17', '6f9814ba70e68c3bce16d253e8d8f86e04a21a2b4172a0f7631040096ba2c47a'), + 'ffc2595aefa80b61621023252b5f0ccb22b6e31d7f1640913cd8ff74ddbd8b41', + '8.8.8.8') + for query_type, query_value, result in zip(query_types, query_values, results): + query = {"module": "otx", query_type: query_value, "config": {"apikey": "1"}} + response = self.misp_modules_post(query) + try: + self.assertIn(self.get_values(response), result) + except KeyError: + # Empty results, which in this case comes from a connection error + continue + + def test_passivetotal(self): + module_name = "passivetotal" + query = {"module": module_name, "ip-src": "149.13.33.14", "config": {}} + if module_name in self.configs: + query["config"] = self.configs[module_name] + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_values(response), 'circl.lu') + except Exception: + self.assertIn(self.get_errors(response), ('We hit an error, time to bail!', 'API quota exceeded.')) + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'Configuration is missing from the request.') + + def test_pdf(self): + filename = 'test.pdf' + with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + encoded = b64encode(f.read()).decode() + query = {"module": "pdf_enrich", "attachment": filename, "data": encoded} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'Pdf test') + + def test_pptx(self): + filename = 'test.pptx' + with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + encoded = b64encode(f.read()).decode() + query = {"module": "pptx_enrich", "attachment": filename, "data": encoded} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), '\npptx test\n') + + def test_qrcode(self): + filename = 'qrcode.jpeg' + with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + encoded = b64encode(f.read()).decode() + query = {"module": "qrcode", "attachment": filename, "data": encoded} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), '1GXZ6v7FZzYBEnoRaG77SJxhu7QkvQmFuh') + + def test_ransomcoindb(self): + query = {"module": "ransomcoindb", + "attributes": {"type": "btc", + "value": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + if 'ransomcoindb' not in self.configs: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), "Ransomcoindb API key is missing") + + def test_rbl(self): + query = {"module": "rbl", "ip-src": "8.8.8.8"} + response = self.misp_modules_post(query) + try: + self.assertTrue(self.get_values(response).startswith('8.8.8.8.query.senderbase.org: "0-0=1|1=GOOGLE')) + except Exception: + self.assertEqual(self.get_errors(response), "No data found by querying known RBLs") + + def test_reversedns(self): + query = {"module": "reversedns", "ip-src": "8.8.8.8"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'dns.google.') + + def test_securitytrails(self): + module_name = "securitytrails" + query_types = ('ip-src', 'domain') + query_values = ('149.13.33.14', 'circl.lu') + results = ('circl.lu', 'ns4.eurodns.com') + if module_name in self.configs: + for query_type, query_value, result in zip(query_types, query_values, results): + query = {"module": module_name, query_type: query_value, "config": self.configs[module_name]} + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_values(response), result) + except Exception: + self.assertTrue(self.get_errors(response).startswith("You've exceeded the usage limits for your account.")) + else: + query = {"module": module_name, query_values[0]: query_types[0]} + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'SecurityTrails authentication is missing') + + def test_shodan(self): + module_name = "shodan" + query = {"module": module_name, "ip-src": "149.13.33.14"} + if module_name in self.configs: + query['config'] = self.configs[module_name] + response = self.misp_modules_post(query) + self.assertIn("circl.lu", self.get_values(response)) + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'Shodan authentication is missing') + + def test_sigma_queries(self): + query = {"module": "sigma_queries", "sigma": self.sigma_rule} + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response)['kibana'].startswith('[\n {\n "_id": "Antivirus-Web-Shell-Detection"')) + + def test_sigma_syntax(self): + query = {"module": "sigma_syntax_validator", "sigma": self.sigma_rule} + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response).startswith('Syntax valid:')) + + def test_sourcecache(self): + input_value = "https://www.misp-project.org/feeds/" + query = {"module": "sourcecache", "link": input_value} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), input_value) + self.assertTrue(self.get_data(response).startswith('PCFET0NUWVBFIEhUTUw+CjwhLS0KCUFyY2FuYSBieSBIVE1MN')) + + def test_stix2_pattern_validator(self): + query = {"module": "stix2_pattern_syntax_validator", "stix2-pattern": "[ipv4-addr:value = '8.8.8.8']"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'Syntax valid') + + def test_threatcrowd(self): + query_types = ('domain', 'ip-src', 'md5', 'whois-registrant-email') + query_values = ('circl.lu', '149.13.33.4', '616eff3e9a7575ae73821b4668d2801c', 'hostmaster@eurodns.com') + results = ('149.13.33.14', 'cve.circl.lu', 'devilreturns.com', 'navabi.lu') + for query_type, query_value, result in zip(query_types, query_values, results): + query = {"module": "threatcrowd", query_type: query_value} + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response), result) + + def test_threatminer(self): + query_types = ('domain', 'ip-src', 'md5') + query_values = ('circl.lu', '149.13.33.4', 'b538dbc6160ef54f755a540e06dc27cd980fc4a12005e90b3627febb44a1a90f') + results = ('149.13.33.14', 'f6ecb9d5c21defb1f622364a30cb8274f817a1a2', 'http://www.circl.lu/') + for query_type, query_value, result in zip(query_types, query_values, results): + query = {"module": "threatminer", query_type: query_value} + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response), result) + + def test_urlhaus(self): + query_types = ('domain', 'ip-src', 'sha256', 'url') + query_values = ('www.bestwpdesign.com', '79.118.195.239', + 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + 'http://79.118.195.239:1924/.i') + results = ('url', 'url', 'virustotal-report', 'virustotal-report') + for query_type, query_value, result in zip(query_types[:2], query_values[:2], results[:2]): + query = {"module": "urlhaus", + "attribute": {"type": query_type, + "value": query_value, + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_attribute(response), result) + for query_type, query_value, result in zip(query_types[2:], query_values[2:], results[2:]): + query = {"module": "urlhaus", + "attribute": {"type": query_type, + "value": query_value, + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_object(response), result) + + def test_urlscan(self): + module_name = "urlscan" + query = {"module": module_name, "url": "https://circl.lu/team"} + if module_name in self.configs: + query['config'] = self.configs[module_name] + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'circl.lu') + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'Urlscan apikey is missing') + + def test_virustotal_public(self): + module_name = "virustotal_public" + query_types = ('domain', 'ip-src', 'sha256', 'url') + query_values = ('circl.lu', '149.13.33.14', + 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + 'http://194.169.88.56:49151/.i') + results = ('whois', 'asn', 'file', 'virustotal-report') + if module_name in self.configs: + for query_type, query_value, result in zip(query_types, query_values, results): + query = {"module": module_name, + "attribute": {"type": query_type, + "value": query_value}, + "config": self.configs[module_name]} + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_object(response), result) + except Exception: + self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") + else: + query = {"module": module_name, + "attribute": {"type": query_types[0], + "value": query_values[0]}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") + + def test_virustotal(self): + module_name = "virustotal" + query_types = ('domain', 'ip-src', 'sha256', 'url') + query_values = ('circl.lu', '149.13.33.14', + 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + 'http://194.169.88.56:49151/.i') + results = ('domain-ip', 'asn', 'virustotal-report', 'virustotal-report') + if module_name in self.configs: + for query_type, query_value, result in zip(query_types, query_values, results): + query = {"module": module_name, + "attribute": {"type": query_type, + "value": query_value}, + "config": self.configs[module_name]} + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_object(response), result) + except Exception: + self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") + else: + query = {"module": module_name, + "attribute": {"type": query_types[0], + "value": query_values[0]}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") + + def test_vulners(self): + module_name = "vulners" + query = {"module": module_name, "vulnerability": "CVE-2010-3333"} + if module_name in self.configs: + query['config'] = self.configs[module_name] + response = self.misp_modules_post(query) + self.assertTrue(self.get_values(response).endswith('"RTF Stack Buffer Overflow Vulnerability."')) + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), "A Vulners api key is required for this module.") + + def test_wikidata(self): + query = {"module": "wiki", "text": "Google"} + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_values(response), 'http://www.wikidata.org/entity/Q95') + except KeyError: + self.assertEqual(self.get_errors(response), 'Something went wrong, look in the server logs for details') + except Exception: + self.assertEqual(self.get_values(response), 'No additional data found on Wikidata') + + def test_xforceexchange(self): + module_name = "xforceexchange" + query_types = ('domain', 'ip-src', 'md5', 'url', 'vulnerability') + query_values = ('mediaget.com', '61.255.239.86', '474b9ccf5ab9d72ca8a333889bbb34f0', + 'mediaget.com', 'CVE-2014-2601') + results = ('domain-ip', 'domain-ip', 'url', 'domain-ip', 'vulnerability') + if module_name in self.configs: + for query_type, query_value, result in zip(query_types, query_values, results): + query = {"module": module_name, + "attribute": {"type": query_type, + "value": query_value, + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, + "config": self.configs[module_name]} + response = self.misp_modules_post(query) + self.assertEqual(self.get_object(response), result) + else: + query = {"module": module_name, + "attribute": {"type": query_types[0], + "value": query_values[0], + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}} + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), "An API authentication is required (key and password).") + + def test_xlsx(self): + filename = 'test.xlsx' + with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: + encoded = b64encode(f.read()).decode() + query = {"module": "xlsx_enrich", "attachment": filename, "data": encoded} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), ' header\n0 xlsx test') + + def test_yara_query(self): + query = {"module": "yara_query", "md5": "b2a5abfeef9e36964281a31e17b57c97"} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'import "hash"\r\nrule MD5 {\r\n\tcondition:\r\n\t\thash.md5(0, filesize) == "b2a5abfeef9e36964281a31e17b57c97"\r\n}') + + def test_yara_validator(self): + query = {"module": "yara_syntax_validator", "yara": 'import "hash"\r\nrule MD5 {\r\n\tcondition:\r\n\t\thash.md5(0, filesize) == "b2a5abfeef9e36964281a31e17b57c97"\r\n}'} + response = self.misp_modules_post(query) + self.assertEqual(self.get_values(response), 'Syntax valid') diff --git a/tests/test_files/misp-logo.png b/tests/test_files/misp-logo.png new file mode 100644 index 0000000..5f2d4dd Binary files /dev/null and b/tests/test_files/misp-logo.png differ diff --git a/tests/test_files/qrcode.jpeg b/tests/test_files/qrcode.jpeg new file mode 100644 index 0000000..c013c9d Binary files /dev/null and b/tests/test_files/qrcode.jpeg differ diff --git a/tests/test_files/test.ods b/tests/test_files/test.ods new file mode 100644 index 0000000..080bb4a Binary files /dev/null and b/tests/test_files/test.ods differ diff --git a/tests/test_files/test.odt b/tests/test_files/test.odt new file mode 100644 index 0000000..a554904 Binary files /dev/null and b/tests/test_files/test.odt differ diff --git a/tests/test_files/test.pdf b/tests/test_files/test.pdf new file mode 100644 index 0000000..79d960a Binary files /dev/null and b/tests/test_files/test.pdf differ diff --git a/tests/test_files/test.pptx b/tests/test_files/test.pptx new file mode 100644 index 0000000..c1223b2 Binary files /dev/null and b/tests/test_files/test.pptx differ diff --git a/tests/test_files/test.xlsx b/tests/test_files/test.xlsx new file mode 100644 index 0000000..093daf2 Binary files /dev/null and b/tests/test_files/test.xlsx differ diff --git a/tests/yara_hash_module_test.yara b/tests/yara_hash_module_test.yara new file mode 100644 index 0000000..4674eef --- /dev/null +++ b/tests/yara_hash_module_test.yara @@ -0,0 +1,7 @@ +import "hash" +rule oui { + condition: + hash.md5(0, filesize) == "8764605c6f388c89096b534d33565802" and + hash.sha1(0, filesize) == "46aba99aa7158e4609aaa72b50990842fd22ae86" and + hash.sha256(0, filesize) == "ec5aedf5ecc6bdadd4120932170d1b10f6cfa175cfda22951dfd882928ab279b" +} diff --git a/tests/yara_pe_module_test.yara b/tests/yara_pe_module_test.yara new file mode 100644 index 0000000..4998b84 --- /dev/null +++ b/tests/yara_pe_module_test.yara @@ -0,0 +1,5 @@ +import "pe" +rule my_pe { + condition: + pe.imphash() == "eecc824da5b175f530705611127a6b41" +} diff --git a/tests/yara_test.py b/tests/yara_test.py new file mode 100644 index 0000000..ea88f03 --- /dev/null +++ b/tests/yara_test.py @@ -0,0 +1,22 @@ +import sys +try: + import yara +except (OSError, ImportError): + sys.exit("yara is missing, use 'pip3 install -I -r REQUIREMENTS' from the root of this repository to install it.") + +# Usage: python3 yara_test.py [yara files] +# with any yara file(s) in order to test if yara library is correctly installed. +# (it is also validating yara syntax) +# +# If no argument is given, this script takes the 2 yara test rules in the same directory +# in order to test if both yara modules we need work properly. + +files = sys.argv[1:] if len(sys.argv) > 1 else ['yara_hash_module_test.yara', 'yara_pe_module_test.yara'] + +for file_ in files: + try: + yara.compile(file_) + status = "Valid syntax" + except Exception as e: + status = e + print("{}: {}".format(file_, status)) diff --git a/tools/update_misp_modules.sh b/tools/update_misp_modules.sh new file mode 100755 index 0000000..372d146 --- /dev/null +++ b/tools/update_misp_modules.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e +set -x + +# Updates the MISP Modules while respecting the current permissions +# It aims to support the two following installation methods: +# * Everything is runinng on the same machine following the MISP installation guide. +# * The modules are installed using pipenv on a different machine from the one where MISP is running. + +if [ -d "/var/www/MISP" ] && [ -d "/usr/local/src/misp-modules" ] +then + echo "MISP is installed on the same machine, following the recommanded install script. Using MISP virtualenv." + PATH_TO_MISP="/var/www/MISP" + PATH_TO_MISP_MODULES="/usr/local/src/misp-modules" + + pushd ${PATH_TO_MISP_MODULES} + USER=`stat -c "%U" .` + sudo -H -u ${USER} git pull + sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -r REQUIREMENTS + sudo -H -u ${USER} ${PATH_TO_MISP}/venv/bin/pip install -U -e . + + service misp-modules restart + + popd +else + if ! [ -x "$(command -v pipenv)" ]; then + echo 'Error: pipenv not available, unable to automatically update.' >&2 + exit 1 + fi + + echo "Standalone mode, use pipenv from the current directory." + git pull + pipenv install +fi + +