diff --git a/.travis.yml b/.travis.yml index 4d551b2..dca4f44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,8 @@ before_install: install: - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr libfuzzy-dev libcaca-dev liblua5.3-dev - pip install pipenv - - pipenv install --dev + - pip install -r REQUIREMENTS + # - pipenv install --dev # install gtcaca - git clone git://github.com/stricaud/gtcaca.git - mkdir -p gtcaca/build @@ -37,20 +38,22 @@ install: - popd script: - - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 & + - pip install coverage + - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 & - pid=$! - sleep 5 - - pipenv run nosetests --with-coverage --cover-package=misp_modules + - nosetests --with-coverage --cover-package=misp_modules - kill -s KILL $pid - pushd ~/ - - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & + - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & - pid=$! - popd - sleep 5 - - pipenv run nosetests --with-coverage --cover-package=misp_modules + - nosetests --with-coverage --cover-package=misp_modules - kill -s KILL $pid - - pipenv run flake8 --ignore=E501,W503,E226 misp_modules + - pip install flake8 + - flake8 --ignore=E501,W503,E226 misp_modules after_success: - - pipenv run coverage combine .coverage* - - pipenv run codecov + - coverage combine .coverage* + - codecov diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 0000000..010d2a7 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,4602 @@ +# Changelog + + +## v2.4.141 (2021-04-19) + +### Changes + +* [tests] LiveCI set for RBL tests (network connectivity issues in the CI) [Alexandre Dulaunoy] + +* [rbl] Added a timeout parameter to change the resolver timeout & lifetime if needed. [chrisr3d] + +* [rbl] Small changes on the rbl list and the results handling. [chrisr3d] + +* [test] skip some tests if running in the CI (API limitation or specific host issues) [Alexandre Dulaunoy] + +* [tests] historical records in threatcrowd. [Alexandre Dulaunoy] + +* [test] fixing IP addresses. [Alexandre Dulaunoy] + +* [passivetotal] new test IP address. [Alexandre Dulaunoy] + +* [farsight] make PEP happy. [Alexandre Dulaunoy] + +* [requirements] openpyxl added. [Alexandre Dulaunoy] + +* [travis] missing dep. [Alexandre Dulaunoy] + +* [test expansion] IPv4 address of CIRCL updated. [Alexandre Dulaunoy] + +* [coverage] install. [Alexandre Dulaunoy] + +* [pipenv] removed. [Alexandre Dulaunoy] + +* [travis] get rid of pipenv. [Alexandre Dulaunoy] + +* [Pipfile.lock] updated. [Alexandre Dulaunoy] + +* [doc] fix index of mkdocs. [Alexandre Dulaunoy] + +* [documentation] updated. [Alexandre Dulaunoy] + +* [farsight_passivedns] Making first_time and last_time results human readable. [chrisr3d] + + - We get the datetime format instead of the raw + timestamp + +* Bump deps. [Raphaël Vinot] + +* [farsight_passivedns] Making first_time and last_time results human readable. [chrisr3d] + + - We get the datetime format instead of the raw + timestamp + +* [farsight_passivedns] Added input types for more flex queries. [chrisr3d] + + - Standard types still supported as before + - Name or ip lookup, with optional flex queries + - New attribute types added will only send flex + queries to the DNSDB API + +* [doc] fix #460 - rh install. [Alexandre Dulaunoy] + +* [requirements] fix 463. [Alexandre Dulaunoy] + +### Fix + +* [tests] Fixed btc_steroids test assertion. [chrisr3d] + +* [ocr_enrich] Making Pep8 happy. [chrisr3d] + +* [tests] Fixed variable names that have been changed with the latest commit. [chrisr3d] + +* [ocr_enrich] Fixed tesseract input format. [chrisr3d] + + - It looks like the `image_to_string` method now + assumes RGB format and the `imdecode` method + seems to give BGR format, so we convert the + image array before + +* [tests] Fixed tests for some modules waiting for standard MISP Attribute format as input. [chrisr3d] + +* [tests] Fixed hibp test which requires an API key. [chrisr3d] + +* [hibp] Fixed config handling to avoir KeyError exceptions. [chrisr3d] + +* [test] dns module. [Alexandre Dulaunoy] + +* [main] Disable duplicate JSON decoding. [Jakub Onderka] + +* [cve_advanced] Some CVEs are not in CWE format but in NVD-CWE-Other. [Alexandre Dulaunoy] + +* [farsight_passivedns] Fixed lookup_rdata_name results desclaration. [chrisr3d] + + - Getting generator as a list as it is already the + case for all the other results, so it avoids + issues to read the results by accidently looping + through the generator before it is actually + needed, which would lose the content of the + generator + - Also removed print that was accidently introduced + with the last commit + +* [farsight_passivedns] Excluding last_seen value for now, in order to get the available results. [chrisr3d] + + - With last_seen set we can easily get results + included in a certain time frame (between first + seen and last seen), but we do not get the + latest results. In order to get those ones, we + skip filtering on the time_last_before value + +* [farsight_passivedns] Fixed lookup_rdata_name results desclaration. [chrisr3d] + + - Getting generator as a list as it is already the + case for all the other results, so it avoids + issues to read the results by accidently looping + through the generator before it is actually + needed, which would lose the content of the + generator + - Also removed print that was accidently introduced + with the last commit + +* Making pep8 happy. [chrisr3d] + +* [farsight_passivedns] Fixed queries to the API. [chrisr3d] + + - Since flex queries input may be email addresses, + we nake sure we replace '@' by '.' in the flex + queries input. + - We also run the flex queries with the input as + is first, before runnning them as second time + with '.' characters escaped: '\\.' + +* Google.py module. [Jürgen Löhel] + + The search result does not include always 3 elements. It's better to + enumerate here. + The googleapi fails sometimes. Retry it 3 times. + +* Google.py module. [Jürgen Löhel] + + Corrects import for gh.com/abenassi/Google-Search-API. + +* Consider mail body as UTF-8 encoded. [Jakub Onderka] + +### Other + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [Alexandre Dulaunoy] + +* Fix; [tests] Changes on assertion statements that should fix the passivetotal, rbl & shodan tests. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [Alexandre Dulaunoy] + +* Merge pull request #435 from JakubOnderka/remove-duplicate-decoding. [Alexandre Dulaunoy] + + fix: [main] Remove duplicate JSON decoding + +* Add: [farsight_passivedns] Adding first_seen & last_seen (when available) in passivedns objects. [chrisr3d] + + - The object_relation `time_first` is added as the + `first_seen` value of the object + - Same with `time_last` -> `last_seen` + +* Merge branch 'main' of github.com:MISP/misp-modules into new_features. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into new_features. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into new_features. [chrisr3d] + +* Merge pull request #484 from GreyNoise-Intelligence/main. [Alexandre Dulaunoy] + + Update to GreyNoise expansion module + +* Update community api to released ver. [Brad Chiappetta] + +* Fix ver info. [Brad Chiappetta] + +* Updates for greynoise community api. [Brad Chiappetta] + +* Merge pull request #485 from jgwilson42/patch-1. [Alexandre Dulaunoy] + + Update README.md + +* Update README.md. [James Wilson] + + Ensure that the clone of misp-modules is owned by www-data + +* Merge pull request #482 from MISP/new_features. [Alexandre Dulaunoy] + + Farsight_passivedns module updated with new input types compatible with flex queries + +* Add: [farsight_passivedns] New lookup argument based on the first_seen & last_seen fields. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into new_features. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into new_features. [chrisr3d] + +* Merge pull request #481 from cocaman/main. [Alexandre Dulaunoy] + + Adding ThreatFox enrichment module + +* Adding additional tags. [Corsin Camichel] + +* First version of ThreatFox enrichment module. [Corsin Camichel] + +* Merge pull request #480 from cocaman/patch-1. [Alexandre Dulaunoy] + + updating "hibp" for API version 3 + +* Updating "hibp" for API version 3. [Corsin Camichel] + +* Merge pull request #477 from jloehel/fix/google-module. [Alexandre Dulaunoy] + + Fix/google module + +* Merge pull request #476 from digihash/patch-1. [Alexandre Dulaunoy] + + Update README.md + +* Update README.md. [Kevin Holvoet] + + Added fix based on https://github.com/MISP/MISP/issues/4045 + +* Merge pull request #475 from adammchugh/patch-3. [Alexandre Dulaunoy] + + Fixed the censys version + +* Fixed the censys version. [adammchugh] + + Unsure how I managed to get the version so wrong, but I have updated it to the current version and confirmed as working. + +* Merge pull request #474 from JakubOnderka/patch-4. [Alexandre Dulaunoy] + + fix: Consider mail body as UTF-8 encoded + +* Merge pull request #473 from adammchugh/patch-2. [Alexandre Dulaunoy] + + Change to pandas version requirement to address pip install failure + +* Included missing dependencies for censys and pyfaup. [adammchugh] + + Added censys dependency + Added pyfaup dependency + +* Change to pandas version requirement to address pip install failure. [adammchugh] + + Updated pandas version to 1.1.5 to allow pip install as defined at https://github.com/MISP/misp-modules to complete successfully. + +* Merge pull request #470 from adammchugh/patch-1. [Alexandre Dulaunoy] + + Update assemblyline_submit.py - Add verify SSL option + +* Update assemblyline_submit.py. [adammchugh] + +* Update assemblyline_query.py. [adammchugh] + +* Update assemblyline_submit.py. [adammchugh] + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [Alexandre Dulaunoy] + +* Update README long hyphen is not standard ASCII hyphen. [Alexandre Dulaunoy] + + Fix #464 + + +## v2.4.137 (2021-01-25) + +### Changes + +* Bump deps. [Raphaël Vinot] + +* Bump requirements. [Raphaël Vinot] + +* [pipenv] Enable email extras for PyMISP. [Jakub Onderka] + +### Fix + +* Bump PyMISP dep to latest. [Raphaël Vinot] + +* Use PyMISP from PyPi. [Raphaël Vinot] + +* Use pymisp from pypi. [Raphaël Vinot] + +* [pipenv] Missing clamd. [Jakub Onderka] + +### Other + +* Merge pull request #466 from NoDataFound/main. [Alexandre Dulaunoy] + + Corrected VMray rest API import + +* Corrected VMray rest API import. [Cory Kennedy] + + When loading misp-modules, the VMray module ```modules/expansion/vmray_submit.py ``` incorrectly imports the library. VMray's documentation and examples here: https://pypi.org/project/vmray-rest-api/#history also reflect this change as the correct import. + +* Merge pull request #457 from trustar/main. [Alexandre Dulaunoy] + + added more explicit error messages for indicators that return no enri… + +* Added more explicit error messages for indicators that return no enrichment data. [Jesse Hedden] + +* Merge pull request #452 from kuselfu/main. [Alexandre Dulaunoy] + + update vmray_import, add vmray_summary_json_import + +* Fix imports and unused variables. [Jens Thom] + +* Resolve merge conflict. [Jens Thom] + +* Merge remote-tracking branch 'upstream/main' into main. [Jens Thom] + +* Merge pull request #451 from JakubOnderka/versions-update. [Alexandre Dulaunoy] + + fix: [pipenv] Missing clamd + +* Merge pull request #450 from JakubOnderka/versions-update. [Alexandre Dulaunoy] + + chg: [pipenv] Enable email extras for PyMISP + +* Merge pull request #448 from HacknowledgeCH/export_defender_endpoint. [Alexandre Dulaunoy] + + Export defender endpoint + +* Fixed error reported by LGTM analysis. [milkmix] + +* Added documentation. [milkmix] + +* Added missing quotes. [milkmix] + +* Added URL support. [milkmix] + +* Typo in python src name. [milkmix] + +* Initial work on Defender for Endpoint export module. [milkmix] + +* * add parser for report version v1 and v2 * add summary JSON import module. [Jens Thom] + + +## v2.4.134 (2020-11-18) + +### New + +* [expansion] Added html_to_markdown module. [mokaddem] + + It fetches the HTML from the provided URL, performs a bit of DOM + clean-up then convert it into markdown + +* [clamav] Module for malware scan by ClamAV. [Jakub Onderka] + +* [passivedns, passivessl] Add support for ip-src|port and ip-dst|port. [Jakub Onderka] + +* Censys Expansion module. [Golbark] + +* Expansion module to query MALWAREbazaar API with some hash attribute. [chrisr3d] + +### Changes + +* [pipenv] Updated lock Pipfile again. [chrisr3d] + +* [pipenv] Updated lock Pipfile. [chrisr3d] + +* Added socialscan library in Pipfile and updated the lock file. [chrisr3d] + +* [documentation] Cleaner documentation directories & auto-generation. [chrisr3d] + + Including: + - A move of the previous `doc` and `docs` directories to `documentation` + - `documentation` is now the default directory + - The documentation previously under `doc` is now in `documentation/website` + - The mkdocs previously under `docs` is now in `documentation/mkdocs` + - All single JSON documentation files have been JQed + - Some small improvements to list fields displaying + +* [pipenv] Updated Pipfile. [chrisr3d] + +* [documentation] Updated the farsight-passivedns documentation. [chrisr3d] + +* [cpe] Added default limit to the results. [chrisr3d] + + - Results returned by CVE-search are sorted by + cvss score and limited in number to avoid + potential massive amount of data retuned back + to MISP. + - Users can overwrite the default limit with the + configuration already present as optional, and + can also set the limit to 0 to get the full list + of results + +* [farsight_passivedns] Now using the dnsdb2 python library. [chrisr3d] + + - Also updated the results parsing to check in + each returned result for every field if they are + included, to avoid key errors if any field is + missing + +* [cpe] Support of the new CVE-Search API. [chrisr3d] + +* [doc] Updated the farsight_passivedns module documentation. [chrisr3d] + +* [farsight_passivedns] More context added to the results. [chrisr3d] + + - References between the passive-dns objects and + the initial attribute + - Comment on object attributes mentioning whether + the results come from an rrset or an rdata + lookup + +* [farsight_passivedns] Rework of the module to return MISP objects. [chrisr3d] + + - All the results are parsed as passive-dns MISP + objects + - More love to give to the parsing to add + references between the passive-dns objects and + the input attribute, depending on the type of + the query (rrset or rdata), or the rrtype + (to be determined) + +* [cpe] Changed CVE-Search API default url. [chrisr3d] + +* [clamav] Add reference to original attribute. [Jakub Onderka] + +* [clamav] TCP port connection must be an integer. [Alexandre Dulaunoy] + +* Bump deps. [Raphaël Vinot] + +* Updated expansion modules documentation. [chrisr3d] + + - Added documentation for the missing modules + - Renamed some of the documentation files to match + with the module names and avoid issues within + the documentation file (README.md) with the link + of the miss-spelled module names + +* Updated the bgpranking expansion module test. [chrisr3d] + +* Updated documentation for the recently updated bgpranking module. [chrisr3d] + +* Updated the bgpranking expansion module to return MISP objects. [chrisr3d] + + - The module no longer returns freetext, since the + result returned to the freetext import as text + only allowed MISP to parse the same AS number as + the input attribute. + - The new result returned with the updated module + is an asn object describing more precisely the + AS number, and its ranking for a given day + +* Turned the Shodan expansion module into a misp_standard format module. [chrisr3d] + + - As expected with the misp_standard modules, the + input is a full attribute and the module is able + to return attributes and objects + - There was a lot of data that was parsed as regkey + attributes by the freetext import, the module now + parses properly the different field of the result + of the query returned by Shodan + +* Updated documentation about the greynoise module. [chrisr3d] + +* Updated Greynoise tests following the latest changes on the expansion module. [chrisr3d] + +* Making use of the Greynoise v2 API. [chrisr3d] + +* Bump deps. [Raphaël Vinot] + +* [doc] Added details about faup. [Steve Clement] + +* [doc] in case btc expansion fails, give another hint at why it fails. [Steve Clement] + +* [travis] Added gtcaca and liblua to faup. [Steve Clement] + +* [travis] Added py3.8. [Steve Clement] + +* Bump dependencies. [Raphaël Vinot] + + Should fix https://github.com/MISP/MISP/issues/5739 + +* Quick ransomdncoin test just to make sure the module loads. [chrisr3d] + + - I do not have any api key right now, so the test + should just reach the error + +* Catching missing config issue. [chrisr3d] + +### Fix + +* [pipenv] Removed duplicated dnsdb2 entry that I missed while merging conflict. [chrisr3d] + +* Removed debugging print command. [chrisr3d] + +* [tests] Less specific assertion for the rbl module test. [chrisr3d] + +* [farsight_passivedns] Fixed pep8 backslash issue. [chrisr3d] + +* [farsight_passivedns] Fixed issue with variable name. [chrisr3d] + +* [documentation] Added missing cpe module documentation. [chrisr3d] + +* [cpe] Fixed typo in vulnerable-configuration object relation fields. [chrisr3d] + +* [farsight_passivedns] Fixed typo in the lookup fields. [chrisr3d] + +* [farsight_passivedns] Uncommented mandatory field that was commented for tests. [chrisr3d] + +* [tests] Small fixes on the expansion tests. [chrisr3d] + +* [dnsdb] Avoiding AttributeError with the sys library, probably depending on the python version. [chrisr3d] + +* [documentation] Updated links to the scripts, with the default branch no longer being master, but main. [chrisr3d] + +* Typo. [chrisr3d] + +* Updated Pipfile. [chrisr3d] + +* [cpe] Typos and variable name issues fixed + Making the module available in MISP. [chrisr3d] + +* [cve-advanced] Using the cpe and weakness attribute types. [chrisr3d] + +* [cve_advanced] Avoiding potential MISP object references issues. [chrisr3d] + + - Adding objects as dictionaries in an event may + cause issues in some cases. It is better to pass + the MISP object as is, as it is already a valid + object since the MISPObject class is used + +* [virustotal_public] Resolve key error when user enrich hostname. [chrisr3d] + + - Same as #424 + +* [virustotal] Resolve key error when user enrich hostname. [Jakub Onderka] + +* Typo in EMailObject. [Raphaël Vinot] + + Fix #427 + +* Making pep8 happy. [chrisr3d] + +* Fixed pep8. [chrisr3d] + +* Fixed pep8 + some copy paste issues introduced with the latest commits. [chrisr3d] + +* Avoid issues with the attribute value field name. [chrisr3d] + + - The module setup allows 'value1' as attribute + value field name, but we want to make sure that + users passing standard misp format with 'value' + instead, will not have issues, as well as + keeping the current setup + +* [virustotal] Subdomains is optional in VT response. [Jakub Onderka] + +* Fixed list of sigma backends. [chrisr3d] + +* Fixed validators dependency issues. [chrisr3d] + + - Possible rollback if we get issues with virustotal + +* Removed multiple spaces to comply with pep8. [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +* Removed trustar_import module name in init to avoid validation issues. [chrisr3d] + + (until it is submitted via PR?) + +* [circl_passivessl] Return proper error for IPv6 addresses. [Jakub Onderka] + +* [circl_passivessl] Return not found error. [Jakub Onderka] + + If passivessl returns empty response, return Not found error instead of error in log + +* [circl_passivedns] Return not found error. [Jakub Onderka] + + If passivedns returns empty response, return Not found error instead of error in log + +* [pep] Comply to PEP E261. [Steve Clement] + +* [travis] gtcaca has no build directory. [Steve Clement] + +* [pip] pyfaup required. [Steve Clement] + +* [doc] corrected filenames for 2 docs. [Christophe Vandeplas] + +* Making pep8 happy. [chrisr3d] + +* Catching errors in the reponse of the query to URLhaus. [chrisr3d] + +* Making pep8 happy with indentation. [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +* Removed unused import. [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +* Making the module config available so the module works. [chrisr3d] + +* [VT] Disable SHA512 query for VT. [Jakub Onderka] + +### Other + +* Merge branch 'main' of github.com:MISP/misp-modules into chrisr3d_patch. [chrisr3d] + +* Merge pull request #429 from MISP/new_module. [Christian Studer] + + New module using socialscan to check the availability of an email address or username on some online platforms + +* Merge branch 'main' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Add: Added documentation for the socialscan new module. [chrisr3d] + + - Also quick fix of the message for an invalid + result or response concerning the queried email + address or username + +* Merge branch 'main' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Add: New module using socialscan library to check email addresses and usernames linked to accounts on online platforms. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into chrisr3d_patch. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into chrisr3d_patch. [chrisr3d] + +* Merge pull request #445 from chrisr3d/main. [Christian Studer] + + Added missing cpe module documentation + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [chrisr3d] + +* Add: [farsight-passivedns] Optional feature to submit flex queries. [chrisr3d] + + - The rrset and rdata queries remain the same but + with the parameter `flex_queries`, users can + also get the results of the flex rrnames & flex + rdata regex queries about their domain, hostname + or ip address + - Results can thus include passive-dns objects + containing the `raw_rdata` object_relation added + with 0a3e948 + +* Merge branch 'main' of github.com:MISP/misp-modules into chrisr3d_patch. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into chrisr3d_patch. [chrisr3d] + +* Merge branch 'chrisr3d_patch' of github.com:MISP/misp-modules into main. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into chrisr3d_patch. [chrisr3d] + +* Merge pull request #443 from trustar/main. [Alexandre Dulaunoy] + + fixed typo causing firstSeen and lastSeen to not be pulled from enric… + +* Fixed typo causing firstSeen and lastSeen to not be pulled from enrichment data. [Jesse Hedden] + +* Merge pull request #440 from MISP/chrisr3d_patch. [Alexandre Dulaunoy] + + Farsight passivedns module update + +* Merge pull request #437 from chrisr3d/main. [Alexandre Dulaunoy] + + New expansion module to get the vulnerabilities related to a CPE + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [chrisr3d] + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [chrisr3d] + +* Merge pull request #436 from MISP/new-html-to-markdown. [Christian Studer] + + new: [expansion] Added html_to_markdown module + +* Add: Documentation for the html_to_markdown expansion module. [chrisr3d] + +* Add: Added documentation for the cpe module. [chrisr3d] + +* Add: First shot of an expansio module to query cve-search with a cpe to get the related vulnerabilities. [chrisr3d] + +* Merge pull request #432 from JakubOnderka/clamav. [Alexandre Dulaunoy] + + chg: [clamav] Add reference to original attribute + +* Merge pull request #431 from JakubOnderka/clamav. [Alexandre Dulaunoy] + + new: [clamav] Module for malware scan by ClamAV + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [Raphaël Vinot] + +* Merge pull request #424 from JakubOnderka/vt-subdomains-fix. [Christian Studer] + + fix: [virustotal] Resolve key error when user enrich hostname + +* Merge pull request #426 from hildenjohannes/main. [Alexandre Dulaunoy] + + Recorded Future module: Add proxy support and User-Agent header + +* Add proxy support and User-Agent header. [johannesh] + +* Merge pull request #425 from elhoim/elhoim-patch-1. [Alexandre Dulaunoy] + + Disable correlation for detection-ratio attribute in virustotal.py + +* Disable correlation for detection-ratio in virustotal.py. [David André] + +* Merge pull request #422 from trustar/feat/EN-5047/MISP-manual-update. [Alexandre Dulaunoy] + + Feat/en 5047/misp manual update + +* Merge branch 'main' into feat/EN-5047/MISP-manual-update. [Jesse Hedden] + +* Merge pull request #420 from hildenjohannes/main. [Alexandre Dulaunoy] + + Fix typo error introduced in commit: 3b7a5c4dc2541f3b07baee69a7e8b969… + +* Fix typo error introduced in commit: 3b7a5c4dc2541f3b07baee69a7e8b9694a1627fc. [johannesh] + +* Merge pull request #417 from trustar/feat/EN-4664/trustar-misp. [Alexandre Dulaunoy] + + Feat/en 4664/trustar misp + +* Added description to readme. [Jesse Hedden] + +* Merge branch 'master' of github.com:trustar/misp-modules into feat/EN-4664/trustar-misp. [Jesse Hedden] + +* Removed obsoleted module name. [Jesse Hedden] + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [chrisr3d] + +* Merge pull request #416 from hildenjohannes/main. [Alexandre Dulaunoy] + + Add Recorded Future module documentation + +* Improve wording. [johannesh] + +* Add Recorded Future module documentation. [johannesh] + +* Add: Specific error message for misp_standard format expansion modules. [chrisr3d] + + - Checking if the input format is respected and + displaying an error message if it is not + +* Merge pull request #415 from hildenjohannes/main. [Alexandre Dulaunoy] + + Add Recorded Future expansion module + +* Add Recorded Future expansion module. [johannesh] + +* Added comments. [Jesse Hedden] + +* Added comments. [Jesse Hedden] + +* Added comments. [Jesse Hedden] + +* Added error checking. [Jesse Hedden] + +* Updating to include metadata and alter type of trustar link generated. [Jesse Hedden] + +* Merge pull request #1 from trustar/feat/EN-4664/trustar-misp. [Jesse Hedden] + + Feat/en 4664/trustar misp + +* Merge branch 'main' of github.com:MISP/misp-modules into main. [chrisr3d] + +* Merge pull request #411 from JakubOnderka/vt-subdomains-fix. [Alexandre Dulaunoy] + + fix: [virustotal] Subdomains is optional in VT response + +* Merge remote-tracking branch 'origin' into main. [chrisr3d] + +* Add: Trustar python library added to Pipfile. [chrisr3d] + +* Merge branch 'trustar-feat/EN-4664/trustar-misp' [chrisr3d] + +* Merge branch 'feat/EN-4664/trustar-misp' of https://github.com/trustar/misp-modules into trustar-feat/EN-4664/trustar-misp. [chrisr3d] + +* Removed obsolete file. [Jesse Hedden] + +* Corrected variable name. [Jesse Hedden] + +* Fixed indent. [Jesse Hedden] + +* Fixed incorrect attribute name. [Jesse Hedden] + +* Fixed metatag; convert summaries generator to list for error handling. [Jesse Hedden] + +* Added strip to remove potential whitespace. [Jesse Hedden] + +* Removed extra parameter. [Jesse Hedden] + +* Added try/except for TruSTAR API errors and additional comments. [Jesse Hedden] + +* Added comments and increased page size to max for get_indicator_summaries. [Jesse Hedden] + +* Uploaded TruSTAR logo. [Jesse Hedden] + +* Updated client metatag and version. [Jesse Hedden] + +* Added module documentation. [Jesse Hedden] + +* Added client metatag to trustar client. [Jesse Hedden] + +* Ready for code review. [Jesse Hedden] + +* WIP: initial push. [Jesse Hedden] + +* Initial commit. not a working product. need to create a class to manage the MISP event and TruStar client. [Jesse Hedden] + +* Merge pull request #381 from MISP/new_module. [Christian Studer] + + New module for MALWAREbazaar + +* Merge branch 'main' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge pull request #407 from JakubOnderka/patch-3. [Alexandre Dulaunoy] + + fix: [circl_passivessl] Return proper error for IPv6 addresses + +* Merge pull request #406 from JakubOnderka/ip-port. [Alexandre Dulaunoy] + + new: [passivedns, passivessl] Add support for ip-src|port and ip-dst|port + +* Merge pull request #405 from JakubOnderka/patch-2. [Alexandre Dulaunoy] + + fix: [circl_passivedns] Return not found error + +* Merge pull request #402 from MISP/dependabot/pip/httplib2-0.18.0. [Alexandre Dulaunoy] + + build(deps): bump httplib2 from 0.17.0 to 0.18.0 + +* Build(deps): bump httplib2 from 0.17.0 to 0.18.0. [dependabot[bot]] + + Bumps [httplib2](https://github.com/httplib2/httplib2) from 0.17.0 to 0.18.0. + - [Release notes](https://github.com/httplib2/httplib2/releases) + - [Changelog](https://github.com/httplib2/httplib2/blob/master/CHANGELOG) + - [Commits](https://github.com/httplib2/httplib2/compare/v0.17.0...v0.18.0) + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge pull request #395 from SteveClement/master. [Steve Clement] + + chg: [deps] pyfaup seems to be required but not installed + +* Merge pull request #393 from vmray-labs/update-vmray-module. [Alexandre Dulaunoy] + + Update vmray_submit module + +* Update vmray_submit. [Matthias Meidinger] + + The submit module hat some smaller issues with the reanalyze flag. + The source for the enrichment object has been changed and the robustness + of user supplied config parsing improved. + +* Merge pull request #388 from Golbark/censys_expansion. [Christophe Vandeplas] + + new: usr: Censys Expansion module + +* Fix variable issue in the loop. [Golbark] + +* Adding support for more input types, including multi-types. [Golbark] + +* Add: Added documentation for the latest new modules. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #380 from JakubOnderka/patch-1. [Christian Studer] + + csvimport: Return error if input is not valid UTF-8 + +* Csvimport: Return error if input is not valid UTF-8. [Jakub Onderka] + +* Merge pull request #379 from cudeso/master. [Alexandre Dulaunoy] + + Cytomic Orion MISP Module + +* Documentation for Cytomic Orion. [Koen Van Impe] + +* Update __init__ [Koen Van Impe] + +* Make Travis (a little bit) happy. [Koen Van Impe] + +* Cytomic Orion MISP Module. [Koen Van Impe] + + An expansion module to enrich attributes in MISP and share indicators + of compromise with Cytomic Orion + +* Merge pull request #377 from 0xbennyv/master. [Alexandre Dulaunoy] + + Added SophosLabs Intelix as expansion module + +* Removed Unused Import. [bennyv] + +* Fixed handler error handling for missing config. [bennyv] + +* Fixed formatting in README.md. [bennyv] + +* Updated the README.md for SOPHOSLabs Intelix. [bennyv] + +* Initial Build of SOPHOSLabs Intelix Product. [bennyv] + +* Merge pull request #374 from M0un/projet-m2-oun-gindt. [Christian Studer] + + Rendu projet master2 sécurité par Mathilde OUN et Vincent GINDT // No… + +* Rendu projet master2 sécurité par Mathilde OUN et Vincent GINDT // Nouveau module misp de recherche google sur les urls. [Mathilde Oun et Vincent Gindt] + +* Merge pull request #373 from seanthegeek/patch-1. [Christian Studer] + + Create missing __init__.py for _ransomcoindb + +* Revert change inteded for other patch. [Sean Whalen] + +* Install cmake to build faup. [Sean Whalen] + +* Create __init__.py. [Sean Whalen] + +* Merge pull request #371 from GlennHD/master. [Christian Studer] + + Added GeoIP_City and GeoIP_ASN Database Modules + +* Update geoip_asn.py. [GlennHD] + +* Update geoip_city.py. [GlennHD] + +* Added geoip_asn and geoip_city to load. [GlennHD] + +* Added GeoIP_ASN Enrichment module. [GlennHD] + +* Added GeoIP_City Enrichment module. [GlennHD] + +* Added GeoIP City and GeoIP ASN Info. [GlennHD] + +* Merge pull request #370 from JakubOnderka/vt-query-sha512. [Alexandre Dulaunoy] + + fix: [VT] Disable SHA512 query for VT + +* Merge pull request #368 from andurin/lastline_verifyssl. [Christian Studer] + + Lastline verify_ssl option + +* Lastline verify_ssl option. [Hendrik] + + Helps people with on-prem boxes + + +## v2.4.121 (2020-02-06) + +### Fix + +* Making pep8 happy. [chrisr3d] + +* [tests] Fixed BGP raking module test. [chrisr3d] + +### Other + +* Merge pull request #367 from joesecurity/master. [Christian Studer] + + joe: (1) allow users to disable PE object import (2) set 'to_ids' to False + +* Joe: (1) allow users to disable PE object import (2) set 'to_ids' to False. [Georg Schölly] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #365 from ostefano/analysis. [Alexandre Dulaunoy] + + change: migrate to analysis API when submitting files to Lastline + +* Change: migrate to analysis API when submitting tasks to Lastline. [Stefano Ortolani] + +* Merge pull request #364 from cudeso/master. [Christian Studer] + + 2nd fix for VT Public module + +* 2nd fix for VT Public module. [Koen Van Impe] + +* Fix error message in Public VT module. [Koen Van Impe] + + +## v2.4.120 (2020-01-21) + +### New + +* Updated ipasn and added vt_graph documentation. [chrisr3d] + +* Enrichment module for querying APIVoid with domain attributes. [chrisr3d] + +### Changes + +* Making ipasn module return asn object(s) [chrisr3d] + + - Latest changes on the returned value as string + broke the freetext parser, because no asn number + could be parsed when we return the full json + blob as a freetext attribute + - Now returning asn object(s) with a reference to + the initial attribute + +* Bumped pipfile.lock with up-to-date libraries and new vt_graph_api library requirement. [chrisr3d] + +* Checking attributes category. [chrisr3d] + + - We check the category before adding the + attribute to the event + - Checking if the category is correct and if not, + doing a case insensitive check + - If the category is not correct after the 2 first + tests, we simply delete it from the attribute + and pymisp will give the attribute a default + category value based on the atttribute type, at + the creation of the attribute + +* Regenerated the modules documentation following the latest changes. [chrisr3d] + +* Updated documentation following the latest changes on the passive dns module. [chrisr3d] + +* Made circl_passivedns module able to return MISP objects. [chrisr3d] + +* Updated documentation following the latest changes on the passive ssl module. [chrisr3d] + +* Made circl_passivessl module able to return MISP objects. [chrisr3d] + +* Bump dependencies. [Raphaël Vinot] + +* Install faup in travis. [Raphaël Vinot] + +* Deactive emails tests, need update. [Raphaël Vinot] + +* Update email import module, support objects. [Raphaël Vinot] + +* Bump dependencies. [Raphaël Vinot] + +### Fix + +* Fixed ipasn test input format + module version updated. [chrisr3d] + +* Updated ipasn test following the latest changes on the module. [chrisr3d] + +* Typo. [chrisr3d] + +* Fixed vt_graph imports. [chrisr3d] + +* Fixed pep8 in the new module and related libraries. [chrisr3d] + +* Fixed typo on function import. [chrisr3d] + +* [doc] Added APIVoid logo. [chrisr3d] + +* Making pep8 happy with whitespace after ':' [chrisr3d] + +* [tests] With values, tests are always better ... [chrisr3d] + +* [tests] Fixed copy paste issue. [chrisr3d] + +* [tests] Fixed error catching in passive dns and ssl modules. [chrisr3d] + +* [tests] Avoiding issues with btc addresses. [chrisr3d] + +* Making pep8 happy by having spaces around '+' operators. [chrisr3d] + +* [tests] Added missing variable. [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +* Missing dependency in travis. [Raphaël Vinot] + +* Properly install pymisp with file object dependencies. [Raphaël Vinot] + +* Quick variable name fix. [chrisr3d] + +* OTX tests were failing, new entry. [Raphaël Vinot] + +* Somewhat broken emails needed some love. [Raphaël Vinot] + +* MIssing parameter in skip. [Raphaël Vinot] + +* Missing pushd. [Raphaël Vinot] + +* Missing sudo. [Raphaël Vinot] + +### Other + +* Merge pull request #361 from VirusTotal/master. [Christian Studer] + + add vt_graph export module + +* Add vt-graph-api to the requirements. [Alvaro Garcia] + +* Add vt_graph export module. [Alvaro Garcia] + +* Merge pull request #360 from ec4n6/patch-1. [Alexandre Dulaunoy] + + Fix ipasn.py bug + +* Update ipasn.py. [Erick Cheng] + +* Add: Documentation for the new API Void module. [chrisr3d] + +* Add: [tests] Test case for the APIVoid module. [chrisr3d] + +* Revert "fix: [tests] Fixed copy paste issue" [chrisr3d] + + This reverts commit fd711475dd84749063f9ff15961453f90c804101. + +* Add: Test cases for reworked passive dns and ssl modules. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + + +## v2.4.119 (2019-12-03) + +### Changes + +* Bump dependencies. [Raphaël Vinot] + +* Use MISPObject in ransomcoindb. [Raphaël Vinot] + +* Reintroducing the limit to reduce the number of recursive calls to the API when querying for a domain. [chrisr3d] + +### Fix + +* Making pep8 happy. [chrisr3d] + +* Fixed AssemblyLine input description. [chrisr3d] + +* Fixed input types list since domain should not be submitted to AssemblyLine. [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +* Added missing AssemblyLine logo. [chrisr3d] + +* Avoiding KeyError exception when no result is found. [chrisr3d] + +### Other + +* Merge pull request #356 from ostefano/lastline. [Alexandre Dulaunoy] + + add: Modules to query/import/submit data from/to Lastline + +* Add: Modules to query/import/submit data from/to Lastline. [Stefano Ortolani] + +* Revert "Merge pull request #341 from StefanKelm/master" [Raphaël Vinot] + + This reverts commit 1df0d9152ed3346a9432393177c89e137bfc0c64, reversing + changes made to 6042619c6b7fb40fd77b5328f933e67e839e1e83. + + This PR was a fixing a typo in a test case. The typo is in a 3rd party + service. + +* Merge pull request #341 from StefanKelm/master. [Raphaël Vinot] + + Update test_expansions.py + +* Update test_expansions.py. [StefanKelm] + + Tiniest of typos + +* Merge branch 'aaronkaplan-master' [Raphaël Vinot] + +* Oops , use relative import. [aaronkaplan] + +* Use a helpful user-agent string. [aaronkaplan] + +* Final url fix. [aaronkaplan] + +* Revert "fix url" [aaronkaplan] + + This reverts commit 44130e2bf9842c03fb80245b90a873917b56df74. + +* Revert "fix url again" [aaronkaplan] + + This reverts commit c5924aee2543b268b296a57096e636261676b63c. + +* Fix url again. [aaronkaplan] + +* Fix url. [aaronkaplan] + +* Mention the ransomcoindb in the README file as a new module. [aaronkaplan] + +* Remove pprint. [aaronkaplan] + +* Initial version of the ransomcoindb expansion module. [aaronkaplan] + +* Merge pull request #352 from aaronkaplan/patch-1. [Alexandre Dulaunoy] + + Update README.md + +* Update README.md. [AaronK] + + fixes #351 + +* Add: Added documentation for the AssemblyLine query module. [chrisr3d] + +* Add: Module to query AssemblyLine and parse the results. [chrisr3d] + + - Takes an AssemblyLine submission link to query + the API and get the full submission report + - Parses the potentially malicious files and the + IPs, domains or URLs they are connecting to + - Possible improvement of the parsing filters in + order to include more data in the MISP event + +* Add: Added documentation and description in readme for the AssemblyLine submit module. [chrisr3d] + +* Add: Updated python dependencies to include the assemblyline_client library. [chrisr3d] + +* Add: New expansion module to submit samples and urls to AssemblyLine. [chrisr3d] + + +## v2.4.118 (2019-11-08) + +### Changes + +* Using EQL module description from blaverick62. [chrisr3d] + +* [test expansion] Enhanced results parsing. [chrisr3d] + +* [travis] skip E226 as it's more a question of style. [Alexandre Dulaunoy] + +* [apiosintds] make flake8 happy. [Alexandre Dulaunoy] + +* [Pipfile] apiosintDS added as required by new module. [Alexandre Dulaunoy] + +* [env] Pipfile updated. [Alexandre Dulaunoy] + +* [pipenv] updated. [Alexandre Dulaunoy] + +* Avoids returning empty values + easier results parsing. [chrisr3d] + +* Taking into consideration if a user agent is specified in the module configuration. [chrisr3d] + +* Updated csv import documentation. [chrisr3d] + +### Fix + +* Fixed csv file parsing. [chrisr3d] + +* Fixed Xforce Exchange authentication + rework. [chrisr3d] + + - Now able to return MISP objects + - Support of the xforce exchange authentication + with apikey & apipassword + +* Added urlscan & secuirtytrails modules in __init__ list. [chrisr3d] + +* Avoiding empty config error on passivetotal module. [chrisr3d] + +* More clarity on the exception raised on the securitytrails module. [chrisr3d] + +* Better exceptions handling on the passivetotal module. [chrisr3d] + +* Fixed results parsing for various module tests. [chrisr3d] + +* Fixed variable name. [chrisr3d] + +* Bumped Pipfile.lock with the latest libraries versions. [chrisr3d] + +* Fixed config parsing and the associated error message. [chrisr3d] + +* Fixed config parsing + results parsing. [chrisr3d] + + - Avoiding errors with config field when it is + empty or the apikey is not set + - Parsing all the results instead of only the + first one + +* Fixed VT results. [chrisr3d] + +* Making urlscan module available in MISP for ip attributes. [chrisr3d] + + - As expected in the the handler function + +* Avoiding various modules to fail with uncritical issues. [chrisr3d] + + - Avoiding securitytrails to fail with an unavailable + feature for free accounts + - Avoiding urlhaus to fail with input attribute + fields that are not critical for the query and + results + - Avoiding VT modules to fail when a certain + resource does not exist in the dataset + +* Fixed config field parsing for various modules. [chrisr3d] + + - Same as previous commit + +* [expansion] Better config field handling for various modules. [chrisr3d] + + - Testing if config is present before trying to + look whithin the config field + - The config field should be there when the module + is called form MISP, but it is not always the + case when the module is queried from somewhere else + +* [test expansion] Using CVE with lighter results. [chrisr3d] + +* Avoid issues when some config fields are not set. [chrisr3d] + +* Updated pipfile.lock with the correct geoip2 library info. [chrisr3d] + +* Fixed requirements for pymisp and geoip python libraries. [chrisr3d] + +* Fixed Geoip with the supported python library + fixed Geolite db path management. [chrisr3d] + +* Removed unused self param turning the associated functions into static methods. [chrisr3d] + +* Updates following the latest CVE-search version. [chrisr3d] + + - Support of the new vulnerable configuration + field for CPE version > 2.2 + - Support of different 'unknown CWE' message + +* Fixed module names with - to avoid errors with python paths. [chrisr3d] + +* Fixed tesseract python library issues. [Christian Studer] + + - Avoiding 'tesseract is not installed or it's not in your path' issues + +* Using absolute path to open files instead of relative path. [chrisr3d] + +* Removed unused import\ [chrisr3d] + +* Handling issues when the otx api is queried too often in a short time. [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +* Avoiding empty values + Fixed empty types error + Fixed filename KeyError. [chrisr3d] + +* Fixed ThreatMiner results parsing. [chrisr3d] + +* Catching wikidata errors properly + fixed errors parsing. [chrisr3d] + +* Grouped two if conditions to avoid issues with variable unassigned if the second condition is not true. [chrisr3d] + +* Handling errors and exceptions for expansion modules tests that could fail due to a connection error. [chrisr3d] + +* Considering the case of empty results. [chrisr3d] + +* Catching results exceptions properly. [chrisr3d] + +* Catching exceptions and results properly depending on the cases. [chrisr3d] + +* Handling cases where there is no result from the query. [chrisr3d] + +* DBL spamhaus test. [chrisr3d] + +* Quick typo & dbl spamhaus test fixes. [chrisr3d] + +* Fixed pattern parsing + made the module hover only. [chrisr3d] + +* Travis tests should be happy now. [chrisr3d] + +* Copy paste syntax error. [chrisr3d] + +* Fixed greynoise test following the latest changes on the module. [chrisr3d] + +* Returning results in text format. [chrisr3d] + + - Makes the hover functionality display the full + result instead of skipping the records list + +* Making pep8 happy. [chrisr3d] + +* Avoiding errors with uncommon lines. [chrisr3d] + + - Excluding first from data parsed all lines that + are comments or empty + - Skipping lines with failing indexes + +* Fixed unassigned variable name. [chrisr3d] + +* Removed no longer used variables. [chrisr3d] + +* Csv import rework & improvement. [chrisr3d] + + - More efficient parsing + - Support of multiple csv formats + - Possibility to customise headers + - More improvement to come for external csv file + +* Making pep8 happy. [chrisr3d] + +* [tests] Fixed tests to avoid config issues with the cve module. [chrisr3d] + + - Config currently empty in the module, but being + updated soon with a pending pull request + +### Other + +* Add: Updated documentation with the EQL export module. [chrisr3d] + +* Merge branch 'master' of github.com:blaverick62/misp-modules. [chrisr3d] + +* Added documentation json for new modules. [Braden Laverick] + +* Updated README to include EQL modules. [Braden Laverick] + +* Add: Xforce Exchange module tests. [chrisr3d] + +* Merge pull request #347 from MISP/tests. [Christian Studer] + + More advanced expansion tests + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Add: Updated documentation with the latest modules info. [chrisr3d] + +* Updated README with new modules and fixed some links. [chrisr3d] + +* Add: Added test for vulners module. [chrisr3d] + +* Add: Added qrcode module test with its test image. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Merge pull request #346 from blaverick62/master. [Alexandre Dulaunoy] + + EQL Query Generation Modules + +* Removed extraneous comments and unused imports. [Braden Laverick] + +* Fixed python links. [Braden Laverick] + +* Changed file name to mass eql export. [Braden Laverick] + +* Fixed comments. [Braden Laverick] + +* Added ors for compound queries. [Braden Laverick] + +* Fixed syntax error. [Braden Laverick] + +* Changed to single attribute EQL. [Braden Laverick] + +* Added EQL enrichment module. [Braden Laverick] + +* Fixed string formatting. [Braden Laverick] + +* Fixed type error in JSON parsing. [Braden Laverick] + +* Attempting to import endgame module. [Braden Laverick] + +* Added endgame export to __all__ [Braden Laverick] + +* Added EQL export test module. [Braden Laverick] + +* Add: [test expansion] Added various tests for modules with api authentication. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Add: [test expansion] New modules tests. [chrisr3d] + + - Starting testing some modules with api keys + - Testing new apiosintDS module + +* Merge pull request #344 from davidonzo/master. [Alexandre Dulaunoy] + + Added apiosintDS module to query OSINT.digitalside.it services + +* Added apiosintDS module to query OSINT.digitalside.it services. [Davide] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #345 from 0xmilkmix/fix_geoip2. [Alexandre Dulaunoy] + + updated to geoip2 to support mmdb format + +* Updated to geoip2 to support mmdb format. [milkmix] + +* Add: cve_advanced module test + functions to test attributes and objects results. [chrisr3d] + +* Merge pull request #342 from MISP/tests. [Christian Studer] + + More expansion tests + +* Merge branch 'tests' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Add: Tests for all the office, libreoffice, pdf & OCR enrich modules. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Add: threatminer module test. [chrisr3d] + +* Add: Tests for expansion modules with different input types. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #339 from MISP/tests. [Christian Studer] + + Expansion modules tests update + +* Add: Added tests for the rest of the easily testable expansion modules. [chrisr3d] + + - More tests for more complex modules to come soon + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Merge branch 'tests' of github.com:MISP/misp-modules. [chrisr3d] + +* Add: Tests for sigma queries and syntax validator modules. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into tests. [chrisr3d] + +* Add: More modules tested. [chrisr3d] + +* Add: Added tests for some expansion modules without API key required. [chrisr3d] + + - More tests to come + +* Merge pull request #338 from MISP/features_csvimport. [Christian Studer] + + Fixed the CSV import module + +* Merge pull request #335 from FafnerKeyZee/patch-2. [Christian Studer] + + Travis should not be complaining with the tests after the latest update on "test_cve" + +* Adding custom API. [Fafner [_KeyZee_]] + + Adding the possibility to have our own API server. + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #334 from FafnerKeyZee/patch-1. [Alexandre Dulaunoy] + + Cleaning the error message + +* Cleaning the error message. [Fafner [_KeyZee_]] + + The original message can be confusing is the user change to is own API. + + +## v2.4.116 (2019-09-17) + +### Other + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #329 from 8ear/8ear-add-mkdocs-documentation. [Alexandre Dulaunoy] + + Update mkdocs documentation + +* Fixing Install.md. [8ear] + +* Fix Install.md. [8ear] + +* Change Install documentation. [8ear] + +* Merge pull request #328 from 8ear/8ear-add-docker-capabilitites. [Alexandre Dulaunoy] + + Add Docker Capabilitites + +* Add .travis.yml command for docker build. [8ear] + +* Merge github.com:MISP/misp-modules into 8ear-add-docker-capabilitites. [8ear] + +* Disable not required package virtualenv for final stage. [8ear] + +* Fix entrypoint bug. [8ear] + +* Improve the Dockerfile. [8ear] + +* Add Dockerfile, Entrypoint and Healthcheck script. [8ear] + +* Update install doc. [8ear] + +* Bugfixing for MISP-modules. [8ear] + +* Add: New parameter to specify a custom CVE API to query. [chrisr3d] + + - Any API specified here must return the same + format as the CIRCL CVE search one in order to + be supported by the parsing functions, and + ideally provide response to the same kind of + requests (so the CWE search works as well) + + +## v2.4.114 (2019-08-30) + +### Changes + +* [cuckooimport] Handle archives downloaded from both the WebUI and the API. [Pierre-Jean Grenier] + +### Fix + +* Prevent symlink attacks. [Pierre-Jean Grenier] + +* Have I been pwned API changed again. [Raphaël Vinot] + +### Other + +* Merge pull request #327 from zaphodef/cuckooimport. [Alexandre Dulaunoy] + + fix: prevent symlink attacks + +* Merge pull request #326 from zaphodef/cuckooimport. [Alexandre Dulaunoy] + + chg: [cuckooimport] Handle archives downloaded from both the WebUI and the API + + +## v2.4.113 (2019-08-19) + +### New + +* Rewrite cuckooimport. [Pierre-Jean Grenier] + +### Changes + +* Update PyMISP version. [Pierre-Jean Grenier] + +### Fix + +* Avoiding issues when no CWE id is provided. [chrisr3d] + +* Fixed unnecessary dictionary field call. [chrisr3d] + + - No longer necessary to go under 'Event' field + since PyMISP does not contain it since the + latest update + +### Other + +* Merge pull request #322 from zaphodef/cuckooimport. [Alexandre Dulaunoy] + + Rewrite cuckooimport + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Add: Added initial event to reference it from the vulnerability object created out of it. [chrisr3d] + + +## v2.4.112 (2019-08-02) + +### New + +* First version of an advanced CVE parser module. [chrisr3d] + + - Using cve.circl.lu as well as the initial module + - Going deeper into the CVE parsing + - More parsing to come with the CWE, CAPEC and so on + +### Changes + +* [docs] add additional references. [Alexandre Dulaunoy] + +* [travis] revert. [Alexandre Dulaunoy] + +* [travis] github token. [Alexandre Dulaunoy] + +* [travis] mkdocs disabled for the time being. [Alexandre Dulaunoy] + +* [doc] Fix #317 - update the link to the latest version of the training. [Alexandre Dulaunoy] + +* [doc] README updated to the latest version. [Alexandre Dulaunoy] + +* [docs] symbolic link removed. [Alexandre Dulaunoy] + +* [docs] add logos symbolic link. [Alexandre Dulaunoy] + +* Add print to figure out what's going on on travis. [Raphaël Vinot] + +* Bump dependencies. [Raphaël Vinot] + +* Updated the module to work with the updated VirusTotal API. [chrisr3d] + + - Parsing functions updated to support the updated + format of the VirusTotal API responses + - The module can now return objects + - /!\ This module requires a high number of + requests limit rate to work as expected /!\ + +* Adding references between a domain and their siblings. [chrisr3d] + +* Getting domain siblings attributes uuid for further references. [chrisr3d] + +### Fix + +* Using the attack-pattern object template (copy-paste typo) [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +* Fixed cvss-score object relation name. [chrisr3d] + +* Avoid issues when there is no pe field in a windows file sample analysis. [chrisr3d] + + - For instance: doc file + +* Avoid adding file object twice if a KeyError exception comes for some unexpected reasons. [chrisr3d] + +* Testing if file & registry activities fields exist before trying to parse it. [chrisr3d] + +* Testing if there is some screenshot data before trying to fetch it. [chrisr3d] + +* Fixed direction of the relationship between files, PEs and their sections. [chrisr3d] + + - The file object includes a PE, and the PE + includes sections, not the other way round + +* Fixed variable names. [chrisr3d] + +* Wrong change in last commit. [Raphaël Vinot] + +* Skip tests on haveibeenpwned.com if 403. Make pep8 happy. [Raphaël Vinot] + +* Changed the way references added at the end are saved. [chrisr3d] + + - Some references are saved until they are added + at the end, to make it easier when needed + - Here we changed the way they are saved, from a + dictionary with some keys to identify each part + to the actual dictionary with the keys the + function add_reference needs, so we can directly + use this dictionary as is when the references are + added to the different objects + +* Fixed link in documentation. [chrisr3d] + +* Avoiding issues with non existing sample types. [chrisr3d] + +* Undetected urls are represented in lists. [chrisr3d] + +* Changed function name to avoid confusion with the same variable name. [chrisr3d] + +* Quick fix on siblings & url parsing. [chrisr3d] + +* Typo. [chrisr3d] + +* Parsing detected & undetected urls. [chrisr3d] + +* Various fixes about typo, variable names, data types and so on. [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +### Other + +* Merge pull request #319 from 8ear/8ear-add-mkdocs-documentation. [Alexandre Dulaunoy] + + Add `make deploy` to Makefile + +* Added docker and non-docker make commands. [8ear] + +* Add `make deploy` [8ear] + +* Merge pull request #318 from chrisr3d/master. [Christian Studer] + + Updated cve_advanced module to parse CWE and CAPEC data related to the CVE + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Add: Making vulnerability object reference to its related capec & cwe objects. [chrisr3d] + +* Add: Parsing CAPEC information related to the CVE. [chrisr3d] + +* Add: Parsing CWE related to the CVE. [chrisr3d] + +* Merge pull request #316 from 8ear/8ear-add-mkdocs-documentation. [Alexandre Dulaunoy] + + Add web documentation via mkdocs + +* Fix Bugs. [8ear] + +* Fix Fossa in index.md. [8ear] + +* Delete unused file. [8ear] + +* Change mkdocs deploy method. [8ear] + +* Change index.md. [8ear] + +* Merge branch 'master' into 8ear-add-mkdocs-documentation. [Max H] + +* Add: Parsing linux samples and their elf data. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Add: Parsing apk samples and their permissions. [chrisr3d] + +* Add: Added virustotal_public to the list of available modules. [chrisr3d] + +* Add: TODO comment for the next improvement. [chrisr3d] + +* Add: [documentation] Updated README and documentation with the virustotal modules changes. [chrisr3d] + +* Add: Parsing communicating samples returned by domain reports. [chrisr3d] + +* Add: Parsing downloaded samples as well as the referrer ones. [chrisr3d] + +* Add: Object for VirusTotal public API queries. [chrisr3d] + + - Lighter analysis of the report to avoid reaching + the limit of queries per minute while recursing + on the different elements + +* Add: Updated README file with the new module description. [chrisr3d] + +* Change contribute.md. [8ear] + +* Update index.md. [8ear] + +* Add mkdocs as a great web documentation. [8ear] + +* Merge pull request #1 from fossabot/master. [Max H] + + Add license scan report and status + +* Add license scan report and status. [fossabot] + + +## v2.4.110 (2019-07-08) + +### New + +* [doc] Joe Sandbox added in the list. [Alexandre Dulaunoy] + +* Expansion module to query urlhaus API. [chrisr3d] + + - Using the next version of modules, taking a + MISP attribute as input and able to return + attributes and objects + - Work still in process in the core part + +### Changes + +* [documentation] Making URLhaus visible from the github page. [chrisr3d] + + - Because of the white color, the logo was not + visible at all + +* Moved JoeParser class to make it reachable from expansion & import modules. [chrisr3d] + +* [install] REQUIREMENTS file updated. [Alexandre Dulaunoy] + +* [install] Pipfile.lock updated. [Alexandre Dulaunoy] + +* [requirements] Python API wrapper for the Joe Sandbox API added. [Alexandre Dulaunoy] + +* Bump dependencies. [Raphaël Vinot] + +* [pep8] try/except # noqa. [Steve Clement] + + Not sure how to make flake happy on this one. + +* Updated csvimport to support files from csv export + import MISP objects. [chrisr3d] + +### Fix + +* Added missing add_attribute function. [chrisr3d] + +* [documentation] Fixed json file name. [chrisr3d] + +* [documentation] Fixed some description & logo. [chrisr3d] + +* Testing if an object is not empty before adding it the the event. [chrisr3d] + +* Making travis happy. [chrisr3d] + +* Support of the latest version of sigmatools. [chrisr3d] + +* We will display galaxies with tags. [chrisr3d] + +* Returning tags & galaxies with results. [chrisr3d] + + - Tags may exist with the current version of the + parser + - Galaxies are not yet expected from the parser, + nevertheless the principle is we want to return + them as well if ever we have some galaxies from + parsing a JoeSandbox report. Can be removed if + we never galaxies at all + +* Removed duplicate finalize_results function call. [chrisr3d] + +* Making pep8 happy + added joe_import module in the init list. [chrisr3d] + +* Fixed variable name typo. [chrisr3d] + +* Fixed references between domaininfo/ipinfo & their targets. [chrisr3d] + + - Fixed references when no target id is set + - Fixed domaininfo parsing when no ip is defined + +* Some quick fixes. [chrisr3d] + + - Fixed strptime matching because months are + expressed in abbreviated format + - Made data loaded while the parsing function is + called, in case it has to be called multiple + times at some point + +* Making pep8 & travis happy. [chrisr3d] + +* Added references between processes and the files they drop. [chrisr3d] + +* Avoiding network connection object duplicates. [chrisr3d] + +* Avoid creating a signer info object when the pe is not signed. [chrisr3d] + +* Avoiding dictionary indexes issues. [chrisr3d] + + - Using tuples as a dictionary indexes is better + than using generators... + +* Avoiding attribute & reference duplicates. [chrisr3d] + +* Handling case of multiple processes in behavior field. [chrisr3d] + + - Also starting parsing file activities + +* Testing if some fields exist before trying to import them. [chrisr3d] + + - Testing for pe itself, pe versions and pe signature + +* Removed test print. [chrisr3d] + +* Fixed output format to match with the recent changes on modules. [chrisr3d] + +* Making pep8 happy. [chrisr3d] + +* Checking not MISP header fields. [chrisr3d] + + - Rejecting fields not recognizable by MISP + +* Using pymisp classes & methods to parse the module results. [chrisr3d] + +* Clearer user config messages displayed in the import view. [chrisr3d] + +* Removed unused library. [chrisr3d] + +* Make pep8 happy. [chrisr3d] + +* [pep8] More fixes. [Steve Clement] + +* [pep8] More pep8 happiness. [Steve Clement] + +* [pep8] Fixes. [Steve Clement] + +* Fixed standard MISP csv format header. [root] + + - The csv header we can find in data produced from + MISP restSearch csv format is the one to use to + recognize a csv file produced by MISP + +* Fixed introspection fields for csvimport & goamlimport. [root] + + - Added format field for goaml so the module is + known as returning MISP attributes & objects + - Fixed introspection to make the format, user + config and input source fields visible from + MISP (format also added at the same time) + +* Fixed libraries import that changed with the latest merge. [root] + +* Fixed fields parsing to support files from csv export with additional context. [chrisr3d] + +* Handling the case of Context included in the csv file exported from MISP. [chrisr3d] + +* Fixed changes omissions in handler function. [chrisr3d] + +* Fixed object_id variable name typo. [root] + +* Making json_decode even happier with full json format. [chrisr3d] + + - Using MISPEvent because it is cleaner & easier + - Also cleaner implementation globally + +* Using to_dict on attributes & objects instead of to_json to make json_decode happy in the core part. [chrisr3d] + +### Other + +* Add: [documentation] Added some missing documentation for the most recently added modules. [chrisr3d] + +* Add: [documentation] Added documentation for Joe Sandbox & URLhaus. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #309 from Kortho/patch-2. [Steve Clement] + + changed service pointer + +* Changed service pointer. [Kortho] + + Changed so the service starts the modules in the venv where they are installed + +* Merge pull request #308 from Kortho/patch-1. [Steve Clement] + + Fixed missing dependencies for RHEL install + +* Fixed missing dependencies for RHEL install. [Kortho] + + Added dependencies needed for installing the python library pdftotext + +* Add: Added screenshot of the behavior of the analyzed sample. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #307 from ninoseki/fix-missing-links. [Alexandre Dulaunoy] + + Fix missing links in README.md + +* Fix missing links in README.md. [Manabu Niseki] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge pull request #306 from MISP/new_module. [Alexandre Dulaunoy] + + New modules able to return MISP objects + +* Add: Added new modules to the list. [chrisr3d] + +* Merge branch 'new_module' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge pull request #305 from joesecurity/new_module. [Alexandre Dulaunoy] + + joesandbox_query.py: improve behavior in unexpected circumstances + +* Joesandbox_query.py: improve behavior in unexpected circumstances. [Georg Schölly] + +* Add: New expansion module to query Joe Sandbox API with a report link. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'joesecurity-joesandbox_submit' [Alexandre Dulaunoy] + +* Merge branch 'joesandbox_submit' of https://github.com/joesecurity/misp-modules into joesecurity-joesandbox_submit. [Alexandre Dulaunoy] + +* Add expansion for joe sandbox. [Georg Schölly] + +* Merge pull request #304 from joesecurity/new_module. [Alexandre Dulaunoy] + + add support for url analyses + +* Support url analyses. [Georg Schölly] + +* Improve forwards-compatibility. [Georg Schölly] + +* Add: Parsing MITRE ATT&CK tactic matrix related to the Joe report. [chrisr3d] + +* Add: Parsing domains, urls & ips contacted by processes. [chrisr3d] + +* Add: Starting parsing dropped files. [chrisr3d] + +* Add: Starting parsing network behavior fields. [chrisr3d] + +* Add: Parsing registry activities under processes. [chrisr3d] + +* Add: Parsing processes called by the file analyzed in the joe sandbox report. [chrisr3d] + +* Add: Parsing some object references at the end of the process. [chrisr3d] + +* Add: [new_module] Module to import data from Joe sandbox reports. [chrisr3d] + + - Parsing file, pe and pe-section objects from the + report file info field + - Deeper file info parsing to come + - Other fields parsing to come as well + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge pull request #300 from cudeso/master. [Alexandre Dulaunoy] + + Bugfix for "sources" ; do not include as IDS for "access" registry keys + +* Bugfix for "sources" ; do not include as IDS for "access" registry keys. [Koen Van Impe] + + - Bugfix to query "operations" in files, mutex, registry + - Do not set IDS flag for registry 'access' operations + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* New VMRay modules (#299) [Steve Clement] + + New VMRay modules + +* New VMRay modules. [Koen Van Impe] + + New JSON output format of VMRay + Prepare for automation (via PyMISP) with workflow taxonomy tags + +* Merge pull request #1 from MISP/master. [Koen Van Impe] + + Sync + +* Add: Added urlhaus in the expansion modules init list. [root] + +* Merge branch 'new_module' of https://github.com/MISP/misp-modules into new_module. [root] + +* Merge branch 'features_csvimport' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into features_csvimport. [chrisr3d] + +* Merge branch 'features_csvimport' of github.com:MISP/misp-modules into features_csvimport. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into features_csvimport. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into features_csvimport. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'new_module' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge branch 'master' of https://github.com/MISP/misp-modules into new_module. [root] + +* Merge branch 'master' of https://github.com/MISP/misp-modules into new_module. [root] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + + +## v2.4.106 (2019-04-27) + +### New + +* Devel mode. [Raphaël Vinot] + + Fix #293 + +* Modules for greynoise, haveibeenpwned and macvendors. [Raphaël Vinot] + +* Add missing dependency (backscatter) [Raphaël Vinot] + +* Add systemd launcher. [Raphaël Vinot] + +* Intel471 module. [Raphaël Vinot] + +* [btc] Very simple BTC expansion chg: [req] yara-python is preferred. [Steve Clement] + +* First version of a yara rule creation expansion module. [chrisr3d] + +* Documentation concerning modules explained in markdown file. [chrisr3d] + +* Expansion hover module to check spamhaus DBL for a domain name. [chrisr3d] + +### Changes + +* [doc] install of deps updated. [Alexandre Dulaunoy] + +* Bump REQUIREMENTS. [Raphaël Vinot] + +* Bump dependencies. [Raphaël Vinot] + +* [doc] new MISP expansion modules added for PDF, OCR, DOCX, XLSX, PPTX , ODS and ODT. [Alexandre Dulaunoy] + +* [init] cleanup for pep. [Alexandre Dulaunoy] + +* [pdf-enrich] updated. [Alexandre Dulaunoy] + +* [Pipfile] collection removed. [Alexandre Dulaunoy] + +* Bump dependencies. [Raphaël Vinot] + +* [doc] Added new dependencies and updated RHEL/CentOS howto. (#295) [Steve Clement] + + chg: [doc] Added new dependencies and updated RHEL/CentOS howto. + +* [doc] Added new dependencies and updated RHEL/CentOS howto. [Steve Clement] + +* [init] removed trailing whitespace. [Alexandre Dulaunoy] + +* [ocr] re module not used - removed. [Alexandre Dulaunoy] + +* Bump dependencies, update REQUIREMENTS file. [Raphaël Vinot] + +* [doc] cuckoo_submit module added. [Alexandre Dulaunoy] + +* Require python3 instead of python 3.6. [Raphaël Vinot] + +* [travis] because we all need sudo. [Alexandre Dulaunoy] + +* [travis] because everyone need a bar. [Alexandre Dulaunoy] + +* [doc] qrcode and Cisco FireSight added. [Alexandre Dulaunoy] + +* [qrcode] add requirements. [Alexandre Dulaunoy] + +* [qrcode] added to the __init__ [Alexandre Dulaunoy] + +* [qrcode] flake8 needs some drugs. [Alexandre Dulaunoy] + +* [qrcode] various fixes to make it PEP compliant. [Alexandre Dulaunoy] + +* Bump dependencies. [Raphaël Vinot] + + Fix CVE-2019-11324 (urllib3) + +* Bump Dependencies. [Raphaël Vinot] + +* [doc] Updated README to reflect current virtualenv efforts. TODO: pipenv. [Steve Clement] + +* [doc] new modules added. [Alexandre Dulaunoy] + +* Bump dependencies. [Raphaël Vinot] + +* Bump dependencies. [Raphaël Vinot] + +* Bump Requirements. [Raphaël Vinot] + +* [doc] asciidoctor requirement removed (new PDF module use reportlab) [Alexandre Dulaunoy] + +* Bump dependencies, add update script. [Raphaël Vinot] + +* [doc] PDF export. [Alexandre Dulaunoy] + +* [pdfexport] make flake8 happy. [Alexandre Dulaunoy] + +* [pipenv] fix the temporary issue that python-yara is not officially released. [Alexandre Dulaunoy] + +* [requirements] reportlab added. [Alexandre Dulaunoy] + +* [pipenv] Pipfile.lock updated. [Alexandre Dulaunoy] + +* [requirements] updated. [Alexandre Dulaunoy] + +* [PyMISP] dep updated to the latest version. [Alexandre Dulaunoy] + +* PyMISP requirement. [Alexandre Dulaunoy] + +* [pypi] Made sure url-normalize installs less stric. [Steve Clement] + +* [btc_scam_check] fix spacing for making flake 8 happy. [Alexandre Dulaunoy] + +* [backscatter.io] blind fix regarding undefined value. [Alexandre Dulaunoy] + +* [doc] backscatter.io updated. [Alexandre Dulaunoy] + +* [doc] backscatter.io documentation added. [Alexandre Dulaunoy] + +* [backscatter.io] remove blank line at the end of the file. [Alexandre Dulaunoy] + +* [backscatter.io] Exception handler fixed for recent version of Python. [Alexandre Dulaunoy] + +* Bump dependencies. [Raphaël Vinot] + +* Use pipenv, update bgpranking/ipasn modules. [Raphaël Vinot] + +* [doc] Nexthink module added. [Alexandre Dulaunoy] + +* [doc] osquery export module added. [Alexandre Dulaunoy] + +* [doc] Nexthink export format added. [Alexandre Dulaunoy] + +* [doc] cannot type today. [Alexandre Dulaunoy] + +* [intel471] module added. [Alexandre Dulaunoy] + +* Regenerated documentation markdown file. [chrisr3d] + +* [onyphe] fix #252. [Alexandre Dulaunoy] + +* [btc] Removed simple PoC for btc expansion. [Steve Clement] + +* [doc] btc module added. [Alexandre Dulaunoy] + +* [doc] generated documentation updated. [Alexandre Dulaunoy] + +* [doc] btc module added to documentation. [Alexandre Dulaunoy] + +* [tools] Added psutil as a dependency to detect misp-modules PID. [Steve Clement] + +* [init] Added try/catch in case misp-modules is already running on a port, or port is in use... [Steve Clement] + +* Validating yara rules after their creation. [chrisr3d] + +* [documentation] osquery logo added. [Alexandre Dulaunoy] + +* [documentation] generated. [Alexandre Dulaunoy] + +* [docs] Added some missing dependencies and instructions for virtualenv deployment. [Steve Clement] + +* [doc] documentation generator updated to include links to source code. [Alexandre Dulaunoy] + +* Changed documentation markdown file name. [chrisr3d] + +* Structurded data. [chrisr3d] + +* Modified the mapping dictionary to support misp-objects updates. [chrisr3d] + +* Modified output format. [chrisr3d] + +* Add new dependency (oauth2) [Raphaël Vinot] + +* Dnspython3 has been superseded by the regular dnspython kit. [Raphaël Vinot] + +* Wikidata module added. [Alexandre Dulaunoy] + +* SPARQLWrapper added (for wikidata module) [Alexandre Dulaunoy] + +### Fix + +* Re-enable python 3.6 support. [Raphaël Vinot] + +* CTRL+C is working again. [Raphaël Vinot] + + Fix #292 + +* Make flake8 happy. [Raphaël Vinot] + +* [doc] Small typo fix. [Steve Clement] + +* Pep8 foobar. [Raphaël Vinot] + +* Add the new module sin the list of modules availables. [Raphaël Vinot] + +* Typos in variable names. [Raphaël Vinot] + +* Remove unused import. [Raphaël Vinot] + +* Tornado expects a KILL now. [Raphaël Vinot] + +* [exportpdf] update documentation. [Falconieri] + +* [exportpdf] custom path parameter. [Falconieri] + +* [exportpdf] add parameters. [Falconieri] + +* [exportpdf] mising whitespace. [Falconieri] + +* [exportpdf] problem on one line. [Falconieri] + +* [exportpdf] add configmodule parameter for galaxy. [Falconieri] + +* [reportlab] Textual description parameter. [Falconieri] + +* [pdfexport] Bugfix on PyMisp exportpdf call. [Falconieri] + +* Systemd service. [Raphaël Vinot] + +* Regenerated documentation. [chrisr3d] + +* Description fixed. [chrisr3d] + +* Pep8 related fixes. [Raphaël Vinot] + +* Make flake8 happy. [Raphaël Vinot] + +* Change in the imports in other sigma module. [Raphaël Vinot] + +* Change in the imports. [Raphaël Vinot] + +* Change module name. [Raphaël Vinot] + +* Allow redis details to be retrieved from environment variables. [Ruiwen Chua] + +* Remove tests on python 3.5. [Raphaël Vinot] + +* Make pep8 happy. [Raphaël Vinot] + +* Removed not valid input type. [chrisr3d] + +* Cleaned up not used variables. [chrisr3d] + +* Updated rbl module result format. [chrisr3d] + + - More readable as str than dumped json + +* Added Macaddress.io module in the init list. [chrisr3d] + +* Typo on input type. [chrisr3d] + +* Fixed type of the result in case of exception. [chrisr3d] + + - Set as str since some exception types are not + jsonable + +* Added hostname attribute support as it is intended. [chrisr3d] + +* Threatanalyzer_import - bugfix for TA6.1 behavior. [Christophe Vandeplas] + +* Displaying documentation items of each module by alphabetic order. [chrisr3d] + + - Also regenerated updated documentation markdown + +* Updated yara import error message. [chrisr3d] + + - Better to 'pip install -I -r REQUIREMENTS' to + have the correct yara-python version working + for all the modules, than having another one + failing with yara hash & pe modules + +* Specifying a yara-python version that works for hash & pe yara modules. [chrisr3d] + +* Making yara query an expansion module for single attributes atm. [chrisr3d] + +* Catching errors while parsing additional info in requests. [chrisr3d] + +* Reduced logos size. [chrisr3d] + +* Typo for separator between each explained module. [chrisr3d] + +* Making python 3.5 happy with the exception type ImportError. [chrisr3d] + +* Fixed exception type for python 3.5. [chrisr3d] + +* Fixed exception type. [chrisr3d] + +* Fixed syntax error. [chrisr3d] + +* Fixed indentation error. [chrisr3d] + +* Fixed 1 variable misuse + cleaned up variable names. [chrisr3d] + + - Fixed use of 'domain' variable instead of 'email' + - Cleaned up variable names to avoid redefinition + of built-in variables + +* Avoiding adding attributes that are already in the event. [chrisr3d] + +* Fixed quick variable issue. [chrisr3d] + +* Cleaned up test function not used anymore. [chrisr3d] + +* Multiple attributes parsing support. [chrisr3d] + + - Fixing one of my previous changes not processing + multiple attributes parsing + +* Removed print. [chrisr3d] + +* Some cleanup and output types fixed. [chrisr3d] + + - hashes types specified in output + +* Quick cleanup. [chrisr3d] + +* Quick cleanup. [chrisr3d] + +* Ta_import - bugfixes. [Christophe Vandeplas] + +* [cleanup] Quick clean up on exception type. [chrisr3d] + +* [cleanup] Quick clean up on yaml load function. [chrisr3d] + +* [cleanup] Quick clean up on exception type. [chrisr3d] + +* Put the report location parsing in a try/catch statement as it is an optional field. [chrisr3d] + +* Put the stix2-pattern library import in a try statement. [chrisr3d] + + --> Error more easily caught + +* Removed STIX related libraries, files, documentation, etc. [chrisr3d] + +* Avoid trying to build attributes with not intended fields. [chrisr3d] + + - Previously: if the header field is not an attribute type, then + it was added as an attribute field. + PyMISP then used to skip it if needed + + - Now: Those fields are discarded before they are put in an attribute + +* Using userConfig to define the header instead of moduleconfig. [chrisr3d] + +* Fixed input & output of the module. [chrisr3d] + +* Added an object checking. [Christian Studer] + + - Checking if there are objects in the event, and then if there is at least 1 transaction object + - This prevents the module from crashing, but does not guaranty having a valid GoAML file (depending on objects and their relations) + +* Fixed input & output of the module. [chrisr3d] + + Also updated some functions + +* Fixed typo of the aml type for country codes. [chrisr3d] + +* Typo in references mapping dictionary. [chrisr3d] + +* Added an object checking. [chrisr3d] + + - Checking if there are objects in the event, and then + if there is at least 1 transaction object + - This prevents the module from crashing, but does not + guaranty having a valid GoAML file (depending on + objects and their relations) + +* Added the moduleinfo field need to have MISP event in standard format. [chrisr3d] + +* Missing cve module test. [Alexandre Dulaunoy] + +* Goamlexport added. [Alexandre Dulaunoy] + +* Python version in Travis. [Alexandre Dulaunoy] + +* Solved reading problems for some files. [chrisr3d] + +* Skipping empty lines. [chrisr3d] + +* Make travis happy. [Raphaël Vinot] + +* OpenIOC importer. [Raphaël Vinot] + +* #137 when a CVE is not found, a return message is given. [Alexandre Dulaunoy] + +* Use the proper formatting method and not the horrible % one. [Hannah Ward] + +* Misp-modules are by default installed in /bin. [Alexandre Dulaunoy] + +* Module_config should be set as introspection relies on it. [Alexandre Dulaunoy] + +* Types array. [Alexandre Dulaunoy] + +* Run the server as "python3 misp-modules" [Raphaël Vinot] + +* Stupid off-by-n line... [Alexandre Dulaunoy] + +### Other + +* Merge branch 'master' of github.com:MISP/misp-modules. [Alexandre Dulaunoy] + +* Removed trailing whitespaces. [Sascha Rommelfangen] + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Sascha Rommelfangen] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Raphaël Vinot] + +* New modules added. [Sascha Rommelfangen] + +* New requirements for new modules. [Sascha Rommelfangen] + +* Introduction of new modules. [Sascha Rommelfangen] + +* Merge remote-tracking branch 'upstream/master' [Steve Clement] + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Sascha Rommelfangen] + +* Renamed file. [Sascha Rommelfangen] + +* Renamed module. [Sascha Rommelfangen] + +* Initial version of OCR expansion module. [Sascha Rommelfangen] + +* Merge pull request #291 from Evert0x/submitcuckoo. [Alexandre Dulaunoy] + + Expansion module - File/URL submission to Cuckoo Sandbox + +* Generate latest version of documentation. [Ricardo van Zutphen] + +* Document Cuckoo expansion module. [Ricardo van Zutphen] + +* Use double quotes and provide headers correctly. [Ricardo van Zutphen] + +* Update Cuckoo module to support files and URLs. [Ricardo van Zutphen] + +* Update __init__.py. [Evert0x] + +* Create cuckoo_submit.py. [Evert0x] + +* Brackets are difficult... [Sascha Rommelfangen] + +* Merge branch 'qr-code-module' of https://github.com/rommelfs/misp-modules into rommelfs-qr-code-module. [Alexandre Dulaunoy] + +* Initial version of QR code reader. [Sascha Rommelfangen] + + Module accepts attachments and processes pictures. It tries to identify and analyze an existing QR code. + Identified values can be inserted into the event. + +* Merge branch 'iceone23-patch-1' [Raphaël Vinot] + +* Create cisco_firesight_manager_ACL_rule_export.py. [iceone23] + + Cisco Firesight Manager ACL Rule Export module + +* Merge pull request #289 from SteveClement/master. [Steve Clement] + + fix: [doc] Small typo fix + +* Merge remote-tracking branch 'upstream/master' [Steve Clement] + +* Merge pull request #285 from wesinator/patch-1. [Alexandre Dulaunoy] + + Fix command highlighting + +* Fix command highlighting. [Ԝеѕ] + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Sascha Rommelfangen] + +* Merge pull request #284 from Vincent-CIRCL/master. [Alexandre Dulaunoy] + + fix: [exportpdf] custom path parameter + +* Merge pull request #283 from Vincent-CIRCL/master. [Alexandre Dulaunoy] + + fix: [exportpdf] add parameters + +* Merge pull request #281 from Vincent-CIRCL/master. [Alexandre Dulaunoy] + + fix: [exportpdf] add configmodule parameter for galaxy + +* Merge pull request #282 from cgi1/patch-1. [Alexandre Dulaunoy] + + Adding virtualenv to apt-get install + +* Adding virtualenv to apt-get install. [cgi1] + +* Merge pull request #279 from Vincent-CIRCL/master. [Alexandre Dulaunoy] + + fix: [reportlab] Textual description parameter + +* Chr: Restart the modules after update. [Raphaël Vinot] + +* Fixed a bug when checking malformed BTC addresses. [Sascha Rommelfangen] + +* Merge remote-tracking branch 'upstream/master' [Steve Clement] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Alexandre Dulaunoy] + +* Merge pull request #278 from Vincent-CIRCL/master. [Alexandre Dulaunoy] + + chg: [pdfexport] Fix pdf export, by calling new PyMISP tool for Misp Event export + +* Fix [exportpdf] update parameters for links generation. [Falconieri] + +* Tidy: Remove old dead export code. [Falconieri] + +* Test 1 - PDF call. [Falconieri] + +* Print values. [Vincent-CIRCL] + +* Test update. [Vincent-CIRCL] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Alexandre Dulaunoy] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Alexandre Dulaunoy] + +* Merge pull request #276 from iwitz/patch-1. [Alexandre Dulaunoy] + + Add RHEL installation instructions + +* Add: rhel installation instructions. [iwitz] + +* Add: [doc] Added backscatter.io logo + regenerated documentation. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into new_module. [chrisr3d] + +* Merge pull request #274 from 9b/master. [Alexandre Dulaunoy] + + Backscatter.io expansion module + +* Use the write var on return. [9b] + +* Stubbed module. [9b] + +* Add: New module to check if a bitcoin address has been abused. [chrisr3d] + + - Also related update of documentation + +* Sometimes server doesn't return expected values. fixed. [Sascha Rommelfangen] + +* Merge pull request #266 from MISP/pipenv. [Raphaël Vinot] + + chg: Use pipenv, update bgpranking/ipasn modules, fix imports for sigma + +* Merge pull request #259 from ruiwen/fix_redis. [Alexandre Dulaunoy] + + fix: allow redis details to be retrieved from environment variables + +* Add: [doc] link documentation to README. [Alexandre Dulaunoy] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Alexandre Dulaunoy] + +* Merge pull request #258 from HacknowledgeCH/export_nexthink. [Alexandre Dulaunoy] + + Export nexthink + +* Added 2 blank lines to comply w/ pep8. [milkmix] + +* Removed unused re module. [milkmix] + +* Added documentation. [milkmix] + +* Added domain attributes support. [milkmix] + +* Support for md5 and sha1 hashes. [milkmix] + +* First export feature: sha1 attributes nxql query. [milkmix] + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Sascha Rommelfangen] + +* Add: Added missing expansion modules in readme. [chrisr3d] + +* Add: Completed documentation for expansion modules. [chrisr3d] + +* Add: Updated more expansion documentation files. [chrisr3d] + +* Add: Added new documentation for hashdd module. [chrisr3d] + +* Add: Update to support sha1 & sha256 attributes. [chrisr3d] + +* Add: More documentation on expansion modules. [chrisr3d] + +* Add: Started filling some expansion modules documentation. [chrisr3d] + +* Add: Added yara_query module documentation, update yara_syntax_validator documentation & generated updated documentation markdown. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into chrisr3d_patch. [chrisr3d] + +* Add: Added test files for yara to test yara library & potentially yara syntax. [chrisr3d] + +* Add: Added imphash to input attribute types. [chrisr3d] + +* Cosmetic output change. [Sascha Rommelfangen] + +* Debug removed. [Sascha Rommelfangen] + +* API changes reflected. [Sascha Rommelfangen] + +* Merge pull request #253 from MISP/chrisr3d_patch. [Alexandre Dulaunoy] + + Validation of yara rules + +* Merge branch 'master' of github.com:MISP/misp-modules into chrisr3d_patch. [chrisr3d] + +* Merge pull request #251 from MISP/rommelfs-patch-4. [Raphaël Vinot] + + bug fix regarding leftovers between runs + +* Bug fix regarding leftovers between runs. [Sascha Rommelfangen] + +* Merge pull request #250 from SteveClement/btc. [Steve Clement] + + chg: [btc] Removed simple PoC for btc expansion. + +* Merge pull request #249 from MISP/rommelfs-patch-3. [Steve Clement] + + added btc_steroids + +* Added btc_steroids. [Sascha Rommelfangen] + +* Merge pull request #248 from rommelfs/master. [Sascha Rommelfangen] + + Pull request for master + +* Added btc_steroids to the list. [Sascha Rommelfangen] + +* Initial version of a Bitcoin module. [Sascha Rommelfangen] + +* Merge pull request #247 from SteveClement/btc. [Alexandre Dulaunoy] + + new: [module] Added very simple BitCoin expansion/hover module + +* Merge pull request #245 from chrisr3d/master. [Alexandre Dulaunoy] + + YARA rules from hashes expansion module + +* Updated list of modules in readme. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Add: [documentation] osquery logo. [Alexandre Dulaunoy] + +* Merge pull request #241 from 0xmilkmix/doc_osqueryexport. [Alexandre Dulaunoy] + + Added basic documentation for OS query + +* Merge branch 'master' into doc_osqueryexport. [Alexandre Dulaunoy] + +* Merge pull request #240 from 0xmilkmix/support_osquery_win_named_obj. [Alexandre Dulaunoy] + + super simple support for mutexes through winbaseobj in osquery 3.3 + +* Merge branch 'master' into support_osquery_win_named_obj. [Alexandre Dulaunoy] + +* Merge pull request #242 from 0xmilkmix/module_writting. [Steve Clement] + + chg: [doc] Additional documentation for export module + +* Documentation for export module. [milkmix] + +* Super simple support for mutexes through winbaseobj in osquery 3.3. [milkmix] + +* Added basic documentation. [milkmix] + +* Merge pull request #239 from SteveClement/master. [Steve Clement] + + chg: [docs] Added some missing dependencies and instructions for virtualenv deployment + +* Merge pull request #237 from 0xmilkmix/export_osquery. [Alexandre Dulaunoy] + + Export osquery + +* Merge branch 'master' into export_osquery. [Julien Bachmann] + +* Merge pull request #232 from CodeLineFi/master. [Alexandre Dulaunoy] + + macaddres.io module - Date conversion bug fixed + +* Merge branch 'master' into master. [Alexandre Dulaunoy] + +* Merge pull request #233 from chrisr3d/documentation. [Christian Studer] + + Modules documentation + +* Merge branch 'master' of github.com:MISP/misp-modules into documentation. [chrisr3d] + +* Updated documentation result file. [chrisr3d] + +* Add: Added documentation for expansion modules. [chrisr3d] + +* Add: Started adding logos on documentation for each module. [chrisr3d] + +* Renamed directory to have consistency in names. [chrisr3d] + +* Removed documentation about a module deleted from the repository. [chrisr3d] + +* Merging readme. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into documentation. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into documentation. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into documentation. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into documentation. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into documentation. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into documentation. [chrisr3d] + +* First try of documentation for import & export modules. [chrisr3d] + + - Providing information about the general purpose of + the modules, their requirements, how to use them + (if there are special features), some references + about the format concerned or the vendors, and their + input and output. + - Documentation to be completed by additional fields + of documentation and / or more detailed descriptions + +* Added Documentation explanations on readme file. [chrisr3d] + +* CSV import documentation first try. [chrisr3d] + +* GoAML modules documentation first try. [chrisr3d] + +* Updated README. Added a link to the integration tutorial. [Codelinefi-admin] + +* Fixed a bug with wrong dates conversion. [Codelinefi-admin] + +* Merge branch 'vulnersCom-master' [Alexandre Dulaunoy] + +* Merge branch 'master' of https://github.com/vulnersCom/misp-modules into vulnersCom-master. [Alexandre Dulaunoy] + +* Fixed getting of the Vulners AI score. [isox] + +* Merge pull request #230 from lctrcl/master. [Alexandre Dulaunoy] + +* Merge branch 'master' into master. [lctrcl] + +* Merge pull request #229 from lctrcl/master. [Alexandre Dulaunoy] + + New vulners module added + +* HotFix: Vulners AI score. [Igor Ivanov] + +* Code cleanup and formatting. [Igor Ivanov] + +* Added exploit information. [Igor Ivanov] + +* Initial Vulners module PoC. [Igor Ivanov] + +* Merge pull request #226 from CodeLineFi/master. [Alexandre Dulaunoy] + + New macaddress.io hover module added + +* Macaddress.io hover module added. [Codelinefi-admin] + +* Merge pull request #223 from chrisr3d/master. [Christian Studer] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #222 from chrisr3d/master. [Christian Studer] + + Clean up + fix of some modules + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #221 from MISP/rommelfs-patch-2. [Alexandre Dulaunoy] + + fixed typo + +* Fixed typo. [Sascha Rommelfangen] + + via #220 + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #218 from surbo/patch-1. [Alexandre Dulaunoy] + + Update urlscan.py + +* Update urlscan.py. [SuRb0] + + Added hash to the search so you can take advantage of the new file down load function on urlscan.io. You can use this to pivot on file hashes and find out domains that hosting the same malicious file. + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #217 from threatsmyth/master. [Alexandre Dulaunoy] + + Add error handling for DNS failures, reduce imports, and simplify attribute comments + +* Merge branch 'master' into master. [David J] + +* Merge pull request #215 from threatsmyth/master. [Alexandre Dulaunoy] + + Create urlscan.py + +* Add error handling for DNS failures, reduce imports, and simplify misp_comments. [David J] + +* Create urlscan.py. [David J] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #214 from chrisr3d/chrisr3d_patch. [Alexandre Dulaunoy] + + New module to check DBL Spamhaus + +* Merge branch 'chrisr3d_patch' of github.com:chrisr3d/misp-modules. [chrisr3d] + +* Add: Added DBL spamhaus module documentation and in expansion init file. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Ta_import - bugfixes for TA 6.1. [Christophe Vandeplas] + +* Merge pull request #210 from chrisr3d/master. [Christian Studer] + + Put the report location parsing in a try/catch statement as it is an optional field + +* Merge pull request #209 from cvandeplas/master. [Christophe Vandeplas] + + ta_import - support for TheatAnalyzer 6.1 + +* Ta_import - support for TheatAnalyzer 6.1. [Christophe Vandeplas] + +* Securitytrails.com expansion module added. [Alexandre Dulaunoy] + +* Merge pull request #208 from sebdraven/dnstrails. [Alexandre Dulaunoy] + + module securitytrails + +* Merge branch 'master' into dnstrails. [sebdraven] + +* Merge pull request #206 from chrisr3d/master. [Alexandre Dulaunoy] + + Expansion module displaying SIEM signatures from a sigma rule + +* Merge branch 'master' into master. [Alexandre Dulaunoy] + +* Remove the never release Python code in Travis. [Alexandre Dulaunoy] + +* Remove Python 3.4 and Python 3.7 added. [Alexandre Dulaunoy] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #202 from SteveClement/master. [Alexandre Dulaunoy] + + Removed test modules from view + +* - Removed test modules from view - Moved skeleton expansion module to it's proper place. [Steve Clement] + +* Merge pull request #201 from chrisr3d/master. [Alexandre Dulaunoy] + + add: STIX2 pattern syntax validator + +* Add: Experimental expansion module to display the SIEM signatures from a sigma rule. [chrisr3d] + +* Add: stix2 pattern validator requirements. [chrisr3d] + +* Add: STIX2 pattern syntax validator. [chrisr3d] + +* Merge pull request #199 from SteveClement/master. [Alexandre Dulaunoy] + + Added (Multipage) PDF support to OCR Module, minor refactor + +* - Reverted to <3.6 compatibility. [Steve Clement] + +* - Fixed log output. [Steve Clement] + +* - Forgot to import sys. [Steve Clement] + +* - Added logger functionality for debug sessions. [Steve Clement] + +* - content was already a wand.obj. [Steve Clement] + +* Merge remote-tracking branch 'upstream/master' [Steve Clement] + +* Threatanalyzer_import - order of category tuned. [Christophe Vandeplas] + +* Merge branch 'master' of github.com:SteveClement/misp-modules. [Steve Clement] + +* Merge branch 'master' into master. [Alexandre Dulaunoy] + +* - Some more comments - Removed libmagic, wand can handle it better. [Steve Clement] + +* - Set tornado timeout to 300 seconds. [Steve Clement] + +* - Quick comment ToDo: Avoid using Magic in future releases. [Steve Clement] + +* - added wand requirement - fixed missing return png byte-stream - move module import to handler to catch and report errorz. [Steve Clement] + +* - fixed typo move image back in scope. [Steve Clement] + +* - Added initial PDF support, nothing is processed yet - Test to replace PIL with wand. [Steve Clement] + +* Change type of status. [Sebdraven] + +* Remove print. [Sebdraven] + +* Last commit for release. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add searching_stats. [Sebdraven] + +* Add searching_stats. [Sebdraven] + +* Correct key. [Sebdraven] + +* Correct key. [Sebdraven] + +* Correct param. [Sebdraven] + +* Add searching domains. [Sebdraven] + +* Add searching domains. [Sebdraven] + +* Add return. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add whois expand to test. [Sebdraven] + +* Add whois expand to test. [Sebdraven] + +* Correct index error. [Sebdraven] + +* Error call functions. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add status_ok to true. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add logs. [Sebdraven] + +* Correct out of bound returns. [Sebdraven] + +* Correct key and return of functions. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add logs. [Sebdraven] + +* Correct typo. [Sebdraven] + +* Test whois history. [Sebdraven] + +* History whois dns. [Sebdraven] + +* Correct typo. [Sebdraven] + +* Rename misp modules. [Sebdraven] + +* Add a test to check if the list is not empty. [Sebdraven] + +* Add a test to check if the list is not empty. [Sebdraven] + +* Add logs. [Sebdraven] + +* Debug whois. [Sebdraven] + +* Debug ipv4 or ipv6. [Sebdraven] + +* Add debug. [Sebdraven] + +* Debug. [Sebdraven] + +* Change status. [Sebdraven] + +* Change history dns. [Sebdraven] + +* Add logs to debug. [Sebdraven] + +* Correct call function. [Sebdraven] + +* Add history mx and soa. [Sebdraven] + +* Add history dns and handler exception. [Sebdraven] + +* Add history dns. [Sebdraven] + +* Switch type ip. [Sebdraven] + +* Refactoring expand_whois. [Sebdraven] + +* Correct typo. [Sebdraven] + +* Add ipv6 and ipv4. [Sebdraven] + +* Change type. [Sebdraven] + +* Change type. [Sebdraven] + +* Change loop. [Sebdraven] + +* Add time sleep in each request. [Sebdraven] + +* Control return of records. [Sebdraven] + +* Add history ipv4. [Sebdraven] + +* Add logs. [Sebdraven] + +* Change categories. [Sebdraven] + +* Concat results. [Sebdraven] + +* Change name keys. [Sebdraven] + +* Change return value. [Sebdraven] + +* Add logs. [Sebdraven] + +* Change errors. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add expand whois. [Sebdraven] + +* Typo. [Sebdraven] + +* Add categories and comments. [Sebdraven] + +* Add expand subdomains. [Sebdraven] + +* Add expand subdomains. [Sebdraven] + +* Change categories. [Sebdraven] + +* Changes keys. [Sebdraven] + +* Add status ! [Sebdraven] + +* Add methods. [Sebdraven] + +* Add expand domains. [Sebdraven] + +* Add link pydnstrain in requirements. [Sebdraven] + +* Add new module dnstrails. [Sebdraven] + +* Merge pull request #198 from chrisr3d/master. [Alexandre Dulaunoy] + + Sigma syntax validator expansion module + some updates + +* Updated README to add sigma & some other missing modules. [chrisr3d] + +* Updated the list of modules (removed stiximport) [chrisr3d] + +* Add: Sigma syntax validator expansion module. [chrisr3d] + + --> Checks sigma rules syntax + - Updated the expansion modules list as well + - Updated the requirements list + +* Updated the list of expansion modules. [chrisr3d] + +* Corrected typos and unused imports. [milkmix] + +* Added support for scheduledtasks. [milkmix] + +* Added support for service-displayname, regkey|value. [milkmix] + +* Initial implementation supporting regkey. mutexes support waiting osquery table. [milkmix] + +* Merge pull request #197 from sebdraven/onyphe_full_module. [Alexandre Dulaunoy] + + Onyphe full module + +* Add return handle domains. [Sebdraven] + +* Add search. [Sebdraven] + +* Add domain to expand. [Sebdraven] + +* Correct bugs. [Sebdraven] + +* Add domain expansion. [Sebdraven] + +* Add comment. [Sebdraven] + +* Correct bugs. [Sebdraven] + +* Correct comments. [Sebdraven] + +* Add threat list expansion. [Sebdraven] + +* Change method to concat methods. [Sebdraven] + +* Set status after requests. [Sebdraven] + +* Set status after requests. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add logs. [Sebdraven] + +* Add logs. [Sebdraven] + +* Pep 8. [Sebdraven] + +* Correct bug. [Sebdraven] + +* Add datascan expansion. [Sebdraven] + +* Add reverse infos. [Sebdraven] + +* Add reverse infos. [Sebdraven] + +* Add reverse infos. [Sebdraven] + +* Add reverse infos. [Sebdraven] + +* Add forward infos. [Sebdraven] + +* Add comment of attributes. [Sebdraven] + +* Add comment of attributes. [Sebdraven] + +* Error loops. [Sebdraven] + +* Error method. [Sebdraven] + +* Error type. [Sebdraven] + +* Error keys. [Sebdraven] + +* Add expansion synscan. [Sebdraven] + +* Change key access domains. [Sebdraven] + +* Change add in results. [Sebdraven] + +* Add logs. [Sebdraven] + +* Correct error keys. [Sebdraven] + +* Test patries expansion. [Sebdraven] + +* Add onyphe full module. [Sebdraven] + +* Add onyphe full module and code the stub. [Sebdraven] + +* Merge pull request #194 from chrisr3d/master. [Alexandre Dulaunoy] + + Removed STIX1 related requirements to avoid version issues + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #193 from sebdraven/onyphe_module. [Alexandre Dulaunoy] + + Onyphe module + +* Delete vcs.xml. [sebdraven] + +* Correct codecov. [Sebdraven] + +* Pep 8 compliant. [Sebdraven] + +* Correct type of comments. [Sebdraven] + +* Correct typo. [Sebdraven] + +* Correct typo. [Sebdraven] + +* Add domains forward. [Sebdraven] + +* Add domains. [Sebdraven] + +* Add targeting os. [Sebdraven] + +* Add category for AS number. [Sebdraven] + +* Change keys. [Sebdraven] + +* Change type. [Sebdraven] + +* Add category. [Sebdraven] + +* Add as number with onyphe. [Sebdraven] + +* Add as number with onyphe. [Sebdraven] + +* Error indentation. [Sebdraven] + +* Correct key in map result. [Sebdraven] + +* Correct a bug. [Sebdraven] + +* Add pastebin url imports. [Sebdraven] + +* Add onyphe module. [Sebdraven] + +* Updated requirements to avoid version issues in the MISP packer installation script. [chrisr3d] + +* Update countrycode.py. [Andras Iklody] + +* Add: mixing modules. [Alexandre Dulaunoy] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Alexandre Dulaunoy] + +* Merge pull request #190 from chrisr3d/master. [Alexandre Dulaunoy] + + Updated csv import following our recent discussions + +* Updated delimiter finder function. [chrisr3d] + +* Add: Added user config to specify if there is a header in the csv to import. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #189 from chrisr3d/master. [Andras Iklody] + + Using userConfig to define the header instead of moduleconfig + +* Merge pull request #188 from cvandeplas/master. [Christophe Vandeplas] + + ta import - noise removal + +* Merge branch 'master' into master. [Christophe Vandeplas] + +* Merge pull request #187 from cvandeplas/master. [Christophe Vandeplas] + + threatanalyzer_import - minor generic noise removal + +* Threatanalyzer_import - minor generic noise removal. [Christophe Vandeplas] + +* Ta import - more filter for pollution. [Christophe Vandeplas] + +* Threatanalyzer_import - minor generic noise removal. [Christophe Vandeplas] + +* Merge pull request #185 from cvandeplas/master. [Christophe Vandeplas] + + threatanalyzer_import - loads sample info + pollution fix + +* Threatanalyzer_import - loads sample info + pollution fix. [Christophe Vandeplas] + +* Merge pull request #184 from cvandeplas/master. [Christophe Vandeplas] + + threatanalyzer_import - fix regkey issue + +* Threatanalyzer_import - fix regkey issue. [Christophe Vandeplas] + +* Merge pull request #177 from TheDr1ver/patch-1. [Alexandre Dulaunoy] + + fix missing comma + +* Fix missing comma. [Nick Driver] + + fix ip-dst and vulnerability input + +* Merge pull request #176 from cudeso/master. [Alexandre Dulaunoy] + + Fix VMRay API access error + +* Fix VMRay API access error. [Koen Van Impe] + + hotfix for the "Unable to access VMRay API" error + +* Merge remote-tracking branch 'MISP/master' [Koen Van Impe] + +* Merge pull request #173 from m3047/master. [Alexandre Dulaunoy] + + Add exception blocks for query errors. + +* Add exception blocks for query errors. [Fred Morris] + +* Merge pull request #170 from P4rs3R/patch-1. [Alexandre Dulaunoy] + + Improving regex (validating e-mail) + +* Improving regex (validating e-mail) [x41\x43] + + Line 48: + The previous regex ` ^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$ ` matched only a small subset of valid e-mail address (e.g.: didn't match domain names longer than 3 chars or user@this-domain.de or user@multiple.level.dom) and needed to be with start (^) and end ($). + This ` [a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])? ` is not perfect (e.g: can't match oriental chars), but imho is much more complete. + + Regex tested with several e-mail addresses with Python 3.6.4 and Python 2.7.14 on Linux 4.14. + +* Merge pull request #169 from chrisr3d/master. [Alexandre Dulaunoy] + + Updated GoAML import including Object References + +* Clarified functions arguments using a class. [chrisr3d] + +* Add: Added Object References in the objects imported. [chrisr3d] + +* Merge pull request #168 from chrisr3d/goaml. [Alexandre Dulaunoy] + + GoAML import module & GoAML export updates + +* Merge branch 'master' of github.com:MISP/misp-modules into goaml. [chrisr3d] + +* Merge pull request #167 from chrisr3d/csvimport. [Alexandre Dulaunoy] + + Updated csvimport + +* Merge branch 'csvimport' of github.com:chrisr3d/misp-modules into goaml. [chrisr3d] + +* Removed print. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into csvimport. [chrisr3d] + +* Merge pull request #165 from chrisr3d/goaml. [Alexandre Dulaunoy] + + fix: Added an object checking + +* Add: added goamlimport. [chrisr3d] + +* Fixed some details about the module output. [chrisr3d] + +* Converting GoAML into MISPEvent. [chrisr3d] + +* Now parsing all the transaction attributes. [chrisr3d] + +* Add: Added dictionary to map aml types into MISP types. [chrisr3d] + +* Typo. [chrisr3d] + +* Merge branch 'master' of github.com:chrisr3d/misp-modules into aml_import. [chrisr3d] + +* Merge pull request #164 from chrisr3d/master. [Alexandre Dulaunoy] + + Latest fixes to make GoAML export module work + +* Add: Added an example file generated by GoAML export module. [chrisr3d] + +* Added GoAML export module in description. [chrisr3d] + +* Reading the entire document, to create a big dictionary containing the data, as a beginning. [chrisr3d] + +* Add: new expansion module to check hashes against hashdd.com including NSLR dataset. [Alexandre Dulaunoy] + +* Merge pull request #163 from chrisr3d/master. [Alexandre Dulaunoy] + + GoAML export + +* Typo. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Quick fix to the invalid hash types offered on all returned hashes, hopefully fixes #162. [Andras Iklody] + +* Explicit name. [chrisr3d] + + Avoiding confusion with the coming import module for goaml + +* Added "t_to" and "t_from" required fields: funds code & country. [chrisr3d] + +* Added a required field & the latest attributes in transaction. [chrisr3d] + +* Added report expected information fields. [chrisr3d] + +* Simplified ObjectReference dictionary reading. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* Add: YARA syntax validator. [Alexandre Dulaunoy] + +* Merge pull request #161 from eCrimeLabs/ecrimelabs_dev. [Alexandre Dulaunoy] + + Added Yara syntax validation expansion module + +* Added Yara syntax validation expansion module. [Dennis Rand] + +* Added some report information. [chrisr3d] + + Also changed the ObjectReference parser to replace + all the if conditions by a dictionary reading + +* Suporting the recent objects added to misp-objects. [chrisr3d] + + - Matching the aml documents structure + - Some parts of the document still need to be added + +* Wip: added location & signatory information. [chrisr3d] + +* Merge branch 'master' of github.com:MISP/misp-modules into test. [chrisr3d] + +* Merge pull request #157 from CenturyLinkCIRT/master. [Alexandre Dulaunoy] + + added csvimport to __init__.py + +* Added csvimport to __init__.py. [Thomas Gardner] + +* Add: CSV import module added. [Alexandre Dulaunoy] + +* Outputting xml format. [chrisr3d] + + Also mapping MISP and GoAML types + +* First tests for the GoAML export module. [chrisr3d] + +* Merge pull request #156 from chrisr3d/master. [Alexandre Dulaunoy] + + CSV import + +* Merge branch 'master' of github.com:MISP/misp-modules. [chrisr3d] + +* 3.7-alpha removed. [Alexandre Dulaunoy] + +* Updated delimiter finder method. [chrisr3d] + +* Fixed data treatment & other updates. [chrisr3d] + +* Updated delimiter parsing & data reading functions. [chrisr3d] + +* First version of csv import module. [chrisr3d] + + - If more than 1 misp type is recognized, for each one an + attribute is created + + - Needs to have header set by user as parameters of the module atm + + - Review needed to see the feasibility with fields that can create + confusion and be interpreted both as misp type or attribute field + (for instance comment is a misp type and an attribute field) + +* Merge pull request #154 from cvandeplas/master. [Raphaël Vinot] + + added CrowdStrike Falcon Intel Indicators expansion module + +* Added CrowdStrike Falcon Intel Indicators expansion module. [Christophe Vandeplas] + +* Add: RBL added. [Alexandre Dulaunoy] + +* Merge pull request #150 from chrisr3d/master. [Alexandre Dulaunoy] + + RBL check module + +* Merge github.com:MISP/misp-modules. [chrisr3d] + +* Merge pull request #149 from cvandeplas/master. [Alexandre Dulaunoy] + + Added ThreatAnalyzer sandbox import + +* Added ThreatAnalyzer sandbox import. [Christophe Vandeplas] + + Experimental module - some parts should be migrated to + +* Check an IPv4 address against known RBLs. [chrisr3d] + +* Fix farsight_passivedns - rdata 404 not found. [Christophe Vandeplas] + +* Added ThreatStream and PDF export. [Alexandre Dulaunoy] + +* Merge branch 'robertnixon2003-master' + a small fix. [Alexandre Dulaunoy] + +* Fix the __init__ import. [Alexandre Dulaunoy] + +* Update threatStream_misp_export.py. [Robert Nixon] + +* Updated __init__.py. [Robert Nixon] + + Added reference to new ThreatStream export module + +* Added threatStream_misp_export.py. [Robert Nixon] + +* Merge branch 'cvandeplas-master' [Alexandre Dulaunoy] + +* Fixes missing init file in dnsdb library folder. [Christophe Vandeplas] + +* New Farsight DNSDB Passive DNS expansion module. [Christophe Vandeplas] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Raphaël Vinot] + +* Merge pull request #144 from attritionorg/patch-1. [Andras Iklody] + + minor touch-ups on error messages for user friendliness + +* Minor touch-ups on error messages for user friendliness. [Jericho] + +* Merge pull request #140 from cudeso/master. [Alexandre Dulaunoy] + + VulnDB Queries + +* VulnDB Queries. [Koen Van Impe] + + Search on CVE at https://vulndb.cyberriskanalytics.com/ + https://www.riskbasedsecurity.com/ + Get extended CVE info, links + CPE + +* Merge remote-tracking branch 'MISP/master' [Koen Van Impe] + +* Add quick and dirty pdf export. [Raphaël Vinot] + +* Merge pull request #139 from Rafiot/master. [Raphaël Vinot] + + fix: OpenIOC importer + +* Merge pull request #135 from DomainTools/domaintools-patch-1. [Raphaël Vinot] + + Added code to allow 3rd party modules + +* Added default parameter for new -m flag. [Viktor von Drakk] + +* Added code to allow 3rd party modules. [Viktor von Drakk] + + The new '-m pip.module.name' feature allows a pip-installed module to be specified on the command line and then loaded into the available modules without having to copy-paste files into the appropriate directories of this package. + +* Broken links fixed. [Alexandre Dulaunoy] + +* ThreatConnect export module added. [Alexandre Dulaunoy] + +* Merge pull request #133 from CenturyLinkCIRT/master. [Alexandre Dulaunoy] + + ThreatConnect export module + +* Added threat_connect_export to export_mod.__init__ [Thomas Gardner] + +* Added test files for threat_connect_export. [Thomas Gardner] + +* Added threat_connect_export.py. [Thomas Gardner] + +* Merge pull request #129 from seamustuohy/utf_hate. [Raphaël Vinot] + + Added support for malformed internationalized email headers + +* Added support for malformed internationalized email headers. [seamus tuohy] + + When an emails contains headers that use Unicode without properly crafing + them to comform to RFC-6323 the email import module would crash. + (See issue #119 & issue #93) + + To address this I have added additional layers of encoding/decoding to + any possibly internationalized email headers. This decodes properly + formed and malformed UTF-8, UTF-16, and UTF-32 headers appropriately. + When an unknown encoding is encountered it is returned as an 'encoded-word' + per RFC2047. + + This commit also adds unit-tests that tests properly formed and malformed + UTF-8, UTF-16, UTF-32, and CJK encoded strings in all header fields; UTF-8, + UTF-16, and UTF-32 encoded message bodies; and emoji testing for headers + and attachment file names. + +* Merge branch 'master' into utf_hate. [seamus tuohy] + +* Added unit tests for UTF emails. [seamus tuohy] + +* OTX and ThreatCrowd added. [Alexandre Dulaunoy] + +* Merge pull request #130 from chrisdoman/master. [Alexandre Dulaunoy] + + Add AlienVault OTX and ThreatCrowd Expansions + +* Add AlienVault OTX and ThreatCrowd Expansions. [Chris Doman] + +* Use proper version of PyMISP. [Raphaël Vinot] + +* Update travis, fix open ioc import. [Raphaël Vinot] + +* Merge pull request #122 from truckydev/master. [Alexandre Dulaunoy] + + Add tags on import with ioc import module + +* Replace tab by space. [Tristan METAYER] + +* Add a field for user to add tag for this import. [Tristan METAYER] + +* Merge pull request #121 from truckydev/master. [Andras Iklody] + + If filename add iocfilename as attachment + +* Typo correction. [Tristan METAYER] + +* Add user config to not add file as attachement in a box. [Tristan METAYER] + +* If filename add iocfilename as attachment. [Tristan METAYER] + +* Merge pull request #118 from truckydev/master. [Alexandre Dulaunoy] + + Add indent field for export + +* Add indent field for export. [Tristan METAYER] + +* Merge pull request #115 from FloatingGhost/master. [Alexandre Dulaunoy] + + fix: Use the proper formatting method and not the horrible % one + +* Missing expansion modules added in README. [Alexandre Dulaunoy] + +* ThreatMiner added. [Alexandre Dulaunoy] + +* Merge pull request #114 from kx499/master. [Alexandre Dulaunoy] + + ThreatMiner Expansion module + +* Bug fixes. [kx499] + +* Threatminer initial commit. [kx499] + +* Cosmetic changes. [Raphaël Vinot] + +* Merge pull request #111 from kx499/master. [Raphaël Vinot] + + Handful of changes to VirusTotal module + +* Bug fixes, tweaks, and python3 learning curve :) [kx499] + +* Initial commit of IPRep module. [kx499] + +* Fixed spacing, addressed error handling for public api, added subdomains, and added context comment. [kx499] + +* OpenIOC import module added. [Alexandre Dulaunoy] + +* Add OpenIOC import module. [Raphaël Vinot] + +* Merge pull request #109 from truckydev/master. [Alexandre Dulaunoy] + + add information about offline installation + +* Add information about offline installation. [truckydev] + +* Merge pull request #106 from truckydev/master. [Alexandre Dulaunoy] + + Lite export of an event + +* Exclude internal reference. [Tristan METAYER] + +* Add lite Export module. [Tristan METAYER] + +* Merge pull request #100 from rmarsollier/master. [Alexandre Dulaunoy] + + Some improvements of virustotal plugin + +* Some improvements of virustotal plugin. [rmarsollier] + +* Merge pull request #96 from johestephan/master. [Raphaël Vinot] + + XForce Exchange v1 (alpha) + +* Passed local run check. [Joerg Stephan] + +* V1. [Joerg Stephan] + +* Removed urrlib2. [Joerg Stephan] + +* Python3 changes. [Joerg Stephan] + +* Merged xforce exchange. [Joerg Stephan] + +* XForce Exchange v1 (alpha) [Joerg Stephan] + +* Merge pull request #56 from RichieB2B/ncsc-nl/mispjson. [Alexandre Dulaunoy] + + Simple import module to import MISP JSON format + +* Updated description to reflect merging use case. [Richard van den Berg] + +* Simple import module to import MISP JSON format. [Richard van den Berg] + +* Merge pull request #92 from seamustuohy/duck_typing_failure. [Alexandre Dulaunoy] + + Email import no longer unzips major compressed text document formats. + +* Email import no longer unzips major compressed text document formats. [seamus tuohy] + + Let this commit serve as a warning about the perils of duck typing. + Word documents (docx,odt,etc) were being uncompressed when they were + attached to emails. The email importer now checks a list of well known + extensions and will not attempt to unzip them. + + It is stuck using a list of extensions instead of using file magic because + many of these formats produce an application/zip mimetype when scanned. + +* Merge branch 'master' of github.com:MISP/misp-modules. [Raphaël Vinot] + +* Merge pull request #91 from Rafiot/master. [Raphaël Vinot] + + Improve email import module + +* Keep zip content as binary. [Raphaël Vinot] + +* Fix tests, cleanup. [Raphaël Vinot] + +* Improve support of email attachments. [Raphaël Vinot] + + Related to #90 + +* Merge pull request #89 from Rafiot/fix_87. [Raphaël Vinot] + + Improve VT support. + +* Standardised key checking. [Hannah Ward] + +* Fixed checking for submission_names in VT JSON. [Hannah Ward] + +* Update virustotal.py. [CheYenBzh] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Raphaël Vinot] + +* Training materials updated + Cuckoo JSON import module was missing. [Alexandre Dulaunoy] + +* Improve support of email importer if headers are missing. [Raphaël Vinot] + + Fix #88 + +* Remove python 3.3 support. [Raphaël Vinot] + +* Fix python 3.6 support. [Raphaël Vinot] + +* Make PEP8 happy. [Raphaël Vinot] + +* Add email_import in the modules loaded by default. [Raphaël Vinot] + +* Make PEP8 happy. [Raphaël Vinot] + +* Fix failing test (bug in the mail parser?) [Raphaël Vinot] + +* Add additional email parsing and tests. [seamus tuohy] + + Added additional attribute parsing and corresponding unit-tests. + E-mail attachment and url extraction added in this commit. This includes + unpacking zipfiles and simple password cracking of encrypted zipfiles. + +* Fixed basic errors. [seamus tuohy] + +* Merged with current master. [seamus tuohy] + +* Merge pull request #85 from rmarsollier/master. [Raphaël Vinot] + + add libjpeg-dev as a dep to allow pillow to be installed succesfully + +* Add libjpeg-dev as a dep to allow pillow to be installed succesfully. [robin.marsollier@conix.fr] + +* GeoIP module added. [Alexandre Dulaunoy] + +* Merge pull request #84 from MISP/amuehlem-master. [Raphaël Vinot] + + Fix PR + +* Do not crash if the dat file is not available. [Raphaël Vinot] + +* Fix path to config file. [Raphaël Vinot] + +* Merge branch 'master' of https://github.com/amuehlem/misp-modules into amuehlem-master. [Raphaël Vinot] + +* Added empty line to end of config file. [Andreas Muehlemann] + +* Removed DEFAULT section from configfile. [Andreas Muehlemann] + +* Fixed more typos. [Andreas Muehlemann] + +* Fixed typo. [Andreas Muehlemann] + +* Changed configparser from python2 to python3. [Andreas Muehlemann] + +* Updated missing parenthesis. [Andreas Muehlemann] + +* Merge branch 'geoip_country' [Andreas Muehlemann] + +* Removed unneeded config option for misp. [Andreas Muehlemann] + +* Removed debug message. [Andreas Muehlemann] + +* Added config option to geoip_country.py. [Andreas Muehlemann] + +* Added pygeoip to the REQUIREMENTS list. [Andreas Muehlemann] + +* Updated geoip_country to __init__.py. [Andreas Muehlemann] + +* Added geoip_country.py. [Andreas Muehlemann] + +* Better error reporting. [Raphaël Vinot] + +* Catch exception. [Raphaël Vinot] + +* Add reverse lookup. [Raphaël Vinot] + +* Refactoring of domaintools expansion module. [Raphaël Vinot] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Raphaël Vinot] + +* Merge pull request #83 from stoep/master. [Alexandre Dulaunoy] + + Added cuckooimport.py + +* Added cuckooimport.py. [Ubuntu] + +* DomainTools module added. [Alexandre Dulaunoy] + +* Remove domaintools tests. [Raphaël Vinot] + +* Add test for domaintools. [Raphaël Vinot] + +* Merge pull request #78 from deralexxx/patch-2. [Alexandre Dulaunoy] + + Update README.md + +* Update README.md. [Alexander J] + + mentioning import / export modules + +* Merge pull request #76 from deralexxx/patch-1. [Alexandre Dulaunoy] + + Update README.md + +* Update README.md. [Alexander J] + +* Merge pull request #75 from Rafiot/domtools. [Raphaël Vinot] + + Add Domain Tools module + +* Update requirements list. [Raphaël Vinot] + +* Add domaintools to the import list. [Raphaël Vinot] + +* Fix Typo. [Raphaël Vinot] + +* Add domain profile and reputation. [Raphaël Vinot] + +* Add more comments. [Raphaël Vinot] + +* Fix typo. [Raphaël Vinot] + +* Remove json.dumps. [Raphaël Vinot] + +* Avoid passing None in comments. [Raphaël Vinot] + +* Add comments to fields when possible. [Raphaël Vinot] + +* Add initial Domain Tools module. [Raphaël Vinot] + +* Merge pull request #74 from cudeso/master. [Raphaël Vinot] + + Extra VTI detections + +* Merge remote-tracking branch 'MISP/master' [Koen Van Impe] + +* Update README.md. [Raphaël Vinot] + +* Merge pull request #73 from FloatingGhost/master. [Raphaël Vinot] + + Use SpooledTemp, not NamedTemp file + +* Use git for everything we can. [Hannah Ward] + +* Ok we'll use the dep from misp-stix-converter. Surely this'll work? [Hannah Ward] + +* Use the CIRCL pymisp. Silly @rafiot ;) [Hannah Ward] + +* Travis should now use the master branch. [Hannah Ward] + +* Maybe it'll take the git repo now? [Hannah Ward] + +* Added pymisp to reqs. [Hannah Ward] + +* Don't cache anything pls travis. [Hannah Ward] + +* Removed unneeded modules. [Hannah Ward] + +* Use SpooledTemp, not NamedTemp file. [Hannah Ward] + +* VMRay import module added. [Alexandre Dulaunoy] + +* Merge pull request #72 from FloatingGhost/master. [Raphaël Vinot] + + Migrated stiximport to use misp-stix-converter + +* Moved to misp_stix_converter. [Hannah Ward] + +* Merge pull request #70 from cudeso/master. [Raphaël Vinot] + + Submit malware samples + +* Extra VTI detections. [Koen Van Impe] + +* Submit malware samples. [Koen Van Impe] + + _submit now includes malware samples (zipped content from misp) + _import checks when no vti_results are returned + bugfix + +* Fix STIX import module. [Raphaël Vinot] + +* Multiple clanges in the vmray modules. [Raphaël Vinot] + + * Generic fix to load modules requiring a local library + * Fix python3 support + * PEP8 related cleanups + +* Merge pull request #68 from cudeso/master. [Andras Iklody] + + VMRay Import & Submit module + +* VMRay Import & Submit module. [Koen Van Impe] + + * First commit + * No support for archives (yet) submit + +* Merge pull request #59 from rgraf/master. [Alexandre Dulaunoy] + + label replaced by text, which is existing attribute + +* Label replaced by text, which is existing attribute. [Roman Graf] + +* Adding basic test mockup. [seamus tuohy] + +* Adding more steps to module testing. [seamus tuohy] + +* Added attachment and url support. [seamus tuohy] + +* Added email meta-data import module. [seamus tuohy] + + This email meta-data import module collects basic meta-data from an e-mail + and populates an event with it. It populates the email subject, source + addresses, destination addresses, subject, and any attachment file names. + This commit also contains unit-tests for this module as well as updates to + the readme. Readme updates are additions aimed to make it easier for + outsiders to build modules. + +* Merge pull request #58 from rgraf/master. [Alexandre Dulaunoy] + + Added expansion for Wikidata. + +* Added expansion for Wikidata. Analyst can query Wikidata by label to get additional information for particular term. [Roman Graf] + +* Merge pull request #55 from amuehlem/reversedns. [Raphaël Vinot] + + added new module reversedns.py, added reversedns to __init__.py + +* Added new module reversedns.py, added reversedns to __init__.py. [Andreas Muehlemann] + +* Merge pull request #53 from MISP/Rafiot-patch-1. [Alexandre Dulaunoy] + + Dump host info as text + +* Dump host info as text. [Raphaël Vinot] + +* Fix typo. [Raphaël Vinot] + +* Merge pull request #52 from Rafiot/master. [Alexandre Dulaunoy] + + Add simple Shodan module + +* Add simple Shodan module. [Raphaël Vinot] + +* Merge pull request #49 from FloatingGhost/master. [Alexandre Dulaunoy] + + Removed useless pickle storage of stiximport + +* Removed useless pickle storage of stiximport. [Hannah Ward] + +* Create LICENSE. [Alexandre Dulaunoy] + +* Update README.md. [Andras Iklody] + +* Typo fixed. [Alexandre Dulaunoy] + +* CEF export module added. [Alexandre Dulaunoy] + +* Cef_export module added. [Alexandre Dulaunoy] + +* Merge pull request #47 from FloatingGhost/CEF_Export. [Alexandre Dulaunoy] + + CEF export, fixes in CountryCode, virustotal + +* Removed silly subdomain module. [Hannah Ward] + +* Added CEF export module. [Hannah Ward] + +* Now searches within observable_compositions. [Hannah Ward] + +* Removed calls to print. [Hannah Ward] + +* Added body.json to gitignore. [Hannah Ward] + +* Added virustotal tests. [Hannah Ward] + +* CountryCode JSON now is only grabbed once per server run. [Hannah Ward] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Raphaël Vinot] + +* Merge pull request #46 from Rafiot/master. [Raphaël Vinot] + + Make misp-modules really asynchronous + +* Add timeout for the modules, cleanup. [Raphaël Vinot] + +* Fix python 3.3 and 3.4. [Raphaël Vinot] + +* Make misp-modules really asynchronous. [Raphaël Vinot] + +* Improve tornado parallel. [Raphaël Vinot] + +* Coroutine decorator added to post handler. [Alexandre Dulaunoy] + +* -d option added - enabling debug on queried modules. [Alexandre Dulaunoy] + +* New modules added to __init__ [Alexandre Dulaunoy] + +* README updated for the new modules. [Alexandre Dulaunoy] + +* Merge pull request #45 from FloatingGhost/master. [Alexandre Dulaunoy] + + 2 new modules -- VirusTotal and CountryCode + +* Modified readme with virustotal/countrycode. [Hannah Ward] + +* Added virustotal module. [Hannah Ward] + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Hannah Ward] + +* Merge pull request #44 from Rafiot/travis. [Alexandre Dulaunoy] + + Add coverage, update logging + +* Add coverage, update logging. [Raphaël Vinot] + +* Merge pull request #43 from FloatingGhost/master. [Alexandre Dulaunoy] + + StixImport now uses TemporaryFile rather than a named file in /tmp + +* Improved virustotal module. [Hannah Ward] + +* Added countrycode, working on virustotal. [Hannah Ward] + +* Added lookup by country code. [Hannah Ward] + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Hannah Ward] + +* Fix a link to the STIX import module reference. [Alexandre Dulaunoy] + +* Stiximport now uses temporary files to store stix data. [Hannah Ward] + + Set max size in config, in bytes + +* Merge pull request #42 from MISP/pr/41. [Alexandre Dulaunoy] + + Cleanup on the stix import module + +* Merge remote-tracking branch 'origin/master' into pr/41. [Raphaël Vinot] + +* Add info about the import modules. [Alexandre Dulaunoy] + +* Make PEP8 happy \o/ [Raphaël Vinot] + +* Move stiximport.py to misp_modules/modules/import_mod/ [Raphaël Vinot] + +* There was a missing comma. [Hannah Ward] + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Hannah Ward] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Alexandre Dulaunoy] + +* Merge pull request #40 from Rafiot/master. [Alexandre Dulaunoy] + + Remove bin script, use cleaner way. Fix last commit. + +* Remove bin script, use cleaner way. Fix last commit. [Raphaël Vinot] + +* Merge pull request #39 from Rafiot/master. [Alexandre Dulaunoy] + + Use entry_points instead of scripts in the install. + +* Use entry_points instead of scripts. [Raphaël Vinot] + +* Pip --upgrade must be always called (to have modules updated) [Alexandre Dulaunoy] + +* Added STIX to setup.py. [Hannah Ward] + +* Added STIX to reqs. [Hannah Ward] + +* Merge branch 'stix_import' [Hannah Ward] + +* Added tests, also disregards related_observables. Because they're useless. [Hannah Ward] + +* Fixed observables within an indicator not being added. [Hannah Ward] + +* Stiximport will now consume campaigns. [Hannah Ward] + +* Stiximport will now identify file hashes. [Hannah Ward] + +* I can't spell. [Hannah Ward] + +* Added STIXImport to readme. [Hannah Ward] + +* Threat actors now get imported by stix. [Hannah Ward] + +* Added docs to stiximport. [Hannah Ward] + +* Added stix import -- works for IPs/Domains. [Hannah Ward] + +* Update to the DNS module to support domain|ip. [iglocska] + +* Small change to the skeleton export. [iglocska] + +* Merge remote-tracking branch 'origin/import-test' [iglocska] + +* Added test export module. [Iglocska] + +* Merge branch 'master' of github.com:MISP/misp-modules. [Alexandre Dulaunoy] + +* Merge pull request #37 from Rafiot/master. [Raphaël Vinot] + + Update documentation. + +* Update documentation. [Raphaël Vinot] + + Fix https://github.com/MISP/MISP/issues/1424 + +* Merge branch 'import-test' of github.com:MISP/misp-modules into import-test. [Alexandre Dulaunoy] + +* Merge pull request #36 from Rafiot/import-test. [Alexandre Dulaunoy] + + Pass the server port as integer to the uwhois client + +* Pass the server port as integer to the uwhois client. [Raphaël Vinot] + +* Merge pull request #35 from Rafiot/import-test. [Alexandre Dulaunoy] + + Add whois module + +* Add whois module. [Raphaël Vinot] + +* First version of an Optical Character Recognition (OCR) module for MISP. [Alexandre Dulaunoy] + +* First version of the import skeleton. [Iglocska] + +* Added simple import skeleton. [Iglocska] + +* Merge pull request #33 from Rafiot/master. [Raphaël Vinot] + + fix: run the server as "python3 misp-modules" + +* Added category to the return format description. [Iglocska] + +* Merge pull request #31 from treyka/patch-1. [Alexandre Dulaunoy] + + Refine the installation procedure + +* Refine the installation procedure. [Trey Darley] + + Tweak this to make it more inline with the MISP installation docs, start misp-modules at startup via /etc/rc.local + +* Install documentation updated. [Alexandre Dulaunoy] + +* Merge pull request #28 from Rafiot/pip. [Alexandre Dulaunoy] + + Make it a package + +* Also run travis tests on the system-wide instance. [Raphaël Vinot] + +* Fix typos in the readme. [Raphaël Vinot] + +* Fix travis. [Raphaël Vinot] + +* Make sure misp-modules can be launched from anywhere. [Raphaël Vinot] + +* Proper testcases. [Raphaël Vinot] + +* Make it a package. [Raphaël Vinot] + +* Merge pull request #29 from iglocska/master. [Alexandre Dulaunoy] + + Added skeleton structure for new modules + +* Added skeleton structure for new modules. [Iglocska] + +* Fixed a bug introduced by previous commit if started from the current directory. [Alexandre Dulaunoy] + +* Merge pull request #26 from Rafiot/master. [Alexandre Dulaunoy] + + Automatic chdir when the modules are started + +* Automatic chdir when the modules are started. [Raphaël Vinot] + +* Merge pull request #25 from eu-pi/eupi_expansion_fix. [Alexandre Dulaunoy] + + [EUPI] Fix expansion for empty EUPI response + +* [EUPI] Fix expansion for empty EUPI response. [Rogdham] + + Offer no enrichment instead of displaying an error message + +* Merge pull request #24 from eu-pi/eupi_hover. [Alexandre Dulaunoy] + + [EUPI] Change module for a simple hover status + +* [EUPI] Simplify hover. [Rogdham] + +* Merge pull request #23 from Rafiot/master. [Raphaël Vinot] + + [EUPI] Return error message if unknown + +* [EUPI] Return error message is unknown. [Raphaël Vinot] + +* Merge pull request #22 from Rafiot/master. [Raphaël Vinot] + + [EUPI] Do not return empty results + +* [EUPI] Do not return empty results. [Raphaël Vinot] + +* ASN History added. [Alexandre Dulaunoy] + +* Merge pull request #21 from Rafiot/master. [Raphaël Vinot] + + [ASN description] Fix input type + +* [ASN description] Fix input type. [Raphaël Vinot] + +* Merge pull request #20 from Rafiot/master. [Raphaël Vinot] + + Add ASN Description expansion module + +* Add ASN Description expansion module. [Raphaël Vinot] + +* Merge pull request #19 from Rafiot/master. [Raphaël Vinot] + + Fix last commit + +* Fix last commit. [Raphaël Vinot] + +* Merge pull request #18 from Rafiot/master. [Raphaël Vinot] + + Improve rendering of IP ASN + +* Improve rendering of IP ASN. [Raphaël Vinot] + +* Merge pull request #17 from Rafiot/master. [Raphaël Vinot] + + Fix again IPASN module + +* Fix again IPASN module. [Raphaël Vinot] + +* Merge pull request #16 from Rafiot/master. [Raphaël Vinot] + + Fix IPASN module + +* Fix IPASN module. [Raphaël Vinot] + +* Ipasn module added. [Alexandre Dulaunoy] + +* Merge pull request #15 from Rafiot/master. [Alexandre Dulaunoy] + + Add IPASN history module + +* Add IPASN history module. [Raphaël Vinot] + +* Merge pull request #14 from eu-pi/listen-addr. [Alexandre Dulaunoy] + + Add option to specify listen address + +* Add option to specify listen address. [Rogdham] + +* EUPI module added. [Alexandre Dulaunoy] + +* Merge pull request #13 from Rafiot/master. [Raphaël Vinot] + + Fix eupi module + +* Fix eupi module. [Raphaël Vinot] + +* Merge pull request #12 from Rafiot/master. [Raphaël Vinot] + + Add EUPI module + +* Add redis server. [Raphaël Vinot] + +* Add EUPI module. [Raphaël Vinot] + +* Skip modules that cannot import. [Alexandre Dulaunoy] + +* Skip dot files. [Alexandre Dulaunoy] + +* Value is not required. [Alexandre Dulaunoy] + +* Cache helper added. [Alexandre Dulaunoy] + + The cache helper is a simple helper to cache data + in Redis back-end. The format in the cache is the following: + m::sha1(key) -> value. Default expiration is 86400 seconds. + +* Skeleton for misp-modules helpers added. [Alexandre Dulaunoy] + + Helpers will support modules with basic functionalities + like caching or alike. + +* Option -p added to specify the TCP port of the misp-modules server. [Alexandre Dulaunoy] + +* Intelmq req. removed. [Alexandre Dulaunoy] + +* Argparse used for the test mode. [Alexandre Dulaunoy] + +* Deleted. [Alexandre Dulaunoy] + +* Intelmq is an experimental module (not production ready) [Alexandre Dulaunoy] + +* Merge pull request #11 from Rafiot/master. [Raphaël Vinot] + + Fix test mode + +* Fix test mode. [Raphaël Vinot] + +* Fix install commands. [Raphaël Vinot] + +* Add Travis logo. [Raphaël Vinot] + +* Merge pull request #10 from Rafiot/travis. [Raphaël Vinot] + + Add basic travis file + +* Add basic travis file. [Raphaël Vinot] + +* Merge pull request #9 from Rafiot/master. [Alexandre Dulaunoy] + + Please PEP8 on all expansions + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Raphaël Vinot] + +* Merge pull request #8 from aaronkaplan/master. [Alexandre Dulaunoy] + + initial example of intelmq connector/enrichtment. Need to change to u… + +* Initial example of intelmq connector/enrichtment. Need to change to use the eventDB RESTful API, not the postgresql DB. [aaronkaplan] + +* Update README.md. [Raphaël Vinot] + +* Dns module test with option added. [Alexandre Dulaunoy] + +* New modules added. [Alexandre Dulaunoy] + +* Dns MISP module - option to specify nameserver added. [Alexandre Dulaunoy] + +* Slides reference added. [Alexandre Dulaunoy] + +* Add missing requirements. [Alexandre Dulaunoy] + +* Merge pull request #7 from Rafiot/master. [Alexandre Dulaunoy] + + Make loader more flexible + +* Make PEP8 happy. [Raphaël Vinot] + +* Add CIRCL pssl module. [Raphaël Vinot] + +* Make loader more flexible. [Raphaël Vinot] + +* First module to test the freetext import functionality. [Alexandre Dulaunoy] + +* CIRCL Passive DNS output attributes updated. [Alexandre Dulaunoy] + +* PyPDNS requirement added. [Alexandre Dulaunoy] + +* CIRCL Passive DNS added. [Alexandre Dulaunoy] + +* Tests updated to include CIRCL passive dns. [Alexandre Dulaunoy] + +* Test file for passivetotal updated. [Alexandre Dulaunoy] + +* Merge pull request #5 from passivetotal/master. [Alexandre Dulaunoy] + + Rewrote the entire PassiveTotal extension + +* Rewrote the entire PassiveTotal extension. [Brandon Dixon] + +* Return a text attribute for an hover only module. [Alexandre Dulaunoy] + +* How to start MISP modules. [Alexandre Dulaunoy] + +* 2.4.28 includes misp modules by default. [Alexandre Dulaunoy] + +* Types are now described. [Alexandre Dulaunoy] + +* Debug removed. [Alexandre Dulaunoy] + +* Convert the base64 to ascii. [Iglocska] + +* Module-type added as default. [Alexandre Dulaunoy] + +* Return base64 value of the archived data. [Alexandre Dulaunoy] + +* Merge pull request #2 from iglocska/master. [Alexandre Dulaunoy] + + Some changes to the sourcecache expansion + +* Merge branch 'alternate_response' [Iglocska] + +* Some changes to the sourcecache expansion. [Iglocska] + + - return attachment or malware sample + +* Cve module tests added. [Alexandre Dulaunoy] + +* CVE hover expansion module. [Alexandre Dulaunoy] + + An hover module is a module returning a JSON that can be used + as hover element in the MISP UI. + +* Sourcecache module includes the metadata config. [Alexandre Dulaunoy] + +* README updated to reflect config parameters changes. [Alexandre Dulaunoy] + +* Removed unused attributes. [Alexandre Dulaunoy] + +* Sample JSON files reflecting config changes. [Alexandre Dulaunoy] + +* Config parameters are now exposed via the meta information. [Alexandre Dulaunoy] + + config uses a specific list of values exposed via the + introspection of the module. config is now passed as an additional + dictionary to the request. MISP attributes include only MISP attributes. + +* Sourcecache module added. [Alexandre Dulaunoy] + +* A minimal caching module added to cache link or url from MISP. [Alexandre Dulaunoy] + +* Typo fixed + meta output. [Alexandre Dulaunoy] + +* Minimal functions requirements updated + PR request. [Alexandre Dulaunoy] + +* Exclude dot files from modules list to be loaded. [Alexandre Dulaunoy] + +* Example of module introspection including meta information. [Alexandre Dulaunoy] + +* Module meta added to return version, description and author per module. [Alexandre Dulaunoy] + +* Authentication notes added. [Alexandre Dulaunoy] + +* Passivetotal module added. [Alexandre Dulaunoy] + +* First version of a passivetotal MISP expansion module. [Alexandre Dulaunoy] + +* Default DNS updated. [Alexandre Dulaunoy] + +* Add a note regarding error codes. [Alexandre Dulaunoy] + +* Handling of error added. [Alexandre Dulaunoy] + +* Merge pull request #1 from Rafiot/master. [Alexandre Dulaunoy] + + Make PEP8 happy. + +* Make PEP8 happy. [Raphaël Vinot] + +* Output updated (type of module added) [Alexandre Dulaunoy] + +* Add a version per default. [Alexandre Dulaunoy] + +* Add type per module. [Alexandre Dulaunoy] + +* Format updated following Andras updates. [Alexandre Dulaunoy] + +* Default var directory added. [Alexandre Dulaunoy] + +* Python pip REQUIREMENTS file added. [Alexandre Dulaunoy] + +* Merge branch 'master' of https://github.com/MISP/misp-modules. [Iglocska] + +* Minimal logging added to the server. [Alexandre Dulaunoy] + +* Debug messages removed. [Alexandre Dulaunoy] + +* Minimal documentation added. [Alexandre Dulaunoy] + +* Curl is now silent. [Alexandre Dulaunoy] + +* Changed the output format to include all matching attribute types. [Iglocska] + + - changed the output format to give us a bit more flexibility + - return an array of results + - return the valid misp attribute types for each result + +* Basic test cases added. [Alexandre Dulaunoy] + +* MISP dns expansion module. [Alexandre Dulaunoy] + +* First version of a web services to provide ReST API to MISP expansion services. [Alexandre Dulaunoy] + + diff --git a/Makefile b/Makefile index 1cff13f..2d07388 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,15 @@ .PHONY: prepare_docs generate_docs ci_generate_docs test_docs prepare_docs: - cd doc; python generate_documentation.py + cd documentation; python3 generate_documentation.py mkdir -p docs/expansion/logos docs/export_mod/logos docs/import_mod/logos - cp -R doc/logos/* docs/expansion/logos - cp -R doc/logos/* docs/export_mod/logos - cp -R doc/logos/* docs/import_mod/logos - cp LICENSE docs/license.md + mkdir -p docs/logos + cd documentation; cp -R ./logos/* ../docs/logos + cd documentation; cp -R ./logos/* ../docs/expansion/logos + cd documentation; cp -R ./logos/* ../docs/export_mod/logos + cd documentation; cp -R ./logos/* ../docs/import_mod/logos + cp ./documentation/mkdocs/*.md ./docs + cp LICENSE ../docs/license.md install_requirements: pip install -r docs/REQUIREMENTS.txt diff --git a/Pipfile b/Pipfile index 1169368..cd631be 100644 --- a/Pipfile +++ b/Pipfile @@ -11,16 +11,16 @@ flake8 = "*" [packages] dnspython = "*" -requests = {extras = ["security"],version = "*"} +requests = { extras = ["security"], version = "*" } urlarchiver = "*" passivetotal = "*" pypdns = "*" pypssl = "*" pyeupi = "*" -uwhois = {editable = true,git = "https://github.com/Rafiot/uwhoisd.git",ref = "testing",subdirectory = "client"} -pymisp = {editable = true,extras = ["fileobjects,openioc,pdfexport"],git = "https://github.com/MISP/PyMISP.git"} -pyonyphe = {editable = true,git = "https://github.com/sebdraven/pyonyphe"} -pydnstrails = {editable = true,git = "https://github.com/sebdraven/pydnstrails"} +uwhois = { editable = true, git = "https://github.com/Rafiot/uwhoisd.git", ref = "testing", subdirectory = "client" } +pymisp = { extras = ["fileobjects,openioc,pdfexport,email"], version = "*" } +pyonyphe = { editable = true, git = "https://github.com/sebdraven/pyonyphe" } +pydnstrails = { editable = true, git = "https://github.com/sebdraven/pydnstrails" } pytesseract = "*" pygeoip = "*" beautifulsoup4 = "*" @@ -32,20 +32,20 @@ maclookup = "*" vulners = "*" blockchain = "*" reportlab = "*" -pyintel471 = {editable = true,git = "https://github.com/MISP/PyIntel471.git"} +pyintel471 = { editable = true, git = "https://github.com/MISP/PyIntel471.git" } shodan = "*" Pillow = "*" Wand = "*" SPARQLWrapper = "*" domaintools_api = "*" -misp-modules = {editable = true,path = "."} -pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} -pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} +misp-modules = { editable = true, path = "." } +pybgpranking = { editable = true, git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client" } +pyipasnhistory = { editable = true, git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client" } backscatter = "*" pyzbar = "*" opencv-python = "*" np = "*" -ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git/"} +ODTReader = { editable = true, git = "https://github.com/cartertemm/ODTReader.git/" } python-pptx = "*" python-docx = "*" ezodf = "*" @@ -54,13 +54,17 @@ pandas_ods_reader = "*" pdftotext = "*" lxml = "*" xlrd = "*" -idna-ssl = {markers = "python_version < '3.7'"} +idna-ssl = { markers = "python_version < '3.7'" } jbxapi = "*" geoip2 = "*" apiosintDS = "*" assemblyline_client = "*" vt-graph-api = "*" trustar = "*" +markdownify = "==0.5.3" +socialscan = "*" +dnsdb2 = "*" +clamd = "*" [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 73aeaed..896a592 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c2d937b384431e4b313b29bb02db0bd1d3a866ddcb7c6e91cbfa34f88d351b59" + "sha256": "1e40a27ecb603ad61301ace74783297f581773a131fb83638a03ae2488ba4e12" }, "pipfile-spec": 6, "requires": { @@ -18,27 +18,52 @@ "default": { "aiohttp": { "hashes": [ - "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e", - "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326", - "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a", - "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654", - "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a", - "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4", - "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17", - "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec", - "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd", - "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48", - "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59", - "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965" + "sha256:0b795072bb1bf87b8620120a6373a3c61bfcb8da7e5c2377f4bb23ff4f0b62c9", + "sha256:0d438c8ca703b1b714e82ed5b7a4412c82577040dadff479c08405e2a715564f", + "sha256:16a3cb5df5c56f696234ea9e65e227d1ebe9c18aa774d36ff42f532139066a5f", + "sha256:1edfd82a98c5161497bbb111b2b70c0813102ad7e0aa81cbeb34e64c93863005", + "sha256:2406dc1dda01c7f6060ab586e4601f18affb7a6b965c50a8c90ff07569cf782a", + "sha256:2858b2504c8697beb9357be01dc47ef86438cc1cb36ecb6991796d19475faa3e", + "sha256:2a7b7640167ab536c3cb90cfc3977c7094f1c5890d7eeede8b273c175c3910fd", + "sha256:3228b7a51e3ed533f5472f54f70fd0b0a64c48dc1649a0f0e809bec312934d7a", + "sha256:328b552513d4f95b0a2eea4c8573e112866107227661834652a8984766aa7656", + "sha256:39f4b0a6ae22a1c567cb0630c30dd082481f95c13ca528dc501a7766b9c718c0", + "sha256:3b0036c978cbcc4a4512278e98e3e6d9e6b834dc973206162eddf98b586ef1c6", + "sha256:3ea8c252d8df5e9166bcf3d9edced2af132f4ead8ac422eac723c5781063709a", + "sha256:41608c0acbe0899c852281978492f9ce2c6fbfaf60aff0cefc54a7c4516b822c", + "sha256:59d11674964b74a81b149d4ceaff2b674b3b0e4d0f10f0be1533e49c4a28408b", + "sha256:5e479df4b2d0f8f02133b7e4430098699450e1b2a826438af6bec9a400530957", + "sha256:684850fb1e3e55c9220aad007f8386d8e3e477c4ec9211ae54d968ecdca8c6f9", + "sha256:6ccc43d68b81c424e46192a778f97da94ee0630337c9bbe5b2ecc9b0c1c59001", + "sha256:6d42debaf55450643146fabe4b6817bb2a55b23698b0434107e892a43117285e", + "sha256:710376bf67d8ff4500a31d0c207b8941ff4fba5de6890a701d71680474fe2a60", + "sha256:756ae7efddd68d4ea7d89c636b703e14a0c686688d42f588b90778a3c2fc0564", + "sha256:77149002d9386fae303a4a162e6bce75cc2161347ad2ba06c2f0182561875d45", + "sha256:78e2f18a82b88cbc37d22365cf8d2b879a492faedb3f2975adb4ed8dfe994d3a", + "sha256:7d9b42127a6c0bdcc25c3dcf252bb3ddc70454fac593b1b6933ae091396deb13", + "sha256:8389d6044ee4e2037dca83e3f6994738550f6ee8cfb746762283fad9b932868f", + "sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4", + "sha256:c1e0920909d916d3375c7a1fdb0b1c78e46170e8bb42792312b6eb6676b2f87f", + "sha256:c68fdf21c6f3573ae19c7ee65f9ff185649a060c9a06535e9c3a0ee0bbac9235", + "sha256:c733ef3bdcfe52a1a75564389bad4064352274036e7e234730526d155f04d914", + "sha256:c9c58b0b84055d8bc27b7df5a9d141df4ee6ff59821f922dd73155861282f6a3", + "sha256:d03abec50df423b026a5aa09656bd9d37f1e6a49271f123f31f9b8aed5dc3ea3", + "sha256:d2cfac21e31e841d60dc28c0ec7d4ec47a35c608cb8906435d47ef83ffb22150", + "sha256:dcc119db14757b0c7bce64042158307b9b1c76471e655751a61b57f5a0e4d78e", + "sha256:df3a7b258cc230a65245167a202dd07320a5af05f3d41da1488ba0fa05bc9347", + "sha256:df48a623c58180874d7407b4d9ec06a19b84ed47f60a3884345b1a5099c1818b", + "sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7", + "sha256:f326b3c1bbfda5b9308252ee0dcb30b612ee92b0e105d4abec70335fab5b1245", + "sha256:f411cb22115cb15452d099fec0ee636b06cf81bfb40ed9c02d30c8dc2bc2e3d1" ], - "markers": "python_full_version >= '3.5.3'", - "version": "==3.6.2" + "index": "pypi", + "version": "==3.7.3" }, "antlr4-python3-runtime": { "hashes": [ "sha256:15793f5d0512a372b4e7d2284058ad32ce7dd27126b105fb0b2245130445db33" ], - "markers": "python_version >= '3'", + "index": "pypi", "version": "==4.8" }, "apiosintds": { @@ -53,31 +78,39 @@ "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" ], + "index": "pypi", "version": "==1.4.0" }, "assemblyline-client": { "hashes": [ - "sha256:6f45cab3be3ec60984a5c2049d46dac80d4e3d4f3d9538220518a44c7a6ddb15", - "sha256:971371065f2b41027325bf9fa9c72960262a446c7e08bda57865d34dcc4108b0" + "sha256:6a36a654185ba40d10bdd0213a1926aacb4351290824e406cbff6b6b5b251f5f" ], "index": "pypi", - "version": "==3.7.3" + "version": "==4.0.1" }, "async-timeout": { "hashes": [ "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" ], - "markers": "python_full_version >= '3.5.3'", + "index": "pypi", "version": "==3.0.1" }, "attrs": { "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==19.3.0" + "index": "pypi", + "version": "==20.3.0" + }, + "backoff": { + "hashes": [ + "sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd", + "sha256:b8fba021fac74055ac05eb7c7bfce4723aedde6cd0a504e5326bcb0bdd6d19a4" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.10.0" }, "backscatter": { "hashes": [ @@ -89,12 +122,20 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", - "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", - "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" + "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35", + "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25", + "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666" ], "index": "pypi", - "version": "==4.9.1" + "version": "==4.9.3" + }, + "bidict": { + "hashes": [ + "sha256:4fa46f7ff96dc244abfc437383d987404ae861df797e2fd5b190e233c302be09", + "sha256:929d056e8d0d9b17ceda20ba5b24ac388e2a4d39802b87f9f4d3f45ecba070bf" + ], + "index": "pypi", + "version": "==0.21.2" }, "blockchain": { "hashes": [ @@ -103,59 +144,87 @@ "index": "pypi", "version": "==1.4.4" }, + "censys": { + "hashes": [ + "sha256:0a2e2c73471d37fb55c4c5597d48334cfc56fd39daebbae18c7b07f0d2a51dc4", + "sha256:d7d935ffe3309f8279d566113f2d958803154fd090ad07abe0d3458a396db9fd" + ], + "index": "pypi", + "version": "==1.1.1" + }, "certifi": { "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" ], - "version": "==2020.6.20" + "index": "pypi", + "version": "==2020.12.5" }, "cffi": { "hashes": [ - "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff", - "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b", - "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac", - "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0", - "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384", - "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26", - "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6", - "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b", - "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e", - "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd", - "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2", - "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66", - "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc", - "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8", - "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55", - "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4", - "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5", - "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d", - "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78", - "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa", - "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793", - "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f", - "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a", - "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f", - "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30", - "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f", - "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3", - "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c" + "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e", + "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d", + "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a", + "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec", + "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362", + "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668", + "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c", + "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b", + "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06", + "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698", + "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2", + "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c", + "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7", + "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009", + "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03", + "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b", + "sha256:7ef7d4ced6b325e92eb4d3502946c78c5367bc416398d387b39591532536734e", + "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909", + "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53", + "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35", + "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26", + "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b", + "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01", + "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb", + "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293", + "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd", + "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d", + "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3", + "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d", + "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e", + "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca", + "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d", + "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775", + "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375", + "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b", + "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b", + "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f" ], - "version": "==1.14.0" + "index": "pypi", + "version": "==1.14.4" }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], + "index": "pypi", "version": "==3.0.4" }, + "clamd": { + "hashes": [ + "sha256:5c32546b7d1eb00fd6be00a889d79e00fbf980ed082826ccfa369bce3dcff5e7", + "sha256:d82a2fd814684a35a1b31feadafb2e69c8ebde9403613f6bdaa5d877c0f29560" + ], + "index": "pypi", + "version": "==1.0.2" + }, "click": { "hashes": [ "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "index": "pypi", "version": "==7.1.2" }, "click-plugins": { @@ -163,70 +232,104 @@ "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" ], + "index": "pypi", "version": "==1.1.1" }, "colorama": { "hashes": [ - "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", - "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==0.4.3" + "index": "pypi", + "version": "==0.4.4" + }, + "colorclass": { + "hashes": [ + "sha256:b05c2a348dfc1aff2d502527d78a5b7b7e2f85da94a96c5081210d8e9ee8e18b" + ], + "index": "pypi", + "version": "==2.2.0" + }, + "compressed-rtf": { + "hashes": [ + "sha256:c1c827f1d124d24608981a56e8b8691eb1f2a69a78ccad6440e7d92fde1781dd" + ], + "index": "pypi", + "version": "==1.0.6" }, "configparser": { "hashes": [ - "sha256:2ca44140ee259b5e3d8aaf47c79c36a7ab0d5e94d70bd4105c03ede7a20ea5a1", - "sha256:cffc044844040c7ce04e9acd1838b5f2e5fa3170182f6fda4d2ea8b0099dbadd" + "sha256:005c3b102c96f4be9b8f40dafbd4997db003d07d1caa19f37808be8031475f2a", + "sha256:08e8a59ef1817ac4ed810bb8e17d049566dd6e024e7566f6285c756db2bb4ff8" ], - "markers": "python_version >= '3.6'", - "version": "==5.0.0" + "index": "pypi", + "version": "==5.0.1" }, "cryptography": { "hashes": [ - "sha256:091d31c42f444c6f519485ed528d8b451d1a0c7bf30e8ca583a0cac44b8a0df6", - "sha256:18452582a3c85b96014b45686af264563e3e5d99d226589f057ace56196ec78b", - "sha256:1dfa985f62b137909496e7fc182dac687206d8d089dd03eaeb28ae16eec8e7d5", - "sha256:1e4014639d3d73fbc5ceff206049c5a9a849cefd106a49fa7aaaa25cc0ce35cf", - "sha256:22e91636a51170df0ae4dcbd250d318fd28c9f491c4e50b625a49964b24fe46e", - "sha256:3b3eba865ea2754738616f87292b7f29448aec342a7c720956f8083d252bf28b", - "sha256:651448cd2e3a6bc2bb76c3663785133c40d5e1a8c1a9c5429e4354201c6024ae", - "sha256:726086c17f94747cedbee6efa77e99ae170caebeb1116353c6cf0ab67ea6829b", - "sha256:844a76bc04472e5135b909da6aed84360f522ff5dfa47f93e3dd2a0b84a89fa0", - "sha256:88c881dd5a147e08d1bdcf2315c04972381d026cdb803325c03fe2b4a8ed858b", - "sha256:96c080ae7118c10fcbe6229ab43eb8b090fccd31a09ef55f83f690d1ef619a1d", - "sha256:a0c30272fb4ddda5f5ffc1089d7405b7a71b0b0f51993cb4e5dbb4590b2fc229", - "sha256:bb1f0281887d89617b4c68e8db9a2c42b9efebf2702a3c5bf70599421a8623e3", - "sha256:c447cf087cf2dbddc1add6987bbe2f767ed5317adb2d08af940db517dd704365", - "sha256:c4fd17d92e9d55b84707f4fd09992081ba872d1a0c610c109c18e062e06a2e55", - "sha256:d0d5aeaedd29be304848f1c5059074a740fa9f6f26b84c5b63e8b29e73dfc270", - "sha256:daf54a4b07d67ad437ff239c8a4080cfd1cc7213df57d33c97de7b4738048d5e", - "sha256:e993468c859d084d5579e2ebee101de8f5a27ce8e2159959b6673b418fd8c785", - "sha256:f118a95c7480f5be0df8afeb9a11bd199aa20afab7a96bcf20409b411a3a85f0" + "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d", + "sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7", + "sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901", + "sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c", + "sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244", + "sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6", + "sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5", + "sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e", + "sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c", + "sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0", + "sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812", + "sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a", + "sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030", + "sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302" ], - "version": "==2.9.2" + "index": "pypi", + "version": "==3.3.1" + }, + "dataclasses": { + "hashes": [ + "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf", + "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97" + ], + "markers": "python_version < '3.7'", + "version": "==0.8" }, "decorator": { "hashes": [ "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" ], + "index": "pypi", "version": "==4.4.2" }, "deprecated": { "hashes": [ - "sha256:525ba66fb5f90b07169fdd48b6373c18f1ee12728ca277ca44567a367d9d7f74", - "sha256:a766c1dccb30c5f6eb2b203f87edd1d8588847709c78589e1521d769addc8218" + "sha256:471ec32b2755172046e28102cd46c481f21c6036a0ec027521eba8521aa4ef35", + "sha256:924b6921f822b64ec54f49be6700a126bab0640cfafca78f22c9d429ed590560" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2.10" + "index": "pypi", + "version": "==1.2.11" + }, + "dnsdb2": { + "hashes": [ + "sha256:55c9f43eb231ff21f40a69183fdb0220f175519e108bc19d0c098e07532d0dfe" + ], + "index": "pypi", + "version": "==1.1.2" }, "dnspython": { "hashes": [ - "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01", - "sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d" + "sha256:40f563e1f7a7b80dc5a4e76ad75c23da53d62f1e15e6e517293b04e1f84ead7c", + "sha256:861e6e58faa730f9845aaaa9c6c832851fbf89382ac52915a51f89c71accdd31" ], "index": "pypi", - "version": "==1.16.0" + "version": "==1.15.0" + }, + "dnspython3": { + "hashes": [ + "sha256:6eb9504abafb91cb67ed9dc3d3289a3ccc438533b460eccbf77e36c5323100f4" + ], + "index": "pypi", + "version": "==1.15.0" }, "domaintools-api": { "hashes": [ @@ -236,17 +339,42 @@ "index": "pypi", "version": "==0.5.2" }, + "easygui": { + "hashes": [ + "sha256:690658af9fca3f2f2a55f24421045f9b33ca33c877ed5fb61d4b942d8ec335f3", + "sha256:dbc89afbb1aca83830ea4af568eb2491654e16b2706a14d040757fdf1fafbbfe" + ], + "index": "pypi", + "version": "==0.98.1" + }, + "ebcdic": { + "hashes": [ + "sha256:33b4cb729bc2d0bf46cc1847b0e5946897cb8d3f53520c5b9aa5fa98d7e735f1" + ], + "index": "pypi", + "version": "==1.1.1" + }, "enum-compat": { "hashes": [ "sha256:3677daabed56a6f724451d585662253d8fb4e5569845aafa8bb0da36b1a8751e", "sha256:88091b617c7fc3bbbceae50db5958023c48dc40b50520005aa3bf27f8f7ea157" ], + "index": "pypi", "version": "==0.0.3" }, + "extract-msg": { + "hashes": [ + "sha256:7300a50bfa91405a381826f8e22f39458c7322fea1cd660ef699c4157a58be4b", + "sha256:7ce5761911b2caa9f07042efdecfcc9cf4afe524c3efbfd0aa5fa277faa1cc53" + ], + "index": "pypi", + "version": "==0.28.1" + }, "ez-setup": { "hashes": [ "sha256:303c5b17d552d1e3fb0505d80549f8579f557e13d8dc90e5ecef3c07d7f58642" ], + "index": "pypi", "version": "==0.9" }, "ezodf": { @@ -260,7 +388,7 @@ "hashes": [ "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "index": "pypi", "version": "==0.18.2" }, "futures": { @@ -269,21 +397,23 @@ "sha256:51ecb45f0add83c806c68e4b06106f90db260585b25ef2abfcda0bd95c0132fd", "sha256:c4884a65654a7c45435063e14ae85280eb1f111d94e542396717ba9828c4337f" ], + "index": "pypi", "version": "==3.1.1" }, "geoip2": { "hashes": [ - "sha256:5869e987bc54c0d707264fec4710661332cc38d2dca5a7f9bb5362d0308e2ce0", - "sha256:99ec12d2f1271a73a0a4a2b663fe6ce25fd02289c0a6bef05c0a1c3b30ee95a4" + "sha256:57d8d15de2527e0697bbef44fc16812bba709f03a07ef99297bd56c1df3b1efd", + "sha256:707025542ef076bd8fd80e97138bebdb7812527b2a007d141a27ad98b0370fff" ], "index": "pypi", - "version": "==3.0.0" + "version": "==4.1.0" }, "httplib2": { "hashes": [ "sha256:8af66c1c52c7ffe1aa5dc4bcd7c769885254b0756e6e69f953c7f0ab49a70ba3", "sha256:ca2914b015b6247791c4866782fa6042f495b94401a0f0bd3e1d6e0ba2236782" ], + "index": "pypi", "version": "==0.18.1" }, "idna": { @@ -291,88 +421,138 @@ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "index": "pypi", "version": "==2.10" }, "idna-ssl": { "hashes": [ "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c" ], - "markers": "python_version < '3.7'", + "index": "pypi", "version": "==1.1.0" }, + "imapclient": { + "hashes": [ + "sha256:3eeb97b9aa8faab0caa5024d74bfde59408fbd542781246f6960873c7bf0dd01", + "sha256:60ba79758cc9f13ec910d7a3df9acaaf2bb6c458720d9a02ec33a41352fd1b99" + ], + "index": "pypi", + "version": "==2.1.0" + }, + "importlib-metadata": { + "hashes": [ + "sha256:c9db46394197244adf2f0b08ec5bc3cf16757e9590b02af1fca085c16c0d600a", + "sha256:d2d46ef77ffc85cbf7dac7e81dd663fde71c45326131bea8033b9bad42268ebe" + ], + "markers": "python_version < '3.8'", + "version": "==3.10.0" + }, "isodate": { "hashes": [ "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" ], + "index": "pypi", "version": "==0.6.0" }, "jbxapi": { "hashes": [ - "sha256:58eb7d77a52169309e2322ce874c0f00a7900a515d1d0798ff85973cdb2766e3" + "sha256:7cebd7ea73838ffd9e77b6a3bac6e4e1263b8cb4678b0579361b9257db684893" ], "index": "pypi", - "version": "==3.8.0" + "version": "==3.14.0" + }, + "json-log-formatter": { + "hashes": [ + "sha256:ee187c9a80936cbf1259f73573973450fc24b84a4fb54e53eb0dcff86ea1e759" + ], + "index": "pypi", + "version": "==0.3.0" }, "jsonschema": { "hashes": [ "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" ], + "index": "pypi", "version": "==3.2.0" }, + "lark-parser": { + "hashes": [ + "sha256:20bdefdf1b6e9bcb38165ea5cc4f27921a99c6f4c35264a3a953fd60335f1f8c", + "sha256:8b747e1f544dcc2789e3feaddd2a50c6a73bed69d62e9c69760c1e1f7d23495f" + ], + "index": "pypi", + "version": "==0.11.1" + }, "lief": { "hashes": [ - "sha256:276cc63ec12a21bdf01b8d30962692c17499788234f0765247ca7a35872097ec", - "sha256:3e6baaeb52bdc339b5f19688b58fd8d5778b92e50221f920cedfa2bec1f4d5c2", - "sha256:45e5c592b57168c447698381d927eb2386ffdd52afe0c48245f848d4cc7ee05a", - "sha256:6547752b5db105cd41c9fa65d0d7452a4d7541b77ffee716b46246c6d81e172f", - "sha256:83b51e01627b5982662f9550ac1230758aa56945ed86829e4291932d98417da3", - "sha256:895599194ea7495bf304e39317b04df20cccf799fc2751867cc1aa4997cfcdae", - "sha256:8a91cee2568306fe1d2bf84341b459c85368317d01d7105fa49e4f4ede837076", - "sha256:913b36a67707dc2afa72f117bab9856ea3f434f332b04a002a0f9723c8779320", - "sha256:9f604a361a3b1b3ed5fdafed0321c5956cb3b265b5efe2250d1bf8911a80c65b", - "sha256:a487fe7234c04bccd58223dbb79214421176e2629814c7a4a887764cceb5be7c", - "sha256:bc8488fb0661cb436fe4bb4fe947d0f9aa020e9acaed233ccf01ab04d888c68a", - "sha256:bddbf333af62310a10cb738a1df1dc2b140dd9c663b55ba3500c10c249d416d2", - "sha256:cce48d7c97cef85e01e6cfeff55f2068956b5c0257eb9c2d2c6d15e33dd1e4fc", - "sha256:f8b3f66956c56b582b3adc573bf2a938c25fb21c8894b373a113e24c494fc982" + "sha256:0cd2665ff28937755c8acedc2e3fb9de5ba752353e19b51b89297b8be3f63ccb", + "sha256:1a110bd5db41b4fde1a61094658b0366352ed4c0a00b55bde821046a59c2f913", + "sha256:1fb570712fa17ee153aca263ab1f1ec763772ccb46992e415b3dc1c0248466bc", + "sha256:2ee8f9787ea92109f19e5e4d22773042184ac524a3f11399ea5e13d974ac1f05", + "sha256:348ee294567826cad638b7e6cf2ede4ffe03524cd5b6038896f78e5b006d6295", + "sha256:77ba7dd0d48933c0b26c980bda1ab4a7ad3c7031880181fab0b94caad3bc1a4d", + "sha256:8774076dfbcf9b6906be4d9243de4a78fc09d317292251072d460ed1e0eeb96e", + "sha256:93d79a47db9977e6471b21d5efd4e7af4c29c76261d6583141fcf10f6ccd0eba", + "sha256:96d2a8d2310c7accaeb5c6679833c0cd8f0fb6d8c682a5df59d4d868c8881661", + "sha256:9837166402248a4ef29018d498c4eccc11cbc92ee4083da046fa747d3863b43d", + "sha256:9f2bd417090df21548af3a0216f3a02056291348c0826a5ff78e3f3046283978", + "sha256:ad19b9ef5c2a2552a82683bbf92ef3e986b422d552968b378195e356191aa545", + "sha256:afe4d15b519dd7d97732e85b7a2b11154c97a40ce517e1044b5cd5f80074ce36", + "sha256:b0a55424b3ffa08d16bf8ee6e5e9474b0a4b392ca4d94545c16e47e6366e41d4", + "sha256:ccf977099153eaefa351e72e84dfa829334699521ef00434b50647d80de2cc56", + "sha256:d95cf1224c7b311b8d25dbd4de627d28717266e62b9721f1dc4c8427f929a60f", + "sha256:e6ff05e6ebcb9bea8833fcb558d4db3bc2a78031c4792fcac9f9612fa78258b3", + "sha256:e8c66834a0ee9ed1899db165c09ca04aba8dee574de1ccc866d82fbf0c059bb8", + "sha256:f31fde4e7174b4bc9b67ff22fd93fa15fc3faa1592ac669f3addc95d9e66168e", + "sha256:f9b00c396c9f45c5f4cf37c034428ad525d6d7c7d38fc6c49ddc4b558201151b" ], - "version": "==0.10.1" + "index": "pypi", + "version": "==0.11.0" }, "lxml": { "hashes": [ - "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", - "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", - "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", - "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", - "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", - "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", - "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", - "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", - "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", - "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", - "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", - "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", - "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", - "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", - "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", - "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", - "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", - "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", - "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", - "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", - "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", - "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", - "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", - "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", - "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", - "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", - "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" + "sha256:0448576c148c129594d890265b1a83b9cd76fd1f0a6a04620753d9a6bcfd0a4d", + "sha256:127f76864468d6630e1b453d3ffbbd04b024c674f55cf0a30dc2595137892d37", + "sha256:1471cee35eba321827d7d53d104e7b8c593ea3ad376aa2df89533ce8e1b24a01", + "sha256:2363c35637d2d9d6f26f60a208819e7eafc4305ce39dc1d5005eccc4593331c2", + "sha256:2e5cc908fe43fe1aa299e58046ad66981131a66aea3129aac7770c37f590a644", + "sha256:2e6fd1b8acd005bd71e6c94f30c055594bbd0aa02ef51a22bbfa961ab63b2d75", + "sha256:366cb750140f221523fa062d641393092813b81e15d0e25d9f7c6025f910ee80", + "sha256:42ebca24ba2a21065fb546f3e6bd0c58c3fe9ac298f3a320147029a4850f51a2", + "sha256:4e751e77006da34643ab782e4a5cc21ea7b755551db202bc4d3a423b307db780", + "sha256:4fb85c447e288df535b17ebdebf0ec1cf3a3f1a8eba7e79169f4f37af43c6b98", + "sha256:50c348995b47b5a4e330362cf39fc503b4a43b14a91c34c83b955e1805c8e308", + "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf", + "sha256:535f067002b0fd1a4e5296a8f1bf88193080ff992a195e66964ef2a6cfec5388", + "sha256:5be4a2e212bb6aa045e37f7d48e3e1e4b6fd259882ed5a00786f82e8c37ce77d", + "sha256:60a20bfc3bd234d54d49c388950195d23a5583d4108e1a1d47c9eef8d8c042b3", + "sha256:648914abafe67f11be7d93c1a546068f8eff3c5fa938e1f94509e4a5d682b2d8", + "sha256:681d75e1a38a69f1e64ab82fe4b1ed3fd758717bed735fb9aeaa124143f051af", + "sha256:68a5d77e440df94011214b7db907ec8f19e439507a70c958f750c18d88f995d2", + "sha256:69a63f83e88138ab7642d8f61418cf3180a4d8cd13995df87725cb8b893e950e", + "sha256:6e4183800f16f3679076dfa8abf2db3083919d7e30764a069fb66b2b9eff9939", + "sha256:6fd8d5903c2e53f49e99359b063df27fdf7acb89a52b6a12494208bf61345a03", + "sha256:791394449e98243839fa822a637177dd42a95f4883ad3dec2a0ce6ac99fb0a9d", + "sha256:7a7669ff50f41225ca5d6ee0a1ec8413f3a0d8aa2b109f86d540887b7ec0d72a", + "sha256:7e9eac1e526386df7c70ef253b792a0a12dd86d833b1d329e038c7a235dfceb5", + "sha256:7ee8af0b9f7de635c61cdd5b8534b76c52cd03536f29f51151b377f76e214a1a", + "sha256:8246f30ca34dc712ab07e51dc34fea883c00b7ccb0e614651e49da2c49a30711", + "sha256:8c88b599e226994ad4db29d93bc149aa1aff3dc3a4355dd5757569ba78632bdf", + "sha256:923963e989ffbceaa210ac37afc9b906acebe945d2723e9679b643513837b089", + "sha256:94d55bd03d8671686e3f012577d9caa5421a07286dd351dfef64791cf7c6c505", + "sha256:97db258793d193c7b62d4e2586c6ed98d51086e93f9a3af2b2034af01450a74b", + "sha256:a9d6bc8642e2c67db33f1247a77c53476f3a166e09067c0474facb045756087f", + "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc", + "sha256:d8d3d4713f0c28bdc6c806a278d998546e8efc3498949e3ace6e117462ac0a5e", + "sha256:e0bfe9bb028974a481410432dbe1b182e8191d5d40382e5b8ff39cdd2e5c5931", + "sha256:f4822c0660c3754f1a41a655e37cb4dbbc9be3d35b125a37fab6f82d47674ebc", + "sha256:f83d281bb2a6217cd806f4cf0ddded436790e66f393e124dfe9731f6b3fb9afe", + "sha256:fc37870d6716b137e80d19241d0e2cff7a7643b925dfa49b4c8ebd1295eb506e" ], "index": "pypi", - "version": "==4.5.1" + "version": "==4.6.2" }, "maclookup": { "hashes": [ @@ -382,39 +562,74 @@ "index": "pypi", "version": "==1.0.3" }, + "markdownify": { + "hashes": [ + "sha256:30be8340724e706c9e811c27fe8c1542cf74a15b46827924fff5c54b40dd9b0d", + "sha256:a69588194fd76634f0139d6801b820fd652dc5eeba9530e90d323dfdc0155252" + ], + "index": "pypi", + "version": "==0.5.3" + }, "maxminddb": { "hashes": [ - "sha256:f4d28823d9ca23323d113dc7af8db2087aa4f657fafc64ff8f7a8afda871425b" + "sha256:47e86a084dd814fac88c99ea34ba3278a74bc9de5a25f4b815b608798747c7dc" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==1.5.4" + "index": "pypi", + "version": "==2.0.3" }, "misp-modules": { "editable": true, - "path": "." + "file": "file:///usr/local/src/misp-modules" + }, + "msoffcrypto-tool": { + "hashes": [ + "sha256:56a1fe5e58ca417ca8756e8d7224ae599323996da65f81a35273c0f1e2eaf490" + ], + "index": "pypi", + "version": "==4.11.0" }, "multidict": { "hashes": [ - "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", - "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", - "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", - "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", - "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", - "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", - "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", - "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", - "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", - "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", - "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", - "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", - "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", - "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", - "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", - "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", - "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" + "sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a", + "sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93", + "sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632", + "sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656", + "sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79", + "sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7", + "sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d", + "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5", + "sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224", + "sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26", + "sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea", + "sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348", + "sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6", + "sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76", + "sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1", + "sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f", + "sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952", + "sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a", + "sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37", + "sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9", + "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359", + "sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8", + "sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da", + "sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3", + "sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d", + "sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf", + "sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841", + "sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d", + "sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93", + "sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f", + "sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647", + "sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635", + "sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456", + "sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda", + "sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5", + "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281", + "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80" ], - "markers": "python_version >= '3.5'", - "version": "==4.7.6" + "index": "pypi", + "version": "==5.1.0" }, "np": { "hashes": [ @@ -425,35 +640,43 @@ }, "numpy": { "hashes": [ - "sha256:13af0184177469192d80db9bd02619f6fa8b922f9f327e077d6f2a6acb1ce1c0", - "sha256:26a45798ca2a4e168d00de75d4a524abf5907949231512f372b217ede3429e98", - "sha256:26f509450db547e4dfa3ec739419b31edad646d21fb8d0ed0734188b35ff6b27", - "sha256:30a59fb41bb6b8c465ab50d60a1b298d1cd7b85274e71f38af5a75d6c475d2d2", - "sha256:33c623ef9ca5e19e05991f127c1be5aeb1ab5cdf30cb1c5cf3960752e58b599b", - "sha256:356f96c9fbec59974a592452ab6a036cd6f180822a60b529a975c9467fcd5f23", - "sha256:3c40c827d36c6d1c3cf413694d7dc843d50997ebffbc7c87d888a203ed6403a7", - "sha256:4d054f013a1983551254e2379385e359884e5af105e3efe00418977d02f634a7", - "sha256:63d971bb211ad3ca37b2adecdd5365f40f3b741a455beecba70fd0dde8b2a4cb", - "sha256:658624a11f6e1c252b2cd170d94bf28c8f9410acab9f2fd4369e11e1cd4e1aaf", - "sha256:76766cc80d6128750075378d3bb7812cf146415bd29b588616f72c943c00d598", - "sha256:7b57f26e5e6ee2f14f960db46bd58ffdca25ca06dd997729b1b179fddd35f5a3", - "sha256:7b852817800eb02e109ae4a9cef2beda8dd50d98b76b6cfb7b5c0099d27b52d4", - "sha256:8cde829f14bd38f6da7b2954be0f2837043e8b8d7a9110ec5e318ae6bf706610", - "sha256:a2e3a39f43f0ce95204beb8fe0831199542ccab1e0c6e486a0b4947256215632", - "sha256:a86c962e211f37edd61d6e11bb4df7eddc4a519a38a856e20a6498c319efa6b0", - "sha256:a8705c5073fe3fcc297fb8e0b31aa794e05af6a329e81b7ca4ffecab7f2b95ef", - "sha256:b6aaeadf1e4866ca0fdf7bb4eed25e521ae21a7947c59f78154b24fc7abbe1dd", - "sha256:be62aeff8f2f054eff7725f502f6228298891fd648dc2630e03e44bf63e8cee0", - "sha256:c2edbb783c841e36ca0fa159f0ae97a88ce8137fb3a6cd82eae77349ba4b607b", - "sha256:cbe326f6d364375a8e5a8ccb7e9cd73f4b2f6dc3b2ed205633a0db8243e2a96a", - "sha256:d34fbb98ad0d6b563b95de852a284074514331e6b9da0a9fc894fb1cdae7a79e", - "sha256:d97a86937cf9970453c3b62abb55a6475f173347b4cde7f8dcdb48c8e1b9952d", - "sha256:dd53d7c4a69e766e4900f29db5872f5824a06827d594427cf1a4aa542818b796", - "sha256:df1889701e2dfd8ba4dc9b1a010f0a60950077fb5242bb92c8b5c7f1a6f2668a", - "sha256:fa1fe75b4a9e18b66ae7f0b122543c42debcf800aaafa0212aaff3ad273c2596" + "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94", + "sha256:06fab248a088e439402141ea04f0fffb203723148f6ee791e9c75b3e9e82f080", + "sha256:0eef32ca3132a48e43f6a0f5a82cb508f22ce5a3d6f67a8329c81c8e226d3f6e", + "sha256:1ded4fce9cfaaf24e7a0ab51b7a87be9038ea1ace7f34b841fe3b6894c721d1c", + "sha256:2e55195bc1c6b705bfd8ad6f288b38b11b1af32f3c8289d6c50d47f950c12e76", + "sha256:2ea52bd92ab9f768cc64a4c3ef8f4b2580a17af0a5436f6126b08efbd1838371", + "sha256:36674959eed6957e61f11c912f71e78857a8d0604171dfd9ce9ad5cbf41c511c", + "sha256:384ec0463d1c2671170901994aeb6dce126de0a95ccc3976c43b0038a37329c2", + "sha256:39b70c19ec771805081578cc936bbe95336798b7edf4732ed102e7a43ec5c07a", + "sha256:400580cbd3cff6ffa6293df2278c75aef2d58d8d93d3c5614cd67981dae68ceb", + "sha256:43d4c81d5ffdff6bae58d66a3cd7f54a7acd9a0e7b18d97abb255defc09e3140", + "sha256:50a4a0ad0111cc1b71fa32dedd05fa239f7fb5a43a40663269bb5dc7877cfd28", + "sha256:603aa0706be710eea8884af807b1b3bc9fb2e49b9f4da439e76000f3b3c6ff0f", + "sha256:6149a185cece5ee78d1d196938b2a8f9d09f5a5ebfbba66969302a778d5ddd1d", + "sha256:759e4095edc3c1b3ac031f34d9459fa781777a93ccc633a472a5468587a190ff", + "sha256:7fb43004bce0ca31d8f13a6eb5e943fa73371381e53f7074ed21a4cb786c32f8", + "sha256:811daee36a58dc79cf3d8bdd4a490e4277d0e4b7d103a001a4e73ddb48e7e6aa", + "sha256:8b5e972b43c8fc27d56550b4120fe6257fdc15f9301914380b27f74856299fea", + "sha256:99abf4f353c3d1a0c7a5f27699482c987cf663b1eac20db59b8c7b061eabd7fc", + "sha256:a0d53e51a6cb6f0d9082decb7a4cb6dfb33055308c4c44f53103c073f649af73", + "sha256:a12ff4c8ddfee61f90a1633a4c4afd3f7bcb32b11c52026c92a12e1325922d0d", + "sha256:a4646724fba402aa7504cd48b4b50e783296b5e10a524c7a6da62e4a8ac9698d", + "sha256:a76f502430dd98d7546e1ea2250a7360c065a5fdea52b2dffe8ae7180909b6f4", + "sha256:a9d17f2be3b427fbb2bce61e596cf555d6f8a56c222bd2ca148baeeb5e5c783c", + "sha256:ab83f24d5c52d60dbc8cd0528759532736b56db58adaa7b5f1f76ad551416a1e", + "sha256:aeb9ed923be74e659984e321f609b9ba54a48354bfd168d21a2b072ed1e833ea", + "sha256:c843b3f50d1ab7361ca4f0b3639bf691569493a56808a0b0c54a051d260b7dbd", + "sha256:cae865b1cae1ec2663d8ea56ef6ff185bad091a5e33ebbadd98de2cfa3fa668f", + "sha256:cc6bd4fd593cb261332568485e20a0712883cf631f6f5e8e86a52caa8b2b50ff", + "sha256:cf2402002d3d9f91c8b01e66fbb436a4ed01c6498fffed0e4c7566da1d40ee1e", + "sha256:d051ec1c64b85ecc69531e1137bb9751c6830772ee5c1c426dbcfe98ef5788d7", + "sha256:d6631f2e867676b13026e2846180e2c13c1e11289d67da08d71cacb2cd93d4aa", + "sha256:dbd18bcf4889b720ba13a27ec2f2aac1981bd41203b3a3b27ba7a33f88ae4827", + "sha256:df609c82f18c5b9f6cb97271f03315ff0dbe481a2a02e56aeb1b1a985ce38e60" ], - "markers": "python_version >= '3.6'", - "version": "==1.19.0" + "index": "pypi", + "version": "==1.19.5" }, "oauth2": { "hashes": [ @@ -468,53 +691,80 @@ "git": "https://github.com/cartertemm/ODTReader.git/", "ref": "49d6938693f6faa3ff09998f86dba551ae3a996b" }, - "opencv-python": { + "olefile": { "hashes": [ - "sha256:068928b9907b3d3acd53b129062557d6b0b8b324bfade77f028dbe4dfe482bf2", - "sha256:0e7c91718351449877c2d4141abd64eee1f9c8701bcfaf4e8627bd023e303368", - "sha256:1ab92d807427641ec45d28d5907426aa06b4ffd19c5b794729c74d91cd95090e", - "sha256:31d634dea1b47c231b88d384f90605c598214d0c596443c9bb808e11761829f5", - "sha256:5fdfc0bed37315f27d30ae5ae9bad47ec0a0a28c323739d39c8177b7e0929238", - "sha256:6fa8fac14dd5af4819d475f74af12d65fbbfa391d3110c3a972934a5e6507c24", - "sha256:78cc89ebc808886eb190626ee71ab65e47f374121975f86e4d5f7c0e3ce6bed9", - "sha256:7c7ba11720d01cb572b4b6945d115cb103462c0a28996b44d4e540d06e6a90fd", - "sha256:a37ee82f1b8ed4b4645619c504311e71ce845b78f40055e78d71add5fab7da82", - "sha256:aa3ca1f54054e1c6439fdf1edafa2a2b940a9eaac04a7b422a1cba9b2d7b9690", - "sha256:b9de3dd956574662712da8e285f0f54327959a4e95b96a2847d3c3f5ee7b96e2", - "sha256:c0087b428cef9a32d977390656d91b02245e0e91f909870492df7e39202645dd", - "sha256:d87e506ab205799727f0efa34b3888949bf029a3ada5eb000ff632606370ca6e", - "sha256:d8a55585631f9c9eca4b1a996e9732ae023169cf2f46f69e4518d67d96198226", - "sha256:dcb8da8c5ebaa6360c8555547a4c7beb6cd983dd95ba895bb78b86cc8cf3de2b", - "sha256:e2206bb8c17c0f212f1f356d82d72dd090ff4651994034416da9bf0c29732825", - "sha256:e3c57d6579e5bf85f564d6d48d8ee89868b92879a9232b9975d072c346625e92", - "sha256:ef89cbf332b9a735d8a82e9ff79cc743eeeb775ad1cd7100bc2aa2429b496f07", - "sha256:f45c1c3cdda1857bedd4dfe0bbd49c9419af0cc57f33490341edeae97d18f037", - "sha256:fb3c855347310788e4286b867997be354c55535597966ed5dac876d9166013a4" + "sha256:133b031eaf8fd2c9399b78b8bc5b8fcbe4c31e85295749bb17a87cba8f3c3964" ], "index": "pypi", - "version": "==4.2.0.34" + "version": "==0.46" + }, + "oletools": { + "hashes": [ + "sha256:8481cd60352399e15e9290ac57862a65952e9c83e3526ba833991a5c78f5cca1" + ], + "index": "pypi", + "version": "==0.56" + }, + "opencv-python": { + "hashes": [ + "sha256:30edebc81b260bcfeb760b3600c367c5261dfb2fe41e5d1408d5357d0867b40d", + "sha256:32dee1c9fd3e31e28edef7b56f868e2b40e280b7062304f9fb8a14dbc51547d5", + "sha256:4982fa8ccc38310a2bd93e06334ba090b12b6aff2f6fcb8ff9613e3c9bc48f48", + "sha256:5172cb37dfd8a0b4945b071a493eb36e5f17675a160637fa380f9c1d9d80535c", + "sha256:6d8434a45e8f75c4da5fd0068ce001f4f8e35771cc851d746d4721eeaf517e25", + "sha256:78a6db8467639383caedf1d111da3510a4ee1a0aacf2117821cae2ee8f92ce37", + "sha256:9646875c501788b1b098f282d777b667d6da69801739504f1b2fd1268970d1da", + "sha256:9c77d508e6822f1f40c727d21b822d017622d8305dce7eccf0ab06caac16d5c6", + "sha256:a1dfa0486db367594510c0c799ec7481247dc86e651b69008806d875ab731471", + "sha256:b2b9ac86aec5f2dd531545cebdea1a1ef4f81ef1fb1760d78b4725f9575504f9", + "sha256:bcb27773cfd5340b2b599b303d9f5499838ef4780c20c038f6030175408c64df", + "sha256:c0503bfaa2b7b743d6ff5d81f1dd8428dbf4c33e7e4f836456d11be20c2e7721", + "sha256:c1159d91f29a85c3333edef6ca420284566d9bcdae46dda2fe7282515b48c8b6", + "sha256:c4ea4f8b217f3e8be6247fc0787fb81797d85202c722523f41070124a7a621c7", + "sha256:c8cc1f5ff3c352ebe756119014c4e4ec7ae5ac536d1f66b0316667ced37637c8", + "sha256:d16144c435b816c5536d5ff012c1a2b7e93155017db7103942ff7efb98c4df1f", + "sha256:d8aefcb30b71064dbbaa2b0ace161a36464c29375a83998fbda39a1d1740f942", + "sha256:e27d062fa1098d90f48b6c047351c89816492a08906a021c973ce510b04a7b9d", + "sha256:e2c17714da59d9d516ceef0450766ff9557ee232d62f702665af905193557582", + "sha256:e38fbd7b2db03204ec09930609b7313d6b6d2b271c8fe2c0aa271fa69b726a1b", + "sha256:e77d0feaff37326f62b127098264e2a7099deb476e38432b1083ce11cdedf560", + "sha256:ebe83901971a6755512424c4fe9f63341cca501b7c497bf608dd38ee31ba3f4c", + "sha256:efac9893d9e21cfb599828801c755ecde8f1e657f05ec6f002efe19422456d5a", + "sha256:fc1472b825d26c8a4f1cfb172a90c3cc47733e4af7522276c1c2efe8f6006a8b", + "sha256:ffc75c614b8dc3d8102f3ba15dafd6ec0400c7ffa71a91953d41511964ee50e0" + ], + "index": "pypi", + "version": "==4.5.1.48" }, "pandas": { "hashes": [ - "sha256:02f1e8f71cd994ed7fcb9a35b6ddddeb4314822a0e09a9c5b2d278f8cb5d4096", - "sha256:13f75fb18486759da3ff40f5345d9dd20e7d78f2a39c5884d013456cec9876f0", - "sha256:35b670b0abcfed7cad76f2834041dcf7ae47fd9b22b63622d67cdc933d79f453", - "sha256:4c73f373b0800eb3062ffd13d4a7a2a6d522792fa6eb204d67a4fad0a40f03dc", - "sha256:5759edf0b686b6f25a5d4a447ea588983a33afc8a0081a0954184a4a87fd0dd7", - "sha256:5a7cf6044467c1356b2b49ef69e50bf4d231e773c3ca0558807cdba56b76820b", - "sha256:69c5d920a0b2a9838e677f78f4dde506b95ea8e4d30da25859db6469ded84fa8", - "sha256:8778a5cc5a8437a561e3276b85367412e10ae9fff07db1eed986e427d9a674f8", - "sha256:9871ef5ee17f388f1cb35f76dc6106d40cb8165c562d573470672f4cdefa59ef", - "sha256:9c31d52f1a7dd2bb4681d9f62646c7aa554f19e8e9addc17e8b1b20011d7522d", - "sha256:ab8173a8efe5418bbe50e43f321994ac6673afc5c7c4839014cf6401bbdd0705", - "sha256:ae961f1f0e270f1e4e2273f6a539b2ea33248e0e3a11ffb479d757918a5e03a9", - "sha256:b3c4f93fcb6e97d993bf87cdd917883b7dab7d20c627699f360a8fb49e9e0b91", - "sha256:c9410ce8a3dee77653bc0684cfa1535a7f9c291663bd7ad79e39f5ab58f67ab3", - "sha256:f69e0f7b7c09f1f612b1f8f59e2df72faa8a6b41c5a436dde5b615aaf948f107", - "sha256:faa42a78d1350b02a7d2f0dbe3c80791cf785663d6997891549d0f86dc49125e" + "sha256:0a643bae4283a37732ddfcecab3f62dd082996021b980f580903f4e8e01b3c5b", + "sha256:0de3ddb414d30798cbf56e642d82cac30a80223ad6fe484d66c0ce01a84d6f2f", + "sha256:19a2148a1d02791352e9fa637899a78e371a3516ac6da5c4edc718f60cbae648", + "sha256:21b5a2b033380adbdd36b3116faaf9a4663e375325831dac1b519a44f9e439bb", + "sha256:24c7f8d4aee71bfa6401faeba367dd654f696a77151a8a28bc2013f7ced4af98", + "sha256:26fa92d3ac743a149a31b21d6f4337b0594b6302ea5575b37af9ca9611e8981a", + "sha256:2860a97cbb25444ffc0088b457da0a79dc79f9c601238a3e0644312fcc14bf11", + "sha256:2b1c6cd28a0dfda75c7b5957363333f01d370936e4c6276b7b8e696dd500582a", + "sha256:2c2f7c670ea4e60318e4b7e474d56447cf0c7d83b3c2a5405a0dbb2600b9c48e", + "sha256:3be7a7a0ca71a2640e81d9276f526bca63505850add10206d0da2e8a0a325dae", + "sha256:4c62e94d5d49db116bef1bd5c2486723a292d79409fc9abd51adf9e05329101d", + "sha256:5008374ebb990dad9ed48b0f5d0038124c73748f5384cc8c46904dace27082d9", + "sha256:5447ea7af4005b0daf695a316a423b96374c9c73ffbd4533209c5ddc369e644b", + "sha256:573fba5b05bf2c69271a32e52399c8de599e4a15ab7cec47d3b9c904125ab788", + "sha256:5a780260afc88268a9d3ac3511d8f494fdcf637eece62fb9eb656a63d53eb7ca", + "sha256:70865f96bb38fec46f7ebd66d4b5cfd0aa6b842073f298d621385ae3898d28b5", + "sha256:731568be71fba1e13cae212c362f3d2ca8932e83cb1b85e3f1b4dd77d019254a", + "sha256:b61080750d19a0122469ab59b087380721d6b72a4e7d962e4d7e63e0c4504814", + "sha256:bf23a3b54d128b50f4f9d4675b3c1857a688cc6731a32f931837d72effb2698d", + "sha256:c16d59c15d946111d2716856dd5479221c9e4f2f5c7bc2d617f39d870031e086", + "sha256:c61c043aafb69329d0f961b19faa30b1dab709dd34c9388143fc55680059e55a", + "sha256:c94ff2780a1fd89f190390130d6d36173ca59fcfb3fe0ff596f9a56518191ccb", + "sha256:edda9bacc3843dfbeebaf7a701763e68e741b08fccb889c003b0a52f0ee95782", + "sha256:f10fc41ee3c75a474d3bdf68d396f10782d013d7f67db99c0efbfd0acb99701b" ], "index": "pypi", - "version": "==1.0.5" + "version": "==1.1.5" }, "pandas-ods-reader": { "hashes": [ @@ -533,68 +783,100 @@ "index": "pypi", "version": "==1.0.31" }, - "pdftotext": { + "pcodedmp": { "hashes": [ - "sha256:d37864049581fb13cdcf7b23d4ea23dac7ca2e9c646e8ecac1a39275ab1cae03" + "sha256:025f8c809a126f45a082ffa820893e6a8d990d9d7ddb68694b5a9f0a6dbcd955", + "sha256:4441f7c0ab4cbda27bd4668db3b14f36261d86e5059ce06c0828602cbe1c4278" ], "index": "pypi", - "version": "==2.1.4" + "version": "==1.2.6" + }, + "pdftotext": { + "hashes": [ + "sha256:98aeb8b07a4127e1a30223bd933ef080bbd29aa88f801717ca6c5618380b8aa6" + ], + "index": "pypi", + "version": "==2.1.5" }, "pillow": { "hashes": [ - "sha256:0295442429645fa16d05bd567ef5cff178482439c9aad0411d3f0ce9b88b3a6f", - "sha256:06aba4169e78c439d528fdeb34762c3b61a70813527a2c57f0540541e9f433a8", - "sha256:09d7f9e64289cb40c2c8d7ad674b2ed6105f55dc3b09aa8e4918e20a0311e7ad", - "sha256:0a80dd307a5d8440b0a08bd7b81617e04d870e40a3e46a32d9c246e54705e86f", - "sha256:1ca594126d3c4def54babee699c055a913efb01e106c309fa6b04405d474d5ae", - "sha256:25930fadde8019f374400f7986e8404c8b781ce519da27792cbe46eabec00c4d", - "sha256:431b15cffbf949e89df2f7b48528be18b78bfa5177cb3036284a5508159492b5", - "sha256:52125833b070791fcb5710fabc640fc1df07d087fc0c0f02d3661f76c23c5b8b", - "sha256:5e51ee2b8114def244384eda1c82b10e307ad9778dac5c83fb0943775a653cd8", - "sha256:612cfda94e9c8346f239bf1a4b082fdd5c8143cf82d685ba2dba76e7adeeb233", - "sha256:6d7741e65835716ceea0fd13a7d0192961212fd59e741a46bbed7a473c634ed6", - "sha256:6edb5446f44d901e8683ffb25ebdfc26988ee813da3bf91e12252b57ac163727", - "sha256:725aa6cfc66ce2857d585f06e9519a1cc0ef6d13f186ff3447ab6dff0a09bc7f", - "sha256:8dad18b69f710bf3a001d2bf3afab7c432785d94fcf819c16b5207b1cfd17d38", - "sha256:94cf49723928eb6070a892cb39d6c156f7b5a2db4e8971cb958f7b6b104fb4c4", - "sha256:97f9e7953a77d5a70f49b9a48da7776dc51e9b738151b22dacf101641594a626", - "sha256:9ad7f865eebde135d526bb3163d0b23ffff365cf87e767c649550964ad72785d", - "sha256:a060cf8aa332052df2158e5a119303965be92c3da6f2d93b6878f0ebca80b2f6", - "sha256:c79f9c5fb846285f943aafeafda3358992d64f0ef58566e23484132ecd8d7d63", - "sha256:c92302a33138409e8f1ad16731568c55c9053eee71bb05b6b744067e1b62380f", - "sha256:d08b23fdb388c0715990cbc06866db554e1822c4bdcf6d4166cf30ac82df8c41", - "sha256:d350f0f2c2421e65fbc62690f26b59b0bcda1b614beb318c81e38647e0f673a1", - "sha256:ec29604081f10f16a7aea809ad42e27764188fc258b02259a03a8ff7ded3808d", - "sha256:edf31f1150778abd4322444c393ab9c7bd2af271dd4dafb4208fb613b1f3cdc9", - "sha256:f7e30c27477dffc3e85c2463b3e649f751789e0f6c8456099eea7ddd53be4a8a", - "sha256:ffe538682dc19cc542ae7c3e504fdf54ca7f86fb8a135e59dd6bc8627eae6cce" + "sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6", + "sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded", + "sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865", + "sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174", + "sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032", + "sha256:3de6b2ee4f78c6b3d89d184ade5d8fa68af0848f9b6b6da2b9ab7943ec46971a", + "sha256:47c0d93ee9c8b181f353dbead6530b26980fe4f5485aa18be8f1fd3c3cbc685e", + "sha256:5e2fe3bb2363b862671eba632537cd3a823847db4d98be95690b7e382f3d6378", + "sha256:604815c55fd92e735f9738f65dabf4edc3e79f88541c221d292faec1904a4b17", + "sha256:6c5275bd82711cd3dcd0af8ce0bb99113ae8911fc2952805f1d012de7d600a4c", + "sha256:731ca5aabe9085160cf68b2dbef95fc1991015bc0a3a6ea46a371ab88f3d0913", + "sha256:7612520e5e1a371d77e1d1ca3a3ee6227eef00d0a9cddb4ef7ecb0b7396eddf7", + "sha256:7916cbc94f1c6b1301ac04510d0881b9e9feb20ae34094d3615a8a7c3db0dcc0", + "sha256:81c3fa9a75d9f1afafdb916d5995633f319db09bd773cb56b8e39f1e98d90820", + "sha256:887668e792b7edbfb1d3c9d8b5d8c859269a0f0eba4dda562adb95500f60dbba", + "sha256:93a473b53cc6e0b3ce6bf51b1b95b7b1e7e6084be3a07e40f79b42e83503fbf2", + "sha256:96d4dc103d1a0fa6d47c6c55a47de5f5dafd5ef0114fa10c85a1fd8e0216284b", + "sha256:a3d3e086474ef12ef13d42e5f9b7bbf09d39cf6bd4940f982263d6954b13f6a9", + "sha256:b02a0b9f332086657852b1f7cb380f6a42403a6d9c42a4c34a561aa4530d5234", + "sha256:b09e10ec453de97f9a23a5aa5e30b334195e8d2ddd1ce76cc32e52ba63c8b31d", + "sha256:b6f00ad5ebe846cc91763b1d0c6d30a8042e02b2316e27b05de04fa6ec831ec5", + "sha256:bba80df38cfc17f490ec651c73bb37cd896bc2400cfba27d078c2135223c1206", + "sha256:c3d911614b008e8a576b8e5303e3db29224b455d3d66d1b2848ba6ca83f9ece9", + "sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8", + "sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59", + "sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d", + "sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7", + "sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a", + "sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0", + "sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b", + "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d", + "sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae" ], "index": "pypi", - "version": "==7.2.0" + "version": "==8.1.0" }, "progressbar2": { "hashes": [ - "sha256:13f228cf357f94cdef933c91c1e771e52e1b1931dbae48267be8fcdc2ae2ce36", - "sha256:27abf038efe5b1b5dd91ecc5f704bc88683c1e2a0b2c0fee04de80a648634a0c" + "sha256:ef72be284e7f2b61ac0894b44165926f13f5d995b2bf3cd8a8dedc6224b255a7", + "sha256:fe2738e7ecb7df52ad76307fe610c460c52b50f5335fd26c3ab80ff7655ba1e0" ], - "version": "==3.51.4" + "index": "pypi", + "version": "==3.53.1" }, "psutil": { "hashes": [ - "sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058", - "sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953", - "sha256:60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4", - "sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e", - "sha256:73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f", - "sha256:75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38", - "sha256:a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e", - "sha256:d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8", - "sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26", - "sha256:e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5", - "sha256:f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310" + "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64", + "sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131", + "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c", + "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6", + "sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023", + "sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df", + "sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394", + "sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4", + "sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b", + "sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2", + "sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d", + "sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65", + "sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d", + "sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef", + "sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7", + "sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60", + "sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6", + "sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8", + "sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b", + "sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d", + "sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac", + "sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935", + "sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d", + "sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28", + "sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876", + "sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0", + "sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3", + "sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==5.7.0" + "index": "pypi", + "version": "==5.8.0" }, "pybgpranking": { "editable": true, @@ -607,85 +889,100 @@ "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "index": "pypi", "version": "==2.20" }, "pycryptodome": { "hashes": [ - "sha256:02e51e1d5828d58f154896ddfd003e2e7584869c275e5acbe290443575370fba", - "sha256:03d5cca8618620f45fd40f827423f82b86b3a202c8d44108601b0f5f56b04299", - "sha256:0e24171cf01021bc5dc17d6a9d4f33a048f09d62cc3f62541e95ef104588bda4", - "sha256:132a56abba24e2e06a479d8e5db7a48271a73a215f605017bbd476d31f8e71c1", - "sha256:1e655746f539421d923fd48df8f6f40b3443d80b75532501c0085b64afed9df5", - "sha256:2b998dc45ef5f4e5cf5248a6edfcd8d8e9fb5e35df8e4259b13a1b10eda7b16b", - "sha256:360955eece2cd0fa694a708d10303c6abd7b39614fa2547b6bd245da76198beb", - "sha256:39ef9fb52d6ec7728fce1f1693cb99d60ce302aeebd59bcedea70ca3203fda60", - "sha256:4350a42028240c344ee855f032c7d4ad6ff4f813bfbe7121547b7dc579ecc876", - "sha256:50348edd283afdccddc0938cdc674484533912ba8a99a27c7bfebb75030aa856", - "sha256:54bdedd28476dea8a3cd86cb67c0df1f0e3d71cae8022354b0f879c41a3d27b2", - "sha256:55eb61aca2c883db770999f50d091ff7c14016f2769ad7bca3d9b75d1d7c1b68", - "sha256:6276478ada411aca97c0d5104916354b3d740d368407912722bd4d11aa9ee4c2", - "sha256:67dcad1b8b201308586a8ca2ffe89df1e4f731d5a4cdd0610cc4ea790351c739", - "sha256:709b9f144d23e290b9863121d1ace14a72e01f66ea9c903fbdc690520dfdfcf0", - "sha256:8063a712fba642f78d3c506b0896846601b6de7f5c3d534e388ad0cc07f5a149", - "sha256:80d57177a0b7c14d4594c62bbb47fe2f6309ad3b0a34348a291d570925c97a82", - "sha256:a207231a52426de3ff20f5608f0687261a3329d97a036c51f7d4c606a6f30c23", - "sha256:abc2e126c9490e58a36a0f83516479e781d83adfb134576a5cbe5c6af2a3e93c", - "sha256:b56638d58a3a4be13229c6a815cd448f9e3ce40c00880a5398471b42ee86f50e", - "sha256:bcd5b8416e73e4b0d48afba3704d8c826414764dafaed7a1a93c442188d90ccc", - "sha256:bec2bcdf7c9ce7f04d718e51887f3b05dc5c1cfaf5d2c2e9065ecddd1b2f6c9a", - "sha256:c8bf40cf6e281a4378e25846924327e728a887e8bf0ee83b2604a0f4b61692e8", - "sha256:d8074c8448cfd0705dfa71ca333277fce9786d0b9cac75d120545de6253f996a", - "sha256:dd302b6ae3965afeb5ef1b0d92486f986c0e65183cd7835973f0b593800590e6", - "sha256:de6e1cd75677423ff64712c337521e62e3a7a4fc84caabbd93207752e831a85a", - "sha256:ef39c98d9b8c0736d91937d193653e47c3b19ddf4fc3bccdc5e09aaa4b0c5d21", - "sha256:f521178e5a991ffd04182ed08f552daca1affcb826aeda0e1945cd989a9d4345", - "sha256:f78a68c2c820e4731e510a2df3eef0322f24fde1781ced970bf497b6c7d92982", - "sha256:fbe65d5cfe04ff2f7684160d50f5118bdefb01e3af4718eeb618bfed40f19d94" + "sha256:19cb674df6c74a14b8b408aa30ba8a89bd1c01e23505100fb45f930fbf0ed0d9", + "sha256:1cfdb92dca388e27e732caa72a1cc624520fe93752a665c3b6cd8f1a91b34916", + "sha256:21ef416aa52802d22b9c11598d4e5352285bd9d6b5d868cde3e6bf2b22b4ebfb", + "sha256:27397aee992af69d07502126561d851ba3845aa808f0e55c71ad0efa264dd7d4", + "sha256:28f75e58d02019a7edc7d4135203d2501dfc47256d175c72c9798f9a129a49a7", + "sha256:2a68df525b387201a43b27b879ce8c08948a430e883a756d6c9e3acdaa7d7bd8", + "sha256:411745c6dce4eff918906eebcde78771d44795d747e194462abb120d2e537cd9", + "sha256:46e96aeb8a9ca8b1edf9b1fd0af4bf6afcf3f1ca7fa35529f5d60b98f3e4e959", + "sha256:4ed27951b0a17afd287299e2206a339b5b6d12de9321e1a1575261ef9c4a851b", + "sha256:50826b49fbca348a61529693b0031cdb782c39060fb9dca5ac5dff858159dc5a", + "sha256:5598dc6c9dbfe882904e54584322893eff185b98960bbe2cdaaa20e8a437b6e5", + "sha256:5c3c4865730dfb0263f822b966d6d58429d8b1e560d1ddae37685fd9e7c63161", + "sha256:5f19e6ef750f677d924d9c7141f54bade3cd56695bbfd8a9ef15d0378557dfe4", + "sha256:60febcf5baf70c566d9d9351c47fbd8321da9a4edf2eff45c4c31c86164ca794", + "sha256:62c488a21c253dadc9f731a32f0ac61e4e436d81a1ea6f7d1d9146ed4d20d6bd", + "sha256:6d3baaf82681cfb1a842f1c8f77beac791ceedd99af911e4f5fabec32bae2259", + "sha256:6e4227849e4231a3f5b35ea5bdedf9a82b3883500e5624f00a19156e9a9ef861", + "sha256:6e89bb3826e6f84501e8e3b205c22595d0c5492c2f271cbb9ee1c48eb1866645", + "sha256:70d807d11d508433daf96244ec1c64e55039e8a35931fc5ea9eee94dbe3cb6b5", + "sha256:76b1a34d74bb2c91bce460cdc74d1347592045627a955e9a252554481c17c52f", + "sha256:7798e73225a699651888489fbb1dbc565e03a509942a8ce6194bbe6fb582a41f", + "sha256:834b790bbb6bd18956f625af4004d9c15eed12d5186d8e57851454ae76d52215", + "sha256:843e5f10ecdf9d307032b8b91afe9da1d6ed5bb89d0bbec5c8dcb4ba44008e11", + "sha256:8f9f84059039b672a5a705b3c5aa21747867bacc30a72e28bf0d147cc8ef85ed", + "sha256:9000877383e2189dafd1b2fc68c6c726eca9a3cfb6d68148fbb72ccf651959b6", + "sha256:910e202a557e1131b1c1b3f17a63914d57aac55cf9fb9b51644962841c3995c4", + "sha256:946399d15eccebafc8ce0257fc4caffe383c75e6b0633509bd011e357368306c", + "sha256:a199e9ca46fc6e999e5f47fce342af4b56c7de85fae893c69ab6aa17531fb1e1", + "sha256:a3d8a9efa213be8232c59cdc6b65600276508e375e0a119d710826248fd18d37", + "sha256:a4599c0ca0fc027c780c1c45ed996d5bef03e571470b7b1c7171ec1e1a90914c", + "sha256:b17b0ad9faee14d6318f965d58d323b0b37247e1e0c9c40c23504c00f4af881e", + "sha256:b4e6b269a8ddaede774e5c3adbef6bf452ee144e6db8a716d23694953348cd86", + "sha256:b68794fba45bdb367eeb71249c26d23e61167510a1d0c3d6cf0f2f14636e62ee", + "sha256:b830fae2a46536ee830599c3c4af114f5228f31e54adac370767616a701a99dc", + "sha256:d7ec2bd8f57c559dd24e71891c51c25266a8deb66fc5f02cc97c7fb593d1780a", + "sha256:e15bde67ccb7d4417f627dd16ffe2f5a4c2941ce5278444e884cb26d73ecbc61", + "sha256:eb01f9997e4d6a8ec8a1ad1f676ba5a362781ff64e8189fe2985258ba9cb9706", + "sha256:f381036287c25d9809a08224ce4d012b7b7d50b6ada3ddbc3bc6f1f659365120", + "sha256:faa682c404c218e8788c3126c9a4b8fbcc54dc245b5b6e8ea5b46f3b63bd0c84" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.9.8" + "index": "pypi", + "version": "==3.9.9" }, "pycryptodomex": { "hashes": [ - "sha256:06f5a458624c9b0e04c0086c7f84bcc578567dab0ddc816e0476b3057b18339f", - "sha256:1714675fb4ac29a26ced38ca22eb8ffd923ac851b7a6140563863194d7158422", - "sha256:17272d06e4b2f6455ee2cbe93e8eb50d9450a5dc6223d06862ee1ea5d1235861", - "sha256:2199708ebeed4b82eb45b10e1754292677f5a0df7d627ee91ea01290b9bab7e6", - "sha256:2275a663c9e744ee4eace816ef2d446b3060554c5773a92fbc79b05bf47debda", - "sha256:2710fc8d83b3352b370db932b3710033b9d630b970ff5aaa3e7458b5336e3b32", - "sha256:35b9c9177a9fe7288b19dd41554c9c8ca1063deb426dd5a02e7e2a7416b6bd11", - "sha256:3caa32cf807422adf33c10c88c22e9e2e08b9d9d042f12e1e25fe23113dd618f", - "sha256:48cc2cfc251f04a6142badeb666d1ff49ca6fdfc303fd72579f62b768aaa52b9", - "sha256:4ae6379350a09339109e9b6f419bb2c3f03d3e441f4b0f5b8ca699d47cc9ff7e", - "sha256:4e0b27697fa1621c6d3d3b4edeec723c2e841285de6a8d378c1962da77b349be", - "sha256:58e19560814dabf5d788b95a13f6b98279cf41a49b1e49ee6cf6c79a57adb4c9", - "sha256:8044eae59301dd392fbb4a7c5d64e1aea8ef0be2540549807ecbe703d6233d68", - "sha256:89be1bf55e50116fe7e493a7c0c483099770dd7f81b87ac8d04a43b1a203e259", - "sha256:8fcdda24dddf47f716400d54fc7f75cadaaba1dd47cc127e59d752c9c0fc3c48", - "sha256:914fbb18e29c54585e6aa39d300385f90d0fa3b3cc02ed829b08f95c1acf60c2", - "sha256:93a75d1acd54efed314b82c952b39eac96ce98d241ad7431547442e5c56138aa", - "sha256:9fd758e5e2fe02d57860b85da34a1a1e7037155c4eadc2326fc7af02f9cae214", - "sha256:a2bc4e1a2e6ca3a18b2e0be6131a23af76fecb37990c159df6edc7da6df913e3", - "sha256:a2ee8ba99d33e1a434fcd27d7d0aa7964163efeee0730fe2efc9d60edae1fc71", - "sha256:b2d756620078570d3f940c84bc94dd30aa362b795cce8b2723300a8800b87f1c", - "sha256:c0d085c8187a1e4d3402f626c9e438b5861151ab132d8761d9c5ce6491a87761", - "sha256:c990f2c58f7c67688e9e86e6557ed05952669ff6f1343e77b459007d85f7df00", - "sha256:ccbbec59bf4b74226170c54476da5780c9176bae084878fc94d9a2c841218e34", - "sha256:dc2bed32c7b138f1331794e454a953360c8cedf3ee62ae31f063822da6007489", - "sha256:e070a1f91202ed34c396be5ea842b886f6fa2b90d2db437dc9fb35a26c80c060", - "sha256:e42860fbe1292668b682f6dabd225fbe2a7a4fa1632f0c39881c019e93dea594", - "sha256:e4e1c486bf226822c8dceac81d0ec59c0a2399dbd1b9e04f03c3efa3605db677", - "sha256:ea4d4b58f9bc34e224ef4b4604a6be03d72ef1f8c486391f970205f6733dbc46", - "sha256:f60b3484ce4be04f5da3777c51c5140d3fe21cdd6674f2b6568f41c8130bcdeb" + "sha256:15c03ffdac17731b126880622823d30d0a3cc7203cd219e6b9814140a44e7fab", + "sha256:20fb7f4efc494016eab1bc2f555bc0a12dd5ca61f35c95df8061818ffb2c20a3", + "sha256:28ee3bcb4d609aea3040cad995a8e2c9c6dc57c12183dadd69e53880c35333b9", + "sha256:305e3c46f20d019cd57543c255e7ba49e432e275d7c0de8913b6dbe57a851bc8", + "sha256:3547b87b16aad6afb28c9b3a9cd870e11b5e7b5ac649b74265258d96d8de1130", + "sha256:3642252d7bfc4403a42050e18ba748bedebd5a998a8cba89665a4f42aea4c380", + "sha256:404faa3e518f8bea516aae2aac47d4d960397199a15b4bd6f66cad97825469a0", + "sha256:42669638e4f7937b7141044a2fbd1019caca62bd2cdd8b535f731426ab07bde1", + "sha256:4632d55a140b28e20be3cd7a3057af52fb747298ff0fd3290d4e9f245b5004ba", + "sha256:4a88c9383d273bdce3afc216020282c9c5c39ec0bd9462b1a206af6afa377cf0", + "sha256:4ce1fc1e6d2fd2d6dc197607153327989a128c093e0e94dca63408f506622c3e", + "sha256:55cf4e99b3ba0122dee570dc7661b97bf35c16aab3e2ccb5070709d282a1c7ab", + "sha256:5e486cab2dfcfaec934dd4f5d5837f4a9428b690f4d92a3b020fd31d1497ca64", + "sha256:65ec88c8271448d2ea109d35c1f297b09b872c57214ab7e832e413090d3469a9", + "sha256:6c95a3361ce70068cf69526a58751f73ddac5ba27a3c2379b057efa2f5338c8c", + "sha256:73240335f4a1baf12880ebac6df66ab4d3a9212db9f3efe809c36a27280d16f8", + "sha256:7651211e15109ac0058a49159265d9f6e6423c8a81c65434d3c56d708417a05b", + "sha256:7b5b7c5896f8172ea0beb283f7f9428e0ab88ec248ce0a5b8c98d73e26267d51", + "sha256:836fe39282e75311ce4c38468be148f7fac0df3d461c5de58c5ff1ddb8966bac", + "sha256:871852044f55295449fbf225538c2c4118525093c32f0a6c43c91bed0452d7e3", + "sha256:892e93f3e7e10c751d6c17fa0dc422f7984cfd5eb6690011f9264dc73e2775fc", + "sha256:934e460c5058346c6f1d62fdf3db5680fbdfbfd212722d24d8277bf47cd9ebdc", + "sha256:9736f3f3e1761024200637a080a4f922f5298ad5d780e10dbb5634fe8c65b34c", + "sha256:a1d38a96da57e6103423a446079ead600b450cf0f8ebf56a231895abf77e7ffc", + "sha256:a385fceaa0cdb97f0098f1c1e9ec0b46cc09186ddf60ec23538e871b1dddb6dc", + "sha256:a7cf1c14e47027d9fb9d26aa62e5d603994227bd635e58a8df4b1d2d1b6a8ed7", + "sha256:a9aac1a30b00b5038d3d8e48248f3b58ea15c827b67325c0d18a447552e30fc8", + "sha256:b696876ee583d15310be57311e90e153a84b7913ac93e6b99675c0c9867926d0", + "sha256:bef9e9d39393dc7baec39ba4bac6c73826a4db02114cdeade2552a9d6afa16e2", + "sha256:c885fe4d5f26ce8ca20c97d02e88f5fdd92c01e1cc771ad0951b21e1641faf6d", + "sha256:d2d1388595cb5d27d9220d5cbaff4f37c6ec696a25882eb06d224d241e6e93fb", + "sha256:d2e853e0f9535e693fade97768cf7293f3febabecc5feb1e9b2ffdfe1044ab96", + "sha256:d62fbab185a6b01c5469eda9f0795f3d1a5bba24f5a5813f362e4b73a3c4dc70", + "sha256:f20a62397e09704049ce9007bea4f6bad965ba9336a760c6f4ef1b4192e12d6d", + "sha256:f81f7311250d9480e36dec819127897ae772e7e8de07abfabe931b8566770b8e" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.9.8" + "index": "pypi", + "version": "==3.9.9" }, "pydeep": { "hashes": [ "sha256:22866eb422d1d5907f8076ee792da65caecb172425d27576274e2a8eacf6afc1" ], + "index": "pypi", "version": "==0.4" }, "pydnstrails": { @@ -701,6 +998,14 @@ "index": "pypi", "version": "==1.1" }, + "pyfaup": { + "hashes": [ + "sha256:5648bc3ebd80239aec927aedfc218c3a6ff36de636cc53822bfeb70b0869b1e7", + "sha256:75f96f7da86ffb5402d3fcc2dbf98a511e792cf9100c159e34cdba8996ddc7f9" + ], + "index": "pypi", + "version": "==1.2" + }, "pygeoip": { "hashes": [ "sha256:1938b9dac7b00d77f94d040b9465ea52c938f3fcdcd318b5537994f3c16aef96", @@ -721,14 +1026,18 @@ "subdirectory": "client" }, "pymisp": { - "editable": true, "extras": [ + "email", "fileobjects", "openioc", "pdfexport" ], - "git": "https://github.com/MISP/PyMISP.git", - "ref": "ec28820cf491ca7d385477996afa0547eb6b6830" + "hashes": [ + "sha256:439389a5377a9128018e9e4cc8e7ed9efa45a5efc4e77afbb55310ec7935d197", + "sha256:eed98c96b41a5e13ad1147f331ccb66e4fd9284df87528da5c99d759e29309d8" + ], + "index": "pypi", + "version": "==2.4.137.1" }, "pyonyphe": { "editable": true, @@ -737,17 +1046,18 @@ }, "pyopenssl": { "hashes": [ - "sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504", - "sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507" + "sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51", + "sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b" ], - "version": "==19.1.0" + "index": "pypi", + "version": "==20.0.1" }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "index": "pypi", "version": "==2.4.7" }, "pypdns": { @@ -767,23 +1077,31 @@ }, "pyrsistent": { "hashes": [ - "sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3" + "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" ], - "version": "==0.16.0" + "index": "pypi", + "version": "==0.17.3" }, "pytesseract": { "hashes": [ - "sha256:afd8a5cdf8ab5d35690efbe71cbf5f89419f668ea8dde7649149815d5c5a899a" + "sha256:4ecfc898d00a70fcc38d2bce729de1597c67e7bc5d2fa26094714c9f5b573645" ], "index": "pypi", - "version": "==0.3.4" + "version": "==0.3.7" + }, + "python-baseconv": { + "hashes": [ + "sha256:0539f8bd0464013b05ad62e0a1673f0ac9086c76b43ebf9f833053527cd9931b" + ], + "index": "pypi", + "version": "==1.2.2" }, "python-dateutil": { "hashes": [ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "index": "pypi", "version": "==2.8.1" }, "python-docx": { @@ -793,11 +1111,20 @@ "index": "pypi", "version": "==0.8.10" }, + "python-engineio": { + "hashes": [ + "sha256:33f7a214be5db35c867e97027bfe63676cb003d82aa17a607612b25ba5d84e5b", + "sha256:9f34afa4170f5ba6e3d9ff158752ccf8fbb2145f16554b2f0fc84646675be99a" + ], + "index": "pypi", + "version": "==4.0.0" + }, "python-magic": { "hashes": [ "sha256:356efa93c8899047d1eb7d3eb91e871ba2f5b1376edbaf4cc305e3c872207355", "sha256:b757db2a5289ea3f1ced9e60f072965243ea43a2221430048fd8cacab17be0ce" ], + "index": "pypi", "version": "==0.4.18" }, "python-pptx": { @@ -807,35 +1134,67 @@ "index": "pypi", "version": "==0.6.18" }, + "python-socketio": { + "extras": [ + "client" + ], + "hashes": [ + "sha256:870f8b00a63ef7c9a1f85fd70028624867bf246115e82625f28ef79def8847bb", + "sha256:f53fd0d5bd9f75a70492062f4ae6195ab5d34d67a29024d740f25e468392893e" + ], + "index": "pypi", + "version": "==5.0.4" + }, "python-utils": { "hashes": [ - "sha256:ebaadab29d0cb9dca0a82eab9c405f5be5125dbbff35b8f32cc433fa498dbaa7", - "sha256:f21fc09ff58ea5ebd1fd2e8ef7f63e39d456336900f26bdc9334a03a3f7d8089" + "sha256:10e155d88b706b25ede773354ea782eee6d494294651228469c916eb20f7ee1c", + "sha256:2643aec3bfcd1062accafb915aa624b81a2b0f6dad9d0c1021debead4a35cda7" ], - "version": "==2.4.0" + "index": "pypi", + "version": "==2.5.2" }, "pytz": { "hashes": [ "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" ], + "index": "pypi", "version": "==2019.3" }, "pyyaml": { "hashes": [ - "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", - "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", - "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", - "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", - "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", - "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", - "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", - "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", - "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", - "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", - "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" ], - "version": "==5.3.1" + "index": "pypi", + "version": "==5.4.1" }, "pyzbar": { "hashes": [ @@ -848,17 +1207,18 @@ }, "pyzipper": { "hashes": [ - "sha256:49813f1d415bdd7c87064009b9270c6dd0a96da770cfe57df2c6d2d84a6c085a", - "sha256:bfdc65f616278b38ef03c6ea5a1aca7499caf98cbfcd47fc44f73e68f4307145" + "sha256:80d3acd52e9c9291d88f3d7ae36f30ade38e4639941c198c62285018667dc777", + "sha256:dd4cc0d222e207e3b25570f5214411625cc117b7f463d2d51e22d669c7880992" ], - "markers": "python_version >= '3.5'", - "version": "==0.3.3" + "index": "pypi", + "version": "==0.3.4" }, "rdflib": { "hashes": [ "sha256:78149dd49d385efec3b3adfbd61c87afaf1281c30d3fcaf1b323b34f603fb155", "sha256:88208ea971a87886d60ae2b1a4b2cdc263527af0454c422118d43fe64b357877" ], + "index": "pypi", "version": "==5.0.0" }, "redis": { @@ -866,109 +1226,127 @@ "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "index": "pypi", "version": "==3.5.3" }, "reportlab": { "hashes": [ - "sha256:0f0c2d98e213d51ae527c0301364d3376cb05f6c47251368a9abd4c3197fcefa", - "sha256:1425c7ea60b8691a881ae21ea0f6907a1dc480d84204ccbfea6da41fbee8f594", - "sha256:204f1d245875ab3d076b37c1a18ac8d2e3222842e13cfa282bcd95282be239e5", - "sha256:21627b57249303bf9b5a633099d058ae9f8625fd6f90cfe79348c48fd5a242cd", - "sha256:2e8e3242f80b79f2470f1b5979abbdb41f31b1333543b830749100342f837d40", - "sha256:2eced06dec3f36135c626b9823649ef9cac95c5634d1bc743a15ee470027483b", - "sha256:3472aa0b74a3b2f252dce823f3c3ba6af8a24de0c1729441deaaf50bed6de9f9", - "sha256:3f0353ffefd3afc0061f4794ef608d6c6f32e69816885f4d45c625c20d8eaf5b", - "sha256:4a9f4540a8eddf56d900ceeb8136bd0ca866c208ba3dcbcde73f07405dbadfba", - "sha256:4eea1afb4aa89780734f44175508edff82928fdf460c9bd60bc719dd99041dc3", - "sha256:5803ffebd36de1ada417f50ce65d379ea5a0bf1a2e8f5d5710a031b3b349b726", - "sha256:58f5f72fc8e5932dedcf24789908a81c6b1e13ea4d63bd9a9a39dc698d8c3321", - "sha256:5b588e5f251c76a8d3589023d1c369c7968e0efe2b38ad5948f665edbf6f9e8b", - "sha256:5d922768fe11a58d80694852aba7389d613c15eb1871c5581a2f075996873d57", - "sha256:5d98f297c5cdd5bc0ccf5697c20b03602ee3378c97938d20312662b27cd9a1d6", - "sha256:66d1d96e97a562614943ecb9daf438e392b3d0b033bd5f4a8098ab616dd877da", - "sha256:670650970c7ba7164cf6340bcd182e7e933eff5d65183af98ee77b40cc25a438", - "sha256:67bb95af7bc8ad7925d299f310d15d556d3e7026fe1b60d8e290454604ae0a85", - "sha256:9c999f5d1a600c4970ba293789b6da14e02e3763a8d3d9abe42dcafa8a5318e9", - "sha256:9d62bef5347063a984e63410fa5a69f1d2cc2fdf8d6ed3d0b9d4ea2ccb4b4154", - "sha256:a14a0d603727b6be2e549c52dd42678ab2d06d2721d4580199e3161843e59298", - "sha256:a3a17b46ff1a15eb29370e11796d8914ef4ea67471bdbc4aa9a9eb9284f4e44c", - "sha256:a6d3e20beeba3fd68cec73b8c0785bfa648c06ac76d1f142c60ccb1a8d2506b6", - "sha256:ad7d7003c732f2be42580e3906e92bd9d2aca5e098898c597554be9ca627fad5", - "sha256:af0ee7b50b85543b68b043e61271963ff5671e564e1d620a404c24a24d4f537c", - "sha256:b3eec55274f5ead7e3af2bf0c01b481ffe1b4c6a7dae42b63d85543e9f2f9a0f", - "sha256:b48c21d43a7ab956954591ce3f71db92ce542bb7428db09734425e2b77ac3142", - "sha256:b761905ab85beb79cf7929c9a019f30ad65664e5733d57a30a995e7b9bef06d1", - "sha256:bbae2f054d0f234c3382076efa337802997aca0f3f664e314f65eefb9d694fa9", - "sha256:bd4157d0bc40fb72bb676fc745fdd648022cccaf4ccfbb291af7f48831d0d5d9", - "sha256:bf74cfabf332034f42a54938eb335543cbf92790170300dbe236ba83b7601cd0", - "sha256:c253c8571db2df3886e390a2bfbe917222953054f4643437373b824f64b013cd", - "sha256:ce1277a6acbc62e9966f410f2596ac533ee0cd5df9b69d5fe4406338a169b7d8", - "sha256:ce8f56987e0e456063e311f066a81496b8b9626c846f2cb0ebb554d1a5f40839", - "sha256:d6264a0589ba8032d9c3bdca9a3e87a897ede09b7f6a8ad5e83b57573212e01e", - "sha256:e6fa0c97e3929d00db27e8cf3b2b5771e94f5f179086c4b0e3213dff53637372", - "sha256:f0930f2b6dddd477b3331ec670171a4662336aac1a778e1a30e980a5cbf40b17", - "sha256:f8cb2b4b925ca6b6e4fdefd288a707776ac686c45034f34d4c952f122d11c40b", - "sha256:f9b71539f518323d95850405c49c01fc3d2f0f0b9f3e157de6d2786804fb28a4", - "sha256:fc488e661f99c915362e0373218f8727cecf888eb1b0eb3a8fe1af624a1b9776" + "sha256:009fa61710647cdc62eb373345248d8ebb93583a058990f7c4f9be46d90aa5b1", + "sha256:04a08d284da86882ec3a41a7c719833362ef891b09ee8e2fbb47cee352aa684a", + "sha256:07bff6742fba612da8d1b1f783c436338c6fdc6962828159827d5ca7d2b67935", + "sha256:09fb11ab1500e679fc1b01199d2fed24435499856e75043a9ac0d31dd48fd881", + "sha256:18a876449c9000c391dd3415ebc8454cd7bb9e488977b894886a2d7d018f16cd", + "sha256:18eec161411026dde49767bee4e5e8eeb8014879554811a62581dc7433628d5b", + "sha256:19353aead39fc115a4d6c598d6fb9fa26da7e69160a0443ebb49b02903e704e8", + "sha256:1b85c20e89c22ae902ca973df2afdd2d64d27dc4ffd2b29ebad8c805a213756b", + "sha256:1da3d7a35f918cee905facfa94bd00ae6091cadc06dca1b0b31b69ae02d41d1d", + "sha256:33f3cfdc492575f8af3225701301a7e62fc478358729820c9e0091aff5831378", + "sha256:3b0026c1129147befd4e5a8cf25da8dea1096fce371e7b2412e36d7254019c06", + "sha256:3d7713dddaa8081ed709a1fa2456a43f6a74b0f07d605da8441fd53fef334f69", + "sha256:3e2b4d69763103b9dc9b54c0952dc3cee05cedd06e28c0987fad7f84705b12c0", + "sha256:4ca5233a19a5ceca23546290f43addec2345789c7d65bb32f8b2668aa148351f", + "sha256:5214a289cf01ebbd65e49bae83709671dd9edb601891cf0ae8abf85f3c0b392f", + "sha256:52f8237654acbc78ea2fa6fb4a6a06e5b023b6da93f7889adfe2deba09473fad", + "sha256:5ed00894e0f8281c0b7c0494b4d3067c641fd90c8e5cf933089ec4cc9a48e491", + "sha256:6191961533d49c9d860964d42bada4d7ac3bb28502d984feb8034093f2012fa8", + "sha256:6f3ad2b1afe99c436563cd436d8693d4a12e2c4bd45f70c7705759ff7837fe53", + "sha256:739b743b7ca1ba4b4d64c321de6fccb49b562d0507ea06c817d9cc4faed5cd22", + "sha256:792efba0c0c6e4ee94f6dc95f305451733ee9230a1c7d51cb8e5301a549e0dfb", + "sha256:79d63ca40231ca3860859b39a92daa5219035ba9553da89a5e1b218550744121", + "sha256:83b28104edd58ad65748d2d0e60e0d97e3b91b3e90b4573ea6fe60de6811972c", + "sha256:85650446538cd2f606ca234634142a7ccd74cb6db7cfec250f76a4242e0f2431", + "sha256:9da445cb79e3f740756924c053edc952cde11a65ff5af8acfda3c0a1317136ef", + "sha256:9fabd5fbd24f5971085ffe53150d663f158f7d3050b25c95736e29ebf676d454", + "sha256:a0c377bc45e73c3f15f55d7de69fab270d174749d5b454ab0de502b15430ec2a", + "sha256:a1d3f7022a920d4a5e165d264581f1862e1c1b877ceeabb96fe98cec98125ae5", + "sha256:a315edef5c5610b0c75790142f49487e89ea34397fc247ae8aa890fe6d6dd057", + "sha256:a755cca2dcf023130b03bb671670301a992157d5c3151d838c0b68ef89894536", + "sha256:b1b20208ecdfffd7ca027955c4fe8972b28b30a4b3b80cf25099a08d3b20ed7c", + "sha256:b26d6f416891cef93411d6d478a25db275766081a5fb66368248293ef459f3be", + "sha256:b4ba4c30af7044ee987e61c88a5ffb76031ca0c53666bc85d823b7de55ddbc75", + "sha256:b71faf3b6e4d7058e1af1b8afedaf39a962db4a219affc8177009d8244ec10d4", + "sha256:cfa854bea525f8c913cb77e2bda724d94b965a0eb3bcfc4a645a9baa29bb86e2", + "sha256:dd9687359e466086b9f6fe6d8069034017f8b6ca3080944fae5709767ca6814e", + "sha256:de0c675fc2998a7eaa929c356ba49c84f53a892e9ab25e8ee7d8ebbbdcb2ac16", + "sha256:e2b4e33fea2ce9d3a14ea39191b169e41eb2ac995274f54ac8fd27519974bce8", + "sha256:f3d4a1a273dc141e03b72a553c11bc14dd7a27ec7654a071edcf83eb04f004bc", + "sha256:ff547cf4c1de7e104cad1a378431ff81efcb03e90e40871ee686107da5b91442" ], "index": "pypi", - "version": "==3.5.44" + "version": "==3.5.59" }, "requests": { "extras": [ "security" ], "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" ], "index": "pypi", - "version": "==2.24.0" + "version": "==2.25.1" }, "requests-cache": { "hashes": [ "sha256:813023269686045f8e01e2289cc1e7e9ae5ab22ddd1e2849a9093ab3ab7270eb", "sha256:81e13559baee64677a7d73b85498a5a8f0639e204517b5d05ff378e44a57831a" ], + "index": "pypi", "version": "==0.5.2" }, + "rtfde": { + "hashes": [ + "sha256:18386e4f060cee12a2a8035b0acf0cc99689f5dff1bf347bab7e92351860a21d", + "sha256:b86b5d734950fe8745a5b89133f50554252dbd67c6d1b9265e23ee140e7ea8a2" + ], + "index": "pypi", + "version": "==0.0.2" + }, "shodan": { "hashes": [ - "sha256:31b0740ffaf7c5196a26a0b1edf7d271dffe54ea52bb1b34ba87aa231b5c339b" + "sha256:0b5ec40c954cd48c4e3234e81ad92afdc68438f82ad392fed35b7097eb77b6dd" ], "index": "pypi", - "version": "==1.23.0" + "version": "==1.24.0" }, "sigmatools": { "hashes": [ - "sha256:5453717e452aa7860c5e6ac80bcee4f398d70956fc2ee9859bc7255067da8736", - "sha256:cdfeb8200c09c0a40ea1a015e57f3b8e2ba62a28352ca05fa015674f640871e3" + "sha256:5cca698e11f9f3f2f80b92cb4873f9958898ad23d26ce78ee4a573777f4f2035", + "sha256:719c6c19ff60177f3a155d0dd2b054a4ad7e906dec3e88dae668c2b2d200f82c" ], "index": "pypi", - "version": "==0.17.0" + "version": "==0.18.1" }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "index": "pypi", "version": "==1.15.0" }, + "socialscan": { + "hashes": [ + "sha256:3d0ca2b27d53fa4552312e07f60d3a3f513f7791a5f2bce16d3e0e3f295cd037", + "sha256:871cbc50f577b29f5f55d9c3ec5798d3abef31663f7cbe4d5c47bd5c380f6bae" + ], + "index": "pypi", + "version": "==1.4.1" + }, "socketio-client": { "hashes": [ - "sha256:540d8ab209154d1d9cdb97c170c589a14f7d7f17e19c14e2f59f0307e6175485" + "sha256:ef2e362a85ef2816fb224d727319c4b743d63b4dd9e1da99c622c9643fc4e2a0" ], - "version": "==0.5.6" + "index": "pypi", + "version": "==0.5.7.4" }, "soupsieve": { "hashes": [ - "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", - "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" + "sha256:4bb21a6ee4707bf43b61230e80740e71bfe56e55d1f1f50924b087bb2975c851", + "sha256:6dc52924dc0bc710a5d16794e6b3480b2c7c08b07729505feab2b2c16661ff6e" ], - "markers": "python_version >= '3.5'", - "version": "==2.0.1" + "index": "pypi", + "version": "==2.1" }, "sparqlwrapper": { "hashes": [ @@ -983,61 +1361,113 @@ }, "stix2-patterns": { "hashes": [ - "sha256:587a82545680311431e5610036dd6c8c247347a24243fafdafaae2df4d6d7799", - "sha256:7fcb2fa67efeac2a8c493d367c93d0ce6243a10e2eff715ae9f2983e6b32b95d" + "sha256:174fe5302d2c3223205033af987754132a9ea45a9f8e08aefafbe0549c889ea4", + "sha256:bc46cc4eba44b76a17eab7a3ff67f35203543cdb918ab24c1ebd58403fa27992" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.3.2" }, "tabulate": { "hashes": [ "sha256:ac64cb76d53b1231d364babcd72abbb16855adac7de6665122f97b593f1eb2ba", "sha256:db2723a20d04bcda8522165c73eea7c300eda74e0ce852d9022e0159d7895007" ], + "index": "pypi", "version": "==0.8.7" }, "tornado": { "hashes": [ - "sha256:0fe2d45ba43b00a41cd73f8be321a44936dc1aba233dee979f17a042b83eb6dc", - "sha256:22aed82c2ea340c3771e3babc5ef220272f6fd06b5108a53b4976d0d722bcd52", - "sha256:2c027eb2a393d964b22b5c154d1a23a5f8727db6fda837118a776b29e2b8ebc6", - "sha256:5217e601700f24e966ddab689f90b7ea4bd91ff3357c3600fa1045e26d68e55d", - "sha256:5618f72e947533832cbc3dec54e1dffc1747a5cb17d1fd91577ed14fa0dc081b", - "sha256:5f6a07e62e799be5d2330e68d808c8ac41d4a259b9cea61da4101b83cb5dc673", - "sha256:c58d56003daf1b616336781b26d184023ea4af13ae143d9dda65e31e534940b9", - "sha256:c952975c8ba74f546ae6de2e226ab3cc3cc11ae47baf607459a6728585bb542a", - "sha256:c98232a3ac391f5faea6821b53db8db461157baa788f5d6222a193e9456e1740" + "sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb", + "sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c", + "sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288", + "sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95", + "sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558", + "sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe", + "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791", + "sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d", + "sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326", + "sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b", + "sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4", + "sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c", + "sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910", + "sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5", + "sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c", + "sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0", + "sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675", + "sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd", + "sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f", + "sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c", + "sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea", + "sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6", + "sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05", + "sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd", + "sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575", + "sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a", + "sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37", + "sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795", + "sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f", + "sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32", + "sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c", + "sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01", + "sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4", + "sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2", + "sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921", + "sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085", + "sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df", + "sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102", + "sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5", + "sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68", + "sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5" ], - "markers": "python_version >= '3.5'", - "version": "==6.0.4" + "index": "pypi", + "version": "==6.1" + }, + "tqdm": { + "hashes": [ + "sha256:4621f6823bab46a9cc33d48105753ccbea671b68bab2c50a9f0be23d4065cb5a", + "sha256:fe3d08dd00a526850568d542ff9de9bbc2a09a791da3c334f3213d8d0bbbca65" + ], + "index": "pypi", + "version": "==4.56.0" }, "trustar": { "hashes": [ - "sha256:73336b94012427b66ee61db65fc3c2cea2ed743beaa56cdd5a4c1674ef1a7660" + "sha256:2618a377e3c000a41a47eb34b31ea694215eed4a1d2e3cfca1801ac6baebd958" ], "index": "pypi", - "version": "==0.3.29" + "version": "==0.3.34" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "index": "pypi", + "version": "==3.7.4.3" }, "tzlocal": { "hashes": [ "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44", "sha256:e2cb6c6b5b604af38597403e9852872d7f534962ae2954c7f35efcb1ccacf4a4" ], + "index": "pypi", "version": "==2.1" }, "unicodecsv": { "hashes": [ "sha256:018c08037d48649a0412063ff4eda26eaa81eff1546dbffa51fa5293276ff7fc" ], + "index": "pypi", "version": "==0.14.1" }, "url-normalize": { "hashes": [ - "sha256:1709cb4739e496f9f807a894e361915792f273538e250b1ab7da790544a665c3", - "sha256:1bd7085349dcdf06e52194d0f75ff99fff2eeed0da85a50e4cc2346452c1b8bc" + "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2", + "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.4.2" + "index": "pypi", + "version": "==1.4.3" }, "urlarchiver": { "hashes": [ @@ -1048,11 +1478,11 @@ }, "urllib3": { "hashes": [ - "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", - "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.25.9" + "index": "pypi", + "version": "==1.26.2" }, "uwhois": { "editable": true, @@ -1064,6 +1494,7 @@ "hashes": [ "sha256:f0ac832212e3ee2e9b10e156f19b106888cf1429c291fbc5297aae87685014ae" ], + "index": "pypi", "version": "==0.14.0" }, "vt-graph-api": { @@ -1076,48 +1507,51 @@ }, "vulners": { "hashes": [ - "sha256:00ff8744d07f398880afc1efcab6dac4abb614c84553fa31b2d439f986b8e0db", - "sha256:90a855915b4fb4dbd0325643d9e643602975fcb931162e5dc2e7778d1daa2fd8", - "sha256:f230bfcd42663326b7c9b8fa117752e26cad4ccca528caaab531c5b592af8cb5" + "sha256:065aa63d5626d51cf45260bc6cc3a6ea682977689c036a6daba695905e881ba7", + "sha256:0e1356040f456f87841ccfe9f2f6ed36a256370606d530679d5d9993fe91386c", + "sha256:ab9ed8fbf1d3c80f0d066b13ac9d70d11dc9cb0b77568be65396117a4245e916" ], "index": "pypi", - "version": "==1.5.5" + "version": "==1.5.9" }, "wand": { "hashes": [ - "sha256:d5b75ac13d7485032970926415648586eafeeb1eb62ed6ebd0778358cf9d70e0", - "sha256:df0780b1b54938a43d29279a6588fde11e349550c8958a673d57c26a3e6de7f1" + "sha256:473af4a143d2c87693cc43dbd6f64d91ccfd7f0506cf4da53851cd34f114b39d", + "sha256:ec981b4f07f7582fc564aba8b57763a549392e9ef8b6a338e9da54cdd229cf95" ], "index": "pypi", - "version": "==0.6.1" + "version": "==0.6.5" }, "websocket-client": { "hashes": [ "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549", "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010" ], + "index": "pypi", "version": "==0.57.0" }, "wrapt": { "hashes": [ "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" ], + "index": "pypi", "version": "==1.12.1" }, "xlrd": { "hashes": [ - "sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2", - "sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde" + "sha256:6a33ee89877bd9abc1158129f6e94be74e2679636b8a205b43b85206c3f0bbdd", + "sha256:f72f148f54442c6b056bf931dbc34f986fd0c3b0b6b5a58d013c9aef274d0c88" ], "index": "pypi", - "version": "==1.2.0" + "version": "==2.0.1" }, "xlsxwriter": { "hashes": [ - "sha256:828b3285fc95105f5b1946a6a015b31cf388bd5378fdc6604e4d1b7839df2e77", - "sha256:82a3b0e73e3913483da23791d1a25e4d2dbb3837d1be4129473526b9a270a5cc" + "sha256:9b1ade2d1ba5d9b40a6d1de1d55ded4394ab8002718092ae80a08532c2add2e6", + "sha256:b807c2d3e379bf6a925f472955beef3e07495c1bac708640696876e68675b49b" ], - "version": "==1.2.9" + "index": "pypi", + "version": "==1.3.7" }, "yara-python": { "hashes": [ @@ -1138,113 +1572,179 @@ }, "yarl": { "hashes": [ - "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce", - "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6", - "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce", - "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae", - "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d", - "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f", - "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b", - "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b", - "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb", - "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462", - "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea", - "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70", - "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1", - "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a", - "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b", - "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080", - "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2" + "sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e", + "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434", + "sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366", + "sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3", + "sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec", + "sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959", + "sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e", + "sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c", + "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6", + "sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a", + "sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6", + "sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424", + "sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e", + "sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f", + "sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50", + "sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2", + "sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc", + "sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4", + "sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970", + "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10", + "sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0", + "sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406", + "sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896", + "sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643", + "sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721", + "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478", + "sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724", + "sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e", + "sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8", + "sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96", + "sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25", + "sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76", + "sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2", + "sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2", + "sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c", + "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a", + "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71" ], - "markers": "python_version >= '3.5'", - "version": "==1.4.2" + "index": "pypi", + "version": "==1.6.3" + }, + "zipp": { + "hashes": [ + "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76", + "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098" + ], + "markers": "python_version >= '3.6'", + "version": "==3.4.1" } }, "develop": { "attrs": { "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==19.3.0" + "index": "pypi", + "version": "==20.3.0" }, "certifi": { "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" ], - "version": "==2020.6.20" + "index": "pypi", + "version": "==2020.12.5" }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" ], + "index": "pypi", "version": "==3.0.4" }, "codecov": { "hashes": [ - "sha256:491938ad774ea94a963d5d16354c7299e90422a33a353ba0d38d0943ed1d5091", - "sha256:b67bb8029e8340a7bf22c71cbece5bd18c96261fdebc2f105ee4d5a005bc8728", - "sha256:d8b8109f44edad03b24f5f189dac8de9b1e3dc3c791fa37eeaf8c7381503ec34" + "sha256:6cde272454009d27355f9434f4e49f238c0273b216beda8472a65dc4957f473b", + "sha256:ba8553a82942ce37d4da92b70ffd6d54cf635fc1793ab0a7dc3fecd6ebfb3df8", + "sha256:e95901d4350e99fc39c8353efa450050d2446c55bac91d90fcfd2354e19a6aef" ], "index": "pypi", - "version": "==2.1.7" + "version": "==2.1.11" }, "coverage": { "hashes": [ - "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a", - "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355", - "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65", - "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7", - "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9", - "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1", - "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0", - "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55", - "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c", - "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6", - "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef", - "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019", - "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e", - "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0", - "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf", - "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24", - "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2", - "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c", - "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4", - "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0", - "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd", - "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04", - "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e", - "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730", - "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2", - "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768", - "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796", - "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7", - "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a", - "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489", - "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052" + "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c", + "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6", + "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45", + "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a", + "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03", + "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529", + "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a", + "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a", + "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2", + "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6", + "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759", + "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53", + "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a", + "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4", + "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff", + "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502", + "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793", + "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb", + "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905", + "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821", + "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b", + "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81", + "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0", + "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b", + "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3", + "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184", + "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701", + "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a", + "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82", + "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638", + "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5", + "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083", + "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6", + "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90", + "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465", + "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a", + "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3", + "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e", + "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066", + "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf", + "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b", + "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae", + "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669", + "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873", + "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b", + "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6", + "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb", + "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160", + "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c", + "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079", + "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d", + "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==5.1" + "version": "==5.5" }, "flake8": { "hashes": [ - "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c", - "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208" + "sha256:12d05ab02614b6aee8df7c36b97d1a3b2372761222b19b58621355e82acddcff", + "sha256:78873e372b12b093da7b5e5ed302e8ad9e988b38b063b61ad937f26ca58fc5f0" ], "index": "pypi", - "version": "==3.8.3" + "version": "==3.9.0" }, "idna": { "hashes": [ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "index": "pypi", "version": "==2.10" }, + "importlib-metadata": { + "hashes": [ + "sha256:c9db46394197244adf2f0b08ec5bc3cf16757e9590b02af1fca085c16c0d600a", + "sha256:d2d46ef77ffc85cbf7dac7e81dd663fde71c45326131bea8033b9bad42268ebe" + ], + "markers": "python_version < '3.8'", + "version": "==3.10.0" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -1252,14 +1752,6 @@ ], "version": "==0.6.1" }, - "more-itertools": { - "hashes": [ - "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5", - "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2" - ], - "markers": "python_version >= '3.5'", - "version": "==8.4.0" - }, "nose": { "hashes": [ "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", @@ -1271,11 +1763,11 @@ }, "packaging": { "hashes": [ - "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", - "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", + "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.4" + "version": "==20.9" }, "pluggy": { "hashes": [ @@ -1287,77 +1779,87 @@ }, "py": { "hashes": [ - "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", - "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" + "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3", + "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.9.0" + "version": "==1.10.0" }, "pycodestyle": { "hashes": [ - "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", - "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" + "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068", + "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.6.0" + "version": "==2.7.0" }, "pyflakes": { "hashes": [ - "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", - "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" + "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", + "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.2.0" + "version": "==2.3.1" }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "index": "pypi", "version": "==2.4.7" }, "pytest": { "hashes": [ - "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1", - "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8" + "sha256:671238a46e4df0f3498d1c3270e5deb9b32d25134c99b7d75370a68cfbe9b634", + "sha256:6ad9c7bdf517a808242b998ac20063c41532a570d088d77eec1ee12b0b5574bc" ], "index": "pypi", - "version": "==5.4.3" + "version": "==6.2.3" }, "requests": { "extras": [ "security" ], "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" ], "index": "pypi", - "version": "==2.24.0" + "version": "==2.25.1" }, - "six": { + "toml": { "hashes": [ - "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", - "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.15.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "index": "pypi", + "version": "==3.7.4.3" }, "urllib3": { "hashes": [ - "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", - "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.25.9" + "index": "pypi", + "version": "==1.26.2" }, - "wcwidth": { + "zipp": { "hashes": [ - "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", - "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" + "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76", + "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098" ], - "version": "==0.2.5" + "markers": "python_version >= '3.6'", + "version": "==3.4.1" } } } diff --git a/README.md b/README.md index 77ece38..3d21bef 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,15 @@ # MISP modules -[![Build Status](https://travis-ci.org/MISP/misp-modules.svg?branch=master)](https://travis-ci.org/MISP/misp-modules) -[![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=master)](https://coveralls.io/github/MISP/misp-modules?branch=master) -[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/master/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules) +[![Build Status](https://travis-ci.org/MISP/misp-modules.svg?branch=main)](https://travis-ci.org/MISP/misp-modules) +[![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=main)](https://coveralls.io/github/MISP/misp-modules?branch=main) +[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/main/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules) -MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP). +MISP modules are autonomous modules that can be used to extend [MISP](https://github.com/MISP/MISP) for new services such as expansion, import and export. The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. -MISP modules support is included in MISP starting from version 2.4.28. - -For more information: [Extending MISP with Python modules](https://www.misp-project.org/misp-training/3.1-misp-modules.pdf) slides from MISP training. +For more information: [Extending MISP with Python modules](https://www.misp-project.org/misp-training/3.1-misp-modules.pdf) slides from [MISP training](https://github.com/MISP/misp-training). ## Existing MISP modules @@ -22,7 +20,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [AssemblyLine submit](misp_modules/modules/expansion/assemblyline_submit.py) - an expansion module to submit samples and urls to AssemblyLine. * [AssemblyLine query](misp_modules/modules/expansion/assemblyline_query.py) - an expansion module to query AssemblyLine and parse the full submission report. * [Backscatter.io](misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations. -* [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. +* [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description and its ranking and position in BGP Ranking. * [RansomcoinDB check](misp_modules/modules/expansion/ransomcoindb.py) - An expansion hover module to query the [ransomcoinDB](https://ransomcoindb.concinnity-risks.com): it contains mapping between BTC addresses and malware hashes. Enrich MISP by querying for BTC -> hash or hash -> BTC addresses. * [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused. * [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. @@ -31,6 +29,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate(s) seen. * [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. * [CrowdStrike Falcon](misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. +* [CPE](misp_modules/modules/expansion/cpe.py) - An expansion module to query the CVE Search API with a cpe code, to get its related vulnerabilities. * [CVE](misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). * [CVE advanced](misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). * [Cuckoo submit](misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. @@ -48,6 +47,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [Greynoise](misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. * [hashdd](misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. * [hibp](misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? +* [html_to_markdown](misp_modules/modules/expansion/html_to_markdown.py) - Simple HTML to markdown converter * [intel471](misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). * [IPASN](misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. * [iprep](misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. @@ -75,11 +75,13 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. * [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. -* [SophosLabs Intelix](misp_modules/modules/expansion/sophoslabs_intelix.py) - SophosLabs Intelix is an API for Threat Intelligence and Analysis (free tier availible). [SophosLabs](https://aws.amazon.com/marketplace/pp/B07SLZPMCS) +* [Socialscan](misp_modules/modules/expansion/socialscan.py) - a hover module to check if an email address or a username is used on different online platforms, using the [socialscan](https://github.com/iojw/socialscan) python library +* [SophosLabs Intelix](misp_modules/modules/expansion/sophoslabs_intelix.py) - SophosLabs Intelix is an API for Threat Intelligence and Analysis (free tier available). [SophosLabs](https://aws.amazon.com/marketplace/pp/B07SLZPMCS) * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. * [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). +* [TruSTAR Enrich](misp_modules/modules/expansion/trustar_enrich.py) - an expansion module to enrich MISP data with [TruSTAR](https://www.trustar.co/). * [urlhaus](misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url. * [urlscan](misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io). * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference)) @@ -127,8 +129,8 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick virtualenv libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev build-essential -y sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ -chown -R www-data . -sudo git clone https://github.com/MISP/misp-modules.git +sudo chown -R www-data: . +sudo -u www-data git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS sudo -u www-data /var/www/MISP/venv/bin/pip install . @@ -143,7 +145,7 @@ sudo systemctl enable --now misp-modules As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository. ~~~~bash -sudo yum install rh-ruby22 +sudo yum install rh-python36 rh-ruby22 sudo yum install openjpeg-devel sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel gcc-c++ pkgconfig poppler-cpp-devel python-devel redhat-rpm-config cd /var/www/MISP @@ -164,7 +166,7 @@ After=misp-workers.service Type=simple User=apache Group=apache -ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s' +ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s' Restart=always RestartSec=10 diff --git a/REQUIREMENTS b/REQUIREMENTS index 7541f90..ba5b5e5 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,113 +1,144 @@ +# +# These requirements were autogenerated by pipenv +# To regenerate from the project's Pipfile, run: +# +# pipenv lock --requirements +# + -i https://pypi.org/simple -e . -e git+https://github.com/D4-project/BGP-Ranking.git/@fd9c0e03af9b61d4bf0b67ac73c7208a55178a54#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@fc5e48608afc113e101ca6421bf693b7b9753f9e#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@b5b40ae2c5225a4b349c26294cfc012309a61352#egg=pymisp[fileobjects,openioc,virustotal,pdfexport] --e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client +-e git+https://github.com/Rafiot/uwhoisd.git@783bba09b5a6964f25566089826a1be4b13f2a22#egg=uwhois&subdirectory=client -e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe --e git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python -aiohttp==3.4.4 -antlr4-python3-runtime==4.8 ; python_version >= '3' +aiohttp==3.7.3; python_version >= '3.6' +antlr4-python3-runtime==4.8; python_version >= '3' apiosintds==1.8.3 argparse==1.4.0 -assemblyline-client==3.7.3 -async-timeout==3.0.1 -attrs==19.3.0 +assemblyline-client==4.0.1 +async-timeout==3.0.1; python_full_version >= '3.5.3' +attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' backscatter==0.2.4 -beautifulsoup4==4.8.2 +beautifulsoup4==4.9.3 +bidict==0.21.2; python_version >= '3.6' blockchain==1.4.4 -censys==0.0.8 -certifi==2019.11.28 -cffi==1.14.0 +censys==1.1.1 +certifi==2020.12.5 +cffi==1.14.4 chardet==3.0.4 +clamd==1.0.2 click-plugins==1.1.1 -click==7.1.1 -colorama==0.4.3 -cryptography==2.8 +click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +colorclass==2.2.0 +compressed-rtf==1.0.6 +configparser==5.0.1; python_version >= '3.6' +cryptography==3.3.1 decorator==4.4.2 -deprecated==1.2.7 -dnspython==1.16.0 -domaintools-api==0.3.3 +deprecated==1.2.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +dnsdb2==1.1.2 +dnspython3==1.15.0 +domaintools-api==0.5.2 +easygui==0.98.1 +ebcdic==1.1.1 enum-compat==0.0.3 +extract-msg==0.28.1 ez-setup==0.9 ezodf==0.3.2 -future==0.18.2 +future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' futures==3.1.1 -geoip2==3.0.0 -httplib2==0.17.0 -idna-ssl==1.1.0 ; python_version < '3.7' -idna==2.9 -importlib-metadata==1.6.0 ; python_version < '3.8' +geoip2==4.1.0 +httplib2==0.18.1 +idna-ssl==1.1.0; python_version < '3.7' +idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +imapclient==2.1.0 isodate==0.6.0 -jbxapi==3.4.0 +jbxapi==3.14.0 +json-log-formatter==0.3.0 jsonschema==3.2.0 -lief==0.10.1 -lxml==4.5.0 +lark-parser==0.11.1 +lief==0.11.0 +lxml==4.6.2 maclookup==1.0.3 -maxminddb==1.5.2 -multidict==4.7.5 +markdownify==0.5.3 +maxminddb==2.0.3; python_version >= '3.6' +msoffcrypto-tool==4.11.0 +multidict==5.1.0; python_version >= '3.6' np==1.0.2 -numpy==1.18.2 +numpy==1.19.5; python_version >= '3.6' oauth2==1.9.0.post1 -opencv-python==4.2.0.32 +olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +oletools==0.56 +opencv-python==4.5.1.48 +openpyxl pandas-ods-reader==0.0.7 -pandas==1.0.3 +pandas==1.1.5 passivetotal==1.0.31 -pdftotext==2.1.4 -pillow==7.0.0 -progressbar2==3.50.1 -psutil==5.7.0 -pycparser==2.20 -pycryptodome==3.9.7 -pycryptodomex==3.9.7 +pcodedmp==1.2.6 +pdftotext==2.1.5 +pillow==8.1.0 +progressbar2==3.53.1 +psutil==5.8.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' +pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +pycryptodome==3.9.9; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' +pycryptodomex==3.9.9; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' pydeep==0.4 -pyeupi==1.0 +pyeupi==1.1 +pyfaup==1.2 pygeoip==0.3.2 -pyopenssl==19.1.0 -pyparsing==2.4.6 +pymisp[email,fileobjects,openioc,pdfexport]==2.4.137.1 +pyopenssl==20.0.1 +pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' pypdns==1.5.1 pypssl==2.1 -pyrsistent==0.16.0 -pytesseract==0.3.3 -python-dateutil==2.8.1 +pyrsistent==0.17.3; python_version >= '3.5' +pytesseract==0.3.7 +python-baseconv==1.2.2 +python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-docx==0.8.10 -python-magic==0.4.15 +python-engineio==4.0.0 +python-magic==0.4.18 python-pptx==0.6.18 -python-utils==2.4.0 +python-socketio[client]==5.0.4 +python-utils==2.5.2 pytz==2019.3 -pyyaml==5.3.1 +pyyaml==5.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' pyzbar==0.1.8 -pyzipper==0.3.1 ; python_version >= '3.5' -rdflib==4.2.2 -redis==3.4.1 -reportlab==3.5.42 +pyzipper==0.3.4; python_version >= '3.5' +rdflib==5.0.0 +redis==3.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +reportlab==3.5.59 requests-cache==0.5.2 -requests[security]==2.23.0 -shodan==1.22.0 -sigmatools==0.16.0 -six==1.14.0 -socketio-client==0.5.6 -soupsieve==2.0 +requests[security]==2.25.1 +rtfde==0.0.2 +shodan==1.24.0 +sigmatools==0.18.1 +six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +socialscan==1.4.1 +socketio-client==0.5.7.4 +soupsieve==2.1; python_version >= '3' sparqlwrapper==1.8.5 -stix2-patterns==1.3.0 +stix2-patterns==1.3.2 tabulate==0.8.7 -tornado==6.0.4 -trustar==0.3.28 -url-normalize==1.4.1 +tornado==6.1; python_version >= '3.5' +tqdm==4.56.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +trustar==0.3.34 +typing-extensions==3.7.4.3; python_version < '3.8' +tzlocal==2.1 +unicodecsv==0.14.1 +url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' urlarchiver==0.2 -urllib3==1.25.8 +urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0' validators==0.14.0 vt-graph-api==1.0.1 -vulners==1.5.5 -wand==0.5.9 +vulners==1.5.9 +wand==0.6.5 websocket-client==0.57.0 wrapt==1.12.1 -xlrd==1.2.0 -xlsxwriter==1.2.8 -pyeti-python3==1.0 +xlrd==2.0.1 +xlsxwriter==1.3.7 yara-python==3.8.1 -yarl==1.4.2 -zipp==3.1.0 +yarl==1.6.3; python_version >= '3.6' diff --git a/doc/expansion/bgpranking.json b/doc/expansion/bgpranking.json deleted file mode 100644 index a98b780..0000000 --- a/doc/expansion/bgpranking.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Query BGP Ranking (https://bgpranking-ng.circl.lu/).", - "requirements": ["pybgpranking python library"], - "features": "The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking.\n\n", - "references": ["https://github.com/D4-project/BGP-Ranking/"], - "input": "Autonomous system number.", - "output": "Text containing a description of the ASN, its history, and the position in BGP Ranking." -} diff --git a/doc/expansion/farsight_passivedns.json b/doc/expansion/farsight_passivedns.json deleted file mode 100644 index 2c1bf05..0000000 --- a/doc/expansion/farsight_passivedns.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "Module to access Farsight DNSDB Passive DNS.", - "logo": "logos/farsight.png", - "requirements": ["An access to the Farsight Passive DNS API (apikey)"], - "input": "A domain, hostname or IP address MISP attribute.", - "output": "Text containing information about the input, resulting from the query on the Farsight Passive DNS API.", - "references": ["https://www.farsightsecurity.com/"], - "features": "This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. The API returns then the result of the query with some information about the value queried." -} diff --git a/doc/expansion/greynoise.json b/doc/expansion/greynoise.json deleted file mode 100644 index f1f1003..0000000 --- a/doc/expansion/greynoise.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "Module to access GreyNoise.io API", - "logo": "logos/greynoise.png", - "requirements": [], - "input": "An IP address.", - "output": "Additional information about the IP fetched from Greynoise API.", - "references": ["https://greynoise.io/", "https://github.com/GreyNoise-Intelligence/api.greynoise.io"], - "features": "The module takes an IP address as input and queries Greynoise for some additional information about it. The result is returned as text." -} diff --git a/doc/export_mod/cef_export.json b/doc/export_mod/cef_export.json deleted file mode 100644 index 84bba8e..0000000 --- a/doc/export_mod/cef_export.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Module to export a MISP event in CEF format.", - "requirements": [], - "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in Common Event Format.\nThus, there is no particular feature concerning MISP Events since any event can be exported. However, 4 configuration parameters recognized by CEF format are required and should be provided by users before exporting data: the device vendor, product and version, as well as the default severity of data.", - "references": ["https://community.softwaregrp.com/t5/ArcSight-Connectors/ArcSight-Common-Event-Format-CEF-Guide/ta-p/1589306?attachment-id=65537"], - "input": "MISP Event attributes", - "output": "Common Event Format file" -} diff --git a/doc/export_mod/goamlexport.json b/doc/export_mod/goamlexport.json deleted file mode 100644 index 57a1587..0000000 --- a/doc/export_mod/goamlexport.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "This module is used to export MISP events containing transaction objects into GoAML format.", - "logo": "logos/goAML.jpg", - "requirements": ["PyMISP","MISP objects"], - "features": "The module works as long as there is at least one transaction object in the Event.\n\nThen in order to have a valid GoAML document, please follow these guidelines:\n- For each transaction object, use either a bank-account, person, or legal-entity object to describe the origin of the transaction, and again one of them to describe the target of the transaction.\n- Create an object reference for both origin and target objects of the transaction.\n- A bank-account object needs a signatory, which is a person object, put as object reference of the bank-account.\n- A person can have an address, which is a geolocation object, put as object reference of the person.\n\nSupported relation types for object references that are recommended for each object are the folowing:\n- transaction:\n\t- 'from', 'from_my_client': Origin of the transaction - at least one of them is required.\n\t- 'to', 'to_my_client': Target of the transaction - at least one of them is required.\n\t- 'address': Location of the transaction - optional.\n- bank-account:\n\t- 'signatory': Signatory of a bank-account - the reference from bank-account to a signatory is required, but the relation-type is optional at the moment since this reference will always describe a signatory.\n\t- 'entity': Entity owning the bank account - optional.\n- person:\n\t- 'address': Address of a person - optional.", - "references": ["http://goaml.unodc.org/"], - "input": "MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target.", - "output": "GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities)." -} diff --git a/doc/export_mod/liteexport.json b/doc/export_mod/liteexport.json deleted file mode 100644 index 110577c..0000000 --- a/doc/export_mod/liteexport.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Lite export of a MISP event.", - "requirements": [], - "features": "This module is simply producing a json MISP event format file, but exporting only Attributes from the Event. Thus, MISP Events exported with this module should have attributes that are not internal references, otherwise the resulting event would be empty.", - "references": [], - "input": "MISP Event attributes", - "output": "Lite MISP Event" -} diff --git a/doc/export_mod/nexthinkexport.json b/doc/export_mod/nexthinkexport.json deleted file mode 100644 index 182448c..0000000 --- a/doc/export_mod/nexthinkexport.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "Nexthink NXQL query export module", - "requirements": [], - "features": "This module export an event as Nexthink NXQL queries that can then be used in your own python3 tool or from wget/powershell", - "references": ["https://doc.nexthink.com/Documentation/Nexthink/latest/APIAndIntegrations/IntroducingtheWebAPIV2"], - "input": "MISP Event attributes", - "output": "Nexthink NXQL queries", - "logo": "logos/nexthink.svg" -} diff --git a/doc/export_mod/osqueryexport.json b/doc/export_mod/osqueryexport.json deleted file mode 100644 index 6543cb1..0000000 --- a/doc/export_mod/osqueryexport.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "OSQuery export of a MISP event.", - "requirements": [], - "features": "This module export an event as osquery queries that can be used in packs or in fleet management solution like Kolide.", - "references": [], - "input": "MISP Event attributes", - "output": "osquery SQL queries", - "logo": "logos/osquery.png" -} diff --git a/doc/export_mod/pdfexport.json b/doc/export_mod/pdfexport.json deleted file mode 100644 index f1654dc..0000000 --- a/doc/export_mod/pdfexport.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Simple export of a MISP event to PDF.", - "requirements": ["PyMISP", "reportlab"], - "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option ", - "references": ["https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html"], - "input": "MISP Event", - "output": "MISP Event in a PDF file." -} diff --git a/doc/export_mod/threatStream_misp_export.json b/doc/export_mod/threatStream_misp_export.json deleted file mode 100644 index 3fdc50a..0000000 --- a/doc/export_mod/threatStream_misp_export.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "Module to export a structured CSV file for uploading to threatStream.", - "logo": "logos/threatstream.png", - "requirements": ["csv"], - "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatStream.", - "references": ["https://www.anomali.com/platform/threatstream", "https://github.com/threatstream"], - "input": "MISP Event attributes", - "output": "ThreatStream CSV format file" -} diff --git a/doc/export_mod/threat_connect_export.json b/doc/export_mod/threat_connect_export.json deleted file mode 100644 index 8d19572..0000000 --- a/doc/export_mod/threat_connect_export.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "Module to export a structured CSV file for uploading to ThreatConnect.", - "logo": "logos/threatconnect.png", - "requirements": ["csv"], - "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatConnect.\nUsers should then provide, as module configuration, the source of data they export, because it is required by the output format.", - "references": ["https://www.threatconnect.com"], - "input": "MISP Event attributes", - "output": "ThreatConnect CSV format file" -} diff --git a/doc/generate_documentation.py b/doc/generate_documentation.py deleted file mode 100644 index f86b5a7..0000000 --- a/doc/generate_documentation.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import json - -module_types = ['expansion', 'export_mod', 'import_mod'] -titles = ['Expansion Modules', 'Export Modules', 'Import Modules'] -markdown = ["# MISP modules documentation\n"] -githublink = 'https://github.com/MISP/misp-modules/tree/master/misp_modules/modules' - - -def generate_doc(root_path): - for _path, title in zip(module_types, titles): - markdown.append('\n## {}\n'.format(title)) - current_path = os.path.join(root_path, _path) - files = sorted(os.listdir(current_path)) - githubpath = '{}/{}'.format(githublink, _path) - for _file in files: - modulename = _file.split('.json')[0] - githubref = '{}/{}.py'.format(githubpath, modulename) - markdown.append('\n#### [{}]({})\n'.format(modulename, githubref)) - filename = os.path.join(current_path, _file) - with open(filename, 'rt') as f: - definition = json.loads(f.read()) - if 'logo' in definition: - markdown.append('\n\n'.format(definition.pop('logo'))) - if 'description' in definition: - markdown.append('\n{}\n'.format(definition.pop('description'))) - for field, value in sorted(definition.items()): - if value: - value = ', '.join(value) if isinstance(value, list) else '{}'.format(value.replace('\n', '\n>')) - markdown.append('- **{}**:\n>{}\n'.format(field, value)) - markdown.append('\n-----\n') - with open('README.md', 'w') as w: - w.write(''.join(markdown)) - -def generate_docs_for_mkdocs(root_path): - for _path, title in zip(module_types, titles): - markdown = [] - #markdown.append('## {}\n'.format(title)) - current_path = os.path.join(root_path, _path) - files = sorted(os.listdir(current_path)) - githubpath = '{}/{}'.format(githublink, _path) - for _file in files: - modulename = _file.split('.json')[0] - githubref = '{}/{}.py'.format(githubpath, modulename) - markdown.append('\n#### [{}]({})\n'.format(modulename, githubref)) - filename = os.path.join(current_path, _file) - with open(filename, 'rt') as f: - definition = json.loads(f.read()) - if 'logo' in definition: - markdown.append('\n\n'.format(definition.pop('logo'))) - if 'description' in definition: - markdown.append('\n{}\n'.format(definition.pop('description'))) - for field, value in sorted(definition.items()): - if value: - value = ', '.join(value) if isinstance(value, list) else '{}'.format(value.replace('\n', '\n>')) - markdown.append('- **{}**:\n>{}\n'.format(field, value)) - markdown.append('\n-----\n') - with open(root_path+"/../"+"/docs/"+_path+".md", 'w') as w: - w.write(''.join(markdown)) - -if __name__ == '__main__': - root_path = os.path.dirname(os.path.realpath(__file__)) - generate_doc(root_path) - generate_docs_for_mkdocs(root_path) diff --git a/doc/import_mod/csvimport.json b/doc/import_mod/csvimport.json deleted file mode 100644 index 66a10fd..0000000 --- a/doc/import_mod/csvimport.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Module to import MISP attributes from a csv file.", - "requirements": ["PyMISP"], - "features": "In order to parse data from a csv file, a header is required to let the module know which column is matching with known attribute fields / MISP types.\n\nThis header either comes from the csv file itself or is part of the configuration of the module and should be filled out in MISP plugin settings, each field separated by COMMAS. Fields that do not match with any type known in MISP or are not MISP attribute fields should be ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment, ').\n\nIf the csv file already contains a header that does not start by a '#', you should tick the checkbox 'has_header' to avoid importing it and have potential issues. You can also redefine the header even if it is already contained in the file, by following the rules for headers explained earlier. One reason why you would redefine a header is for instance when you want to skip some fields, or some fields are not valid types.", - "references": ["https://tools.ietf.org/html/rfc4180", "https://tools.ietf.org/html/rfc7111"], - "input": "CSV format file.", - "output": "MISP Event attributes" -} diff --git a/doc/import_mod/cuckooimport.json b/doc/import_mod/cuckooimport.json deleted file mode 100644 index 8091d07..0000000 --- a/doc/import_mod/cuckooimport.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "Module to import Cuckoo JSON.", - "logo": "logos/cuckoo.png", - "requirements": [], - "features": "The module simply imports MISP Attributes from a Cuckoo JSON format file. There is thus no special feature to make it work.", - "references": ["https://cuckoosandbox.org/", "https://github.com/cuckoosandbox/cuckoo"], - "input": "Cuckoo JSON file", - "output": "MISP Event attributes" -} diff --git a/doc/import_mod/email_import.json b/doc/import_mod/email_import.json deleted file mode 100644 index 1f53852..0000000 --- a/doc/import_mod/email_import.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Module to import emails in MISP.", - "requirements": [], - "features": "This module can be used to import e-mail text as well as attachments and urls.\n3 configuration parameters are then used to unzip attachments, guess zip attachment passwords, and extract urls: set each one of them to True or False to process or not the respective corresponding actions.", - "references": [], - "input": "E-mail file", - "output": "MISP Event attributes" -} diff --git a/doc/import_mod/goamlimport.json b/doc/import_mod/goamlimport.json deleted file mode 100644 index f2a1ec2..0000000 --- a/doc/import_mod/goamlimport.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "Module to import MISP objects about financial transactions from GoAML files.", - "logo": "logos/goAML.jpg", - "requirements": ["PyMISP"], - "features": "Unlike the GoAML export module, there is here no special feature to import data from GoAML external files, since the module will import MISP Objects with their References on its own, as it is required for the export module to rebuild a valid GoAML document.", - "references": "http://goaml.unodc.org/", - "input": "GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities).", - "output": "MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target." -} diff --git a/doc/import_mod/mispjson.json b/doc/import_mod/mispjson.json deleted file mode 100644 index dd11405..0000000 --- a/doc/import_mod/mispjson.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Module to import MISP JSON format for merging MISP events.", - "requirements": [], - "features": "The module simply imports MISP Attributes from an other MISP Event in order to merge events together. There is thus no special feature to make it work.", - "references": [], - "input": "MISP Event", - "output": "MISP Event attributes" -} diff --git a/doc/import_mod/ocr.json b/doc/import_mod/ocr.json deleted file mode 100644 index 14bbf0b..0000000 --- a/doc/import_mod/ocr.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Optical Character Recognition (OCR) module for MISP.", - "requirements": [], - "features": "The module tries to recognize some text from an image and import the result as a freetext attribute, there is then no special feature asked to users to make it work.", - "references": [], - "input": "Image", - "output": "freetext MISP attribute" -} diff --git a/doc/import_mod/openiocimport.json b/doc/import_mod/openiocimport.json deleted file mode 100644 index e173392..0000000 --- a/doc/import_mod/openiocimport.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Module to import OpenIOC packages.", - "requirements": ["PyMISP"], - "features": "The module imports MISP Attributes from OpenIOC packages, there is then no special feature for users to make it work.", - "references": ["https://www.fireeye.com/blog/threat-research/2013/10/openioc-basics.html"], - "input": "OpenIOC packages", - "output": "MISP Event attributes" -} diff --git a/doc/import_mod/threatanalyzer_import.json b/doc/import_mod/threatanalyzer_import.json deleted file mode 100644 index 40e4436..0000000 --- a/doc/import_mod/threatanalyzer_import.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Module to import ThreatAnalyzer archive.zip / analysis.json files.", - "requirements": [], - "features": "The module imports MISP Attributes from a ThreatAnalyzer format file. This file can be either ZIP, or JSON format.\nThere is by the way no special feature for users to make the module work.", - "references": ["https://www.threattrack.com/malware-analysis.aspx"], - "input": "ThreatAnalyzer format file", - "output": "MISP Event attributes" -} diff --git a/doc/import_mod/vmray_import.json b/doc/import_mod/vmray_import.json deleted file mode 100644 index b7c0dad..0000000 --- a/doc/import_mod/vmray_import.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "description": "Module to import VMRay (VTI) results.", - "logo": "logos/vmray.png", - "requirements": ["vmray_rest_api"], - "features": "The module imports MISP Attributes from VMRay format, using the VMRay api.\nUsers should then provide as the module configuration the API Key as well as the server url in order to fetch their data to import.", - "references": ["https://www.vmray.com/"], - "input": "VMRay format", - "output": "MISP Event attributes" -} diff --git a/docs/index.md b/docs/index.md index 1297a3b..53b3e77 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,120 +1,4 @@ -# Home -[![Build Status](https://travis-ci.org/MISP/misp-modules.svg?branch=master)](https://travis-ci.org/MISP/misp-modules) -[![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=master)](https://coveralls.io/github/MISP/misp-modules?branch=master) -[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/master/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules) -[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_shield) - -MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP). - -The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities -without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. - -MISP modules support is included in MISP starting from version `2.4.28`. - -For more information: [Extending MISP with Python modules](https://www.circl.lu/assets/files/misp-training/switch2016/2-misp-modules.pdf) slides from MISP training. - - -## Existing MISP modules - -### Expansion modules - -* [Backscatter.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations. -* [BGP Ranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. -* [BTC scam check](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused. -* [BTC transactions](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. -* [CIRCL Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. -* [CIRCL Passive SSL](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. -* [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. -* [CrowdStrike Falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. -* [CVE](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). -* [CVE advanced](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). -* [Cuckoo submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. -* [DBL Spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. -* [DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. -* [docx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). -* [DomainTools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. -* [EUPI](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). -* [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate event query language (EQL) from an attribute. [Event Query Language](https://eql.readthedocs.io/en/latest/) -* [Farsight DNSDB Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. -* [GeoIP](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. -* [Greynoise](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. -* [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. -* [hibp](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? -* [intel471](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). -* [IPASN](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. -* [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. -* [Joe Sandbox submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox. -* [Joe Sandbox query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data. -* [macaddress.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). -* [macvendors](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. -* [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP. -* [ods-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). -* [odt-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). -* [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. -* [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. -* [OTX](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). -* [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. -* [pdf-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser). -* [pptx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser). -* [qrcode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values. -* [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. -* [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. -* [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/). -* [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. -* [Sigma queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. -* [Sigma syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. -* [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. -* [STIX2 pattern syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. -* [ThreatCrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). -* [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). -* [urlhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url. -* [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io). -* [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference)) -* [virustotal_public](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal_public.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a public key and a low request rate limit. (More details about the API: [here](https://developers.virustotal.com/reference)) -* [VMray](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray. -* [VulnDB](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/). -* [Vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API. -* [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd). -* [wikidata](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. -* [xforce](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. -* [xlsx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser). -* [YARA query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes. -* [YARA syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator. - -### Export modules - -* [CEF](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). -* [Cisco FireSight Manager ACL rule](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL. -* [GoAML export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). -* [Lite Export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) module to export a lite event. -* [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) module to export applicable attributes from an event to a mass EQL query. -* [PDF export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. -* [Nexthink query format](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format. -* [osquery](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format. -* [ThreatConnect](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. -* [ThreatStream](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format. - -### Import modules - -* [CSV import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module. -* [Cuckoo JSON](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. -* [Email Import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. -* [GoAML import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. -* [Joe Sandbox import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report. -* [OCR](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. -* [OpenIOC](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. -* [ThreatAnalyzer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. -* [VMRay](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. - - -## How to contribute your own module? - -Fork the project, add your module, test it and make a pull-request. Modules can be also private as you can add a module in your own MISP installation. -For further information please see [Contribute](contribute/). - - -## Licenses -[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_large) - -For further Information see also the [license file](license/). \ No newline at end of file +- [expansion](./expansion) +- [export](./export_mod) +- [import](./import_mod) diff --git a/doc/README.md b/documentation/README.md similarity index 72% rename from doc/README.md rename to documentation/README.md index e173ad4..b25258a 100644 --- a/doc/README.md +++ b/documentation/README.md @@ -2,7 +2,7 @@ ## Expansion Modules -#### [apiosintds](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/apiosintds.py) +#### [apiosintds](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/apiosintds.py) On demand query API for OSINT.digitalside.it project. - **features**: @@ -22,7 +22,7 @@ On demand query API for OSINT.digitalside.it project. ----- -#### [apivoid](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/apivoid.py) +#### [apivoid](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/apivoid.py) @@ -42,7 +42,7 @@ Module to query APIVoid with some domain attributes. ----- -#### [assemblyline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/assemblyline_query.py) +#### [assemblyline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/assemblyline_query.py) @@ -64,7 +64,7 @@ A module tu query the AssemblyLine API with a submission ID to get the submissio ----- -#### [assemblyline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/assemblyline_submit.py) +#### [assemblyline_submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/assemblyline_submit.py) @@ -84,15 +84,13 @@ A module to submit samples and URLs to AssemblyLine for advanced analysis, and r ----- -#### [backscatter_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) +#### [backscatter_io](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/backscatter_io.py) Query backscatter.io (https://backscatter.io/). - **features**: >The module takes a source or destination IP address as input and displays the information known by backscatter.io. -> -> - **input**: >IP addresses. - **output**: @@ -104,17 +102,15 @@ Query backscatter.io (https://backscatter.io/). ----- -#### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) +#### [bgpranking](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/bgpranking.py) Query BGP Ranking (https://bgpranking-ng.circl.lu/). - **features**: ->The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking. -> -> +>The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day. - **input**: >Autonomous system number. - **output**: ->Text containing a description of the ASN, its history, and the position in BGP Ranking. +>An asn object with its related bgp-ranking object. - **references**: >https://github.com/D4-project/BGP-Ranking/ - **requirements**: @@ -122,7 +118,7 @@ Query BGP Ranking (https://bgpranking-ng.circl.lu/). ----- -#### [btc_scam_check](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_scam_check.py) +#### [btc_scam_check](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_scam_check.py) @@ -140,7 +136,7 @@ An expansion hover module to query a special dns blacklist to check if a bitcoin ----- -#### [btc_steroids](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_steroids.py) +#### [btc_steroids](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/btc_steroids.py) @@ -152,7 +148,7 @@ An expansion hover module to get a blockchain balance from a BTC address in MISP ----- -#### [censys_enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/censys_enrich.py) +#### [censys_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/censys_enrich.py) An expansion module to enrich attributes in MISP by quering the censys.io API - **features**: @@ -168,7 +164,7 @@ An expansion module to enrich attributes in MISP by quering the censys.io API ----- -#### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) +#### [circl_passivedns](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/circl_passivedns.py) @@ -182,13 +178,15 @@ Module to access CIRCL Passive DNS. - **ouput**: >Passive DNS objects related to the input attribute. - **references**: ->https://www.circl.lu/services/passive-dns/, https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/ +> - https://www.circl.lu/services/passive-dns/ +> - https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/ - **requirements**: ->pypdns: Passive DNS python library, A CIRCL passive DNS account with username & password +> - pypdns: Passive DNS python library +> - A CIRCL passive DNS account with username & password ----- -#### [circl_passivessl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) +#### [circl_passivessl](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/circl_passivessl.py) @@ -204,11 +202,12 @@ Modules to access CIRCL Passive SSL. - **references**: >https://www.circl.lu/services/passive-ssl/ - **requirements**: ->pypssl: Passive SSL python library, A CIRCL passive SSL account with username & password +> - pypssl: Passive SSL python library +> - A CIRCL passive SSL account with username & password ----- -#### [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) +#### [countrycode](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/countrycode.py) Module to expand country codes. - **features**: @@ -222,7 +221,28 @@ Module to expand country codes. ----- -#### [crowdstrike_falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) +#### [cpe](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cpe.py) + + + +An expansion module to query the CVE search API with a cpe code to get its related vulnerabilities. +- **features**: +>The module takes a cpe attribute as input and queries the CVE search API to get its related vulnerabilities. +>The list of vulnerabilities is then parsed and returned as vulnerability objects. +> +>Users can use their own CVE search API url by defining a value to the custom_API_URL parameter. If no custom API url is given, the default cve.circl.lu api url is used. +> +>In order to limit the amount of data returned by CVE serach, users can also the limit parameter. With the limit set, the API returns only the requested number of vulnerabilities, sorted from the highest cvss score to the lowest one. +- **input**: +>CPE attribute. +- **output**: +>The vulnerabilities related to the CPE. +- **references**: +>https://cve.circl.lu/api/ + +----- + +#### [crowdstrike_falcon](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/crowdstrike_falcon.py) @@ -276,7 +296,7 @@ Module to query Crowdstrike Falcon. ----- -#### [cuckoo_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) +#### [cuckoo_submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cuckoo_submit.py) @@ -289,13 +309,14 @@ An expansion module to submit files and URLs to Cuckoo Sandbox. - **output**: >A text field containing 'Cuckoo task id: ' - **references**: ->https://cuckoosandbox.org/, https://cuckoo.sh/docs/ +> - https://cuckoosandbox.org/ +> - https://cuckoo.sh/docs/ - **requirements**: >Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key) ----- -#### [cve](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) +#### [cve](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cve.py) @@ -307,11 +328,33 @@ An expansion hover module to expand information about CVE id. - **output**: >Text giving information about the CVE related to the Vulnerability. - **references**: ->https://cve.circl.lu/, https://cve.mitre.org/ +> - https://cve.circl.lu/ +> - https://cve.mitre.org/ ----- -#### [cytomic_orion](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cytomic_orion.py) +#### [cve_advanced](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cve_advanced.py) + + + +An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). +- **features**: +>The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to gather additional information. +> +>The result of the query is then parsed to return additional information about the vulnerability, like its cvss score or some references, as well as the potential related weaknesses and attack patterns. +> +>The vulnerability additional data is returned in a vulnerability MISP object, and the related additional information are put into weakness and attack-pattern MISP objects. +- **input**: +>Vulnerability attribute. +- **output**: +>Additional information about the vulnerability, such as its cvss score, some references, or the related weaknesses and attack patterns. +- **references**: +> - https://cve.circl.lu +> - https://cve/mitre.org/ + +----- + +#### [cytomic_orion](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/cytomic_orion.py) @@ -323,13 +366,14 @@ An expansion module to enrich attributes in MISP by quering the Cytomic Orion AP - **output**: >MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines. - **references**: ->https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/, https://www.cytomicmodel.com/solutions/ +> - https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/ +> - https://www.cytomicmodel.com/solutions/ - **requirements**: >Access (license) to Cytomic Orion ----- -#### [dbl_spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) +#### [dbl_spamhaus](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/dbl_spamhaus.py) @@ -351,7 +395,7 @@ Module to check Spamhaus DBL for a domain name. ----- -#### [dns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) +#### [dns](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/dns.py) A simple DNS expansion service to resolve IP address from domain MISP attributes. - **features**: @@ -369,7 +413,7 @@ A simple DNS expansion service to resolve IP address from domain MISP attributes ----- -#### [docx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/docx-enrich.py) +#### [docx_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/docx_enrich.py) @@ -385,7 +429,7 @@ Module to extract freetext from a .docx document. ----- -#### [domaintools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) +#### [domaintools](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/domaintools.py) @@ -418,11 +462,12 @@ DomainTools MISP expansion module. - **references**: >https://www.domaintools.com/ - **requirements**: ->Domaintools python library, A Domaintools API access (username & apikey) +> - Domaintools python library +> - A Domaintools API access (username & apikey) ----- -#### [eql](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eql.py) +#### [eql](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/eql.py) @@ -438,7 +483,7 @@ EQL query generation for a MISP attribute. ----- -#### [eupi](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) +#### [eupi](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/eupi.py) @@ -454,29 +499,72 @@ A module to query the Phishing Initiative service (https://phishing-initiative.l - **references**: >https://phishing-initiative.eu/?lang=en - **requirements**: ->pyeupi: eupi python library, An access to the Phishing Initiative API (apikey & url) +> - pyeupi: eupi python library +> - An access to the Phishing Initiative API (apikey & url) ----- -#### [farsight_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) +#### [farsight_passivedns](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/farsight_passivedns.py) Module to access Farsight DNSDB Passive DNS. - **features**: ->This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. The API returns then the result of the query with some information about the value queried. +>This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API. +> The results of rdata and rrset lookups are then returned and parsed into passive-dns objects. +> +>An API key is required to submit queries to the API. +> It is also possible to define a custom server URL, and to set a limit of results to get. +> This limit is set for each lookup, which means we can have an up to the limit number of passive-dns objects resulting from an rdata query about an IP address, but an up to the limit number of passive-dns objects for each lookup queries about a domain or a hostname (== twice the limit). - **input**: >A domain, hostname or IP address MISP attribute. - **output**: ->Text containing information about the input, resulting from the query on the Farsight Passive DNS API. +>Passive-dns objects, resulting from the query on the Farsight Passive DNS API. - **references**: ->https://www.farsightsecurity.com/ +> - https://www.farsightsecurity.com/ +> - https://docs.dnsdb.info/dnsdb-api/ - **requirements**: >An access to the Farsight Passive DNS API (apikey) ----- -#### [geoip_country](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) +#### [geoip_asn](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/geoip_asn.py) + + +- **descrption**: +>An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get information about its related AS number. +- **features**: +>The module takes an IP address attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the related AS number. +- **input**: +>An IP address MISP attribute. +- **output**: +>Text containing information about the AS number of the IP address. +- **references**: +>https://www.maxmind.com/en/home +- **requirements**: +>A local copy of Maxmind's Geolite database + +----- + +#### [geoip_city](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/geoip_city.py) + + + +An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get information about the city where it is located. +- **features**: +>The module takes an IP address attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the city where this IP address is located. +- **input**: +>An IP address MISP attribute. +- **output**: +>Text containing information about the city where the IP address is located. +- **references**: +>https://www.maxmind.com/en/home +- **requirements**: +>A local copy of Maxmind's Geolite database + +----- + +#### [geoip_country](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/geoip_country.py) @@ -496,23 +584,44 @@ Module to query a local copy of Maxmind's Geolite database. ----- -#### [greynoise](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/greynoise.py) +#### [google_search](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/google_search.py) + + +- **descrption**: +>A hover module to get information about an url using a Google search. +- **features**: +>The module takes an url as input to query the Google search API. The result of the query is then return as raw text. +- **input**: +>An url attribute. +- **output**: +>Text containing the result of a Google search on the input url. +- **references**: +>https://github.com/abenassi/Google-Search-API +- **requirements**: +>The python Google Search API library + +----- + +#### [greynoise](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/greynoise.py) Module to access GreyNoise.io API - **features**: ->The module takes an IP address as input and queries Greynoise for some additional information about it. The result is returned as text. +>The module takes an IP address as input and queries Greynoise for some additional information about it: basically it checks whether a given IP address is “Internet background noise”, or has been observed scanning or attacking devices across the Internet. The result is returned as text. - **input**: >An IP address. - **output**: >Additional information about the IP fetched from Greynoise API. - **references**: ->https://greynoise.io/, https://github.com/GreyNoise-Intelligence/api.greynoise.io +> - https://greynoise.io/ +> - https://github.com/GreyNoise-Intelligence/api.greynoise.io +- **requirements**: +>A Greynoise API key. ----- -#### [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) +#### [hashdd](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/hashdd.py) A hover module to check hashes against hashdd.com including NSLR dataset. - **features**: @@ -526,7 +635,7 @@ A hover module to check hashes against hashdd.com including NSLR dataset. ----- -#### [hibp](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hibp.py) +#### [hibp](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/hibp.py) @@ -542,7 +651,52 @@ Module to access haveibeenpwned.com API. ----- -#### [intelmq_eventdb](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intelmq_eventdb.py) +#### [html_to_markdown](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/html_to_markdown.py) + +Expansion module to fetch the html content from an url and convert it into markdown. +- **features**: +>The module take an URL as input and the HTML content is fetched from it. This content is then converted into markdown that is returned as text. +- **input**: +>URL attribute. +- **output**: +>Markdown content converted from the HTML fetched from the url. +- **requirements**: +>The markdownify python library + +----- + +#### [intel471](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/intel471.py) + + +- **descrption**: +>An expansion module to query Intel471 in order to get additional information about a domain, ip address, email address, url or hash. +- **features**: +>The module uses the Intel471 python library to query the Intel471 API with the value of the input attribute. The result of the query is then returned as freetext so the Freetext import parses it. +- **input**: +>A MISP attribute whose type is included in the following list: +>- hostname +>- domain +>- url +>- ip-src +>- ip-dst +>- email-src +>- email-dst +>- target-email +>- whois-registrant-email +>- whois-registrant-name +>- md5 +>- sha1 +>- sha256 +- **output**: +>Freetext +- **references**: +>https://public.intel471.com/ +- **requirements**: +>The intel471 python library + +----- + +#### [intelmq_eventdb](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/intelmq_eventdb.py) @@ -556,13 +710,15 @@ Module to access intelmqs eventdb. - **output**: >Text giving information about the input using IntelMQ database. - **references**: ->https://github.com/certtools/intelmq, https://intelmq.readthedocs.io/en/latest/Developers-Guide/ +> - https://github.com/certtools/intelmq +> - https://intelmq.readthedocs.io/en/latest/Developers-Guide/ - **requirements**: ->psycopg2: Python library to support PostgreSQL, An access to the IntelMQ database (username, password, hostname and database reference) +> - psycopg2: Python library to support PostgreSQL +> - An access to the IntelMQ database (username, password, hostname and database reference) ----- -#### [ipasn](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) +#### [ipasn](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ipasn.py) Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History). - **features**: @@ -578,7 +734,7 @@ Module to query an IP ASN history service (https://github.com/D4-project/IPASN-H ----- -#### [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) +#### [iprep](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/iprep.py) Module to query IPRep data for IP addresses. - **features**: @@ -594,7 +750,7 @@ Module to query IPRep data for IP addresses. ----- -#### [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) +#### [joesandbox_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_query.py) @@ -614,13 +770,14 @@ This url can by the way come from the result of the [joesandbox_submit expansion - **output**: >MISP attributes & objects parsed from the analysis report. - **references**: ->https://www.joesecurity.org, https://www.joesandbox.com/ +> - https://www.joesecurity.org +> - https://www.joesandbox.com/ - **requirements**: >jbxapi: Joe Sandbox API python3 library ----- -#### [joesandbox_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py) +#### [joesandbox_submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/joesandbox_submit.py) @@ -634,13 +791,14 @@ A module to submit files or URLs to Joe Sandbox for an advanced analysis, and re - **output**: >Link of the report generated in Joe Sandbox. - **references**: ->https://www.joesecurity.org, https://www.joesandbox.com/ +> - https://www.joesecurity.org +> - https://www.joesandbox.com/ - **requirements**: >jbxapi: Joe Sandbox API python3 library ----- -#### [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) +#### [lastline_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_query.py) @@ -659,7 +817,7 @@ The analysis link can also be retrieved from the output of the [lastline_submit] ----- -#### [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) +#### [lastline_submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/lastline_submit.py) @@ -676,7 +834,7 @@ Module to submit a file or URL to Lastline. ----- -#### [macaddress_io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) +#### [macaddress_io](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/macaddress_io.py) @@ -693,13 +851,15 @@ MISP hover module for macaddress.io - **output**: >Text containing information on the MAC address fetched from a query on macaddress.io. - **references**: ->https://macaddress.io/, https://github.com/CodeLineFi/maclookup-python +> - https://macaddress.io/ +> - https://github.com/CodeLineFi/maclookup-python - **requirements**: ->maclookup: macaddress.io python library, An access to the macaddress.io API (apikey) +> - maclookup: macaddress.io python library +> - An access to the macaddress.io API (apikey) ----- -#### [macvendors](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macvendors.py) +#### [macvendors](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/macvendors.py) @@ -711,11 +871,12 @@ Module to access Macvendors API. - **output**: >Additional information about the MAC address. - **references**: ->https://macvendors.com/, https://macvendors.com/api +> - https://macvendors.com/ +> - https://macvendors.com/api ----- -#### [malwarebazaar](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/malwarebazaar.py) +#### [malwarebazaar](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/malwarebazaar.py) Query the MALWAREbazaar API to get additional information about the input hash attribute. - **features**: @@ -731,7 +892,7 @@ Query the MALWAREbazaar API to get additional information about the input hash a ----- -#### [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py) +#### [ocr_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ocr_enrich.py) Module to process some optical character recognition on pictures. - **features**: @@ -745,7 +906,7 @@ Module to process some optical character recognition on pictures. ----- -#### [ods-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ods-enrich.py) +#### [ods_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ods_enrich.py) @@ -757,11 +918,12 @@ Module to extract freetext from a .ods document. - **output**: >Text and freetext parsed from the document. - **requirements**: ->ezodf: Python package to create/manipulate OpenDocumentFormat files., pandas_ods_reader: Python library to read in ODS files. +> - ezodf: Python package to create/manipulate OpenDocumentFormat files. +> - pandas_ods_reader: Python library to read in ODS files. ----- -#### [odt-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/odt-enrich.py) +#### [odt_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/odt_enrich.py) @@ -777,7 +939,7 @@ Module to extract freetext from a .odt document. ----- -#### [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) +#### [onyphe](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onyphe.py) @@ -789,13 +951,15 @@ Module to process a query on Onyphe. - **output**: >MISP attributes fetched from the Onyphe query. - **references**: ->https://www.onyphe.io/, https://github.com/sebdraven/pyonyphe +> - https://www.onyphe.io/ +> - https://github.com/sebdraven/pyonyphe - **requirements**: ->onyphe python library, An access to the Onyphe API (apikey) +> - onyphe python library +> - An access to the Onyphe API (apikey) ----- -#### [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) +#### [onyphe_full](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/onyphe_full.py) @@ -809,13 +973,15 @@ Module to process a full query on Onyphe. - **output**: >MISP attributes fetched from the Onyphe query. - **references**: ->https://www.onyphe.io/, https://github.com/sebdraven/pyonyphe +> - https://www.onyphe.io/ +> - https://github.com/sebdraven/pyonyphe - **requirements**: ->onyphe python library, An access to the Onyphe API (apikey) +> - onyphe python library +> - An access to the Onyphe API (apikey) ----- -#### [otx](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) +#### [otx](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/otx.py) @@ -850,7 +1016,7 @@ Module to get information from AlienVault OTX. ----- -#### [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) +#### [passivetotal](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/passivetotal.py) @@ -896,11 +1062,12 @@ Module to get information from AlienVault OTX. - **references**: >https://www.passivetotal.org/register - **requirements**: ->Passivetotal python library, An access to the PassiveTotal API (apikey) +> - Passivetotal python library +> - An access to the PassiveTotal API (apikey) ----- -#### [pdf-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pdf-enrich.py) +#### [pdf_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/pdf_enrich.py) @@ -916,7 +1083,7 @@ Module to extract freetext from a PDF document. ----- -#### [pptx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pptx-enrich.py) +#### [pptx_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/pptx_enrich.py) @@ -932,7 +1099,7 @@ Module to extract freetext from a .pptx document. ----- -#### [qrcode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/qrcode.py) +#### [qrcode](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/qrcode.py) Module to decode QR codes. - **features**: @@ -942,11 +1109,30 @@ Module to decode QR codes. - **output**: >The URL or bitcoin address the QR code is pointing to. - **requirements**: ->cv2: The OpenCV python library., pyzbar: Python library to read QR codes. +> - cv2: The OpenCV python library. +> - pyzbar: Python library to read QR codes. ----- -#### [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) +#### [ransomcoindb](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/ransomcoindb.py) +- **descrption**: +>Module to access the ransomcoinDB with a hash or btc address attribute and get the associated btc address of hashes. +- **features**: +>The module takes either a hash attribute or a btc attribute as input to query the ransomcoinDB API for some additional data. +> +>If the input is a btc address, we will get the associated hashes returned in a file MISP object. If we query ransomcoinDB with a hash, the response contains the associated btc addresses returned as single MISP btc attributes. +- **input**: +>A hash (md5, sha1 or sha256) or btc attribute. +- **output**: +>Hashes associated to a btc address or btc addresses associated to a hash. +- **references**: +>https://ransomcoindb.concinnity-risks.com +- **requirements**: +>A ransomcoinDB API key. + +----- + +#### [rbl](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/rbl.py) Module to check an IPv4 address against known RBLs. - **features**: @@ -964,7 +1150,25 @@ Module to check an IPv4 address against known RBLs. ----- -#### [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) +#### [recordedfuture](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/recordedfuture.py) + + + +Module to enrich attributes with threat intelligence from Recorded Future. +- **features**: +>Enrich an attribute to add a custom enrichment object to the event. The object contains a copy of the enriched attribute with added tags presenting risk score and triggered risk rules from Recorded Future. Malware and Threat Actors related to the enriched indicator in Recorded Future is matched against MISP's galaxy clusters and applied as galaxy tags. The custom enrichment object also includes a list of related indicators from Recorded Future (IP's, domains, hashes, URL's and vulnerabilities) added as additional attributes. +- **input**: +>A MISP attribute of one of the following types: ip, ip-src, ip-dst, domain, hostname, md5, sha1, sha256, uri, url, vulnerability, weakness. +- **output**: +>A MISP object containing a copy of the enriched attribute with added tags from Recorded Future and a list of new attributes related to the enriched attribute. +- **references**: +>https://www.recordedfuture.com/ +- **requirements**: +>A Recorded Future API token. + +----- + +#### [reversedns](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/reversedns.py) Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. - **features**: @@ -982,7 +1186,7 @@ Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes ----- -#### [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) +#### [securitytrails](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/securitytrails.py) @@ -1011,11 +1215,12 @@ An expansion modules for SecurityTrails. - **references**: >https://securitytrails.com/ - **requirements**: ->dnstrails python library, An access to the SecurityTrails API (apikey) +> - dnstrails python library +> - An access to the SecurityTrails API (apikey) ----- -#### [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) +#### [shodan](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/shodan.py) @@ -1029,11 +1234,12 @@ Module to query on Shodan. - **references**: >https://www.shodan.io/ - **requirements**: ->shodan python library, An access to the Shodan API (apikey) +> - shodan python library +> - An access to the Shodan API (apikey) ----- -#### [sigma_queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) +#### [sigma_queries](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sigma_queries.py) @@ -1051,7 +1257,7 @@ An expansion hover module to display the result of sigma queries. ----- -#### [sigma_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) +#### [sigma_syntax_validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sigma_syntax_validator.py) @@ -1067,11 +1273,46 @@ An expansion hover module to perform a syntax check on sigma rules. - **references**: >https://github.com/Neo23x0/sigma/wiki - **requirements**: ->Sigma python library, Yaml python library +> - Sigma python library +> - Yaml python library ----- -#### [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) +#### [socialscan](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/socialscan.py) + +A hover module to get information on the availability of an email address or username on some online platforms. +- **features**: +>The module takes an email address or username as input and check its availability on some online platforms. The results for each platform are then returned to see if the email address or the username is used, available or if there is an issue with it. +- **input**: +>An email address or usename attribute. +- **output**: +>Text containing information about the availability of an email address or a username in some online platforms. +- **references**: +>https://github.com/iojw/socialscan +- **requirements**: +>The socialscan python library + +----- + +#### [sophoslabs_intelix](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sophoslabs_intelix.py) + + + +An expansion module to query the Sophoslabs intelix API to get additional information about an ip address, url, domain or sha256 attribute. +- **features**: +>The module takes an ip address, url, domain or sha256 attribute and queries the SophosLabs Intelix API with the attribute value. The result of this query is a SophosLabs Intelix hash report, or an ip or url lookup, that is then parsed and returned in a MISP object. +- **input**: +>An ip address, url, domain or sha256 attribute. +- **output**: +>SophosLabs Intelix report and lookup objects +- **references**: +>https://aws.amazon.com/marketplace/pp/B07SLZPMCS +- **requirements**: +>A client_id and client_secret pair to authenticate to the SophosLabs Intelix API + +----- + +#### [sourcecache](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/sourcecache.py) Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page. - **features**: @@ -1087,7 +1328,7 @@ Module to cache web pages of analysis reports, OSINT sources. The module returns ----- -#### [stix2_pattern_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) +#### [stix2_pattern_syntax_validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) @@ -1107,7 +1348,7 @@ An expansion hover module to perform a syntax check on stix2 patterns. ----- -#### [threatcrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) +#### [threatcrowd](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/threatcrowd.py) @@ -1144,7 +1385,7 @@ Module to get information from ThreatCrowd. ----- -#### [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) +#### [threatminer](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/threatminer.py) @@ -1184,7 +1425,7 @@ Module to get information from ThreatMiner. ----- -#### [trustar_enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/trustar_enrich.py) +#### [trustar_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/trustar_enrich.py) @@ -1213,7 +1454,7 @@ Module to get enrich indicators with TruSTAR. ----- -#### [urlhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlhaus.py) +#### [urlhaus](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/urlhaus.py) @@ -1231,7 +1472,7 @@ Query of the URLhaus API to get additional information about the input attribute ----- -#### [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) +#### [urlscan](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/urlscan.py) @@ -1251,7 +1492,7 @@ An expansion module to query urlscan.io. ----- -#### [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) +#### [virustotal](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/virustotal.py) @@ -1269,13 +1510,14 @@ Module to get advanced information from virustotal. - **output**: >MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute. - **references**: ->https://www.virustotal.com/, https://developers.virustotal.com/reference +> - https://www.virustotal.com/ +> - https://developers.virustotal.com/reference - **requirements**: >An access to the VirusTotal API (apikey), with a high request rate limit. ----- -#### [virustotal_public](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal_public.py) +#### [virustotal_public](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/virustotal_public.py) @@ -1293,13 +1535,14 @@ Module to get information from VirusTotal. - **output**: >MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute. - **references**: ->https://www.virustotal.com, https://developers.virustotal.com/reference +> - https://www.virustotal.com +> - https://developers.virustotal.com/reference - **requirements**: >An access to the VirusTotal API (apikey) ----- -#### [vmray_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) +#### [vmray_submit](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vmray_submit.py) @@ -1324,7 +1567,7 @@ Module to submit a sample to VMRay. ----- -#### [vulndb](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) +#### [vulndb](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vulndb.py) @@ -1344,7 +1587,7 @@ Module to query VulnDB (RiskBasedSecurity.com). ----- -#### [vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) +#### [vulners](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/vulners.py) @@ -1360,11 +1603,12 @@ An expansion hover module to expand information about CVE id using Vulners API. - **references**: >https://vulners.com/ - **requirements**: ->Vulners python library, An access to the Vulners API +> - Vulners python library +> - An access to the Vulners API ----- -#### [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) +#### [whois](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/whois.py) Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd). - **features**: @@ -1380,7 +1624,7 @@ Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd). ----- -#### [wiki](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) +#### [wiki](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/wiki.py) @@ -1398,7 +1642,7 @@ An expansion hover module to extract information from Wikidata to have additiona ----- -#### [xforceexchange](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) +#### [xforceexchange](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/xforceexchange.py) @@ -1422,7 +1666,7 @@ An expansion module for IBM X-Force Exchange. ----- -#### [xlsx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xlsx-enrich.py) +#### [xlsx_enrich](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/xlsx_enrich.py) @@ -1438,7 +1682,7 @@ Module to extract freetext from a .xlsx document. ----- -#### [yara_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) +#### [yara_query](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/yara_query.py) @@ -1451,13 +1695,14 @@ An expansion & hover module to translate any hash attribute into a yara rule. - **output**: >YARA rule. - **references**: ->https://virustotal.github.io/yara/, https://github.com/virustotal/yara-python +> - https://virustotal.github.io/yara/ +> - https://github.com/virustotal/yara-python - **requirements**: >yara-python python library ----- -#### [yara_syntax_validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) +#### [yara_syntax_validator](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/expansion/yara_syntax_validator.py) @@ -1477,7 +1722,7 @@ An expansion hover module to perform a syntax check on if yara rules are valid o ## Export Modules -#### [cef_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) +#### [cef_export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/cef_export.py) Module to export a MISP event in CEF format. - **features**: @@ -1492,7 +1737,7 @@ Module to export a MISP event in CEF format. ----- -#### [cisco_firesight_manager_ACL_rule_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) +#### [cisco_firesight_manager_ACL_rule_export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) @@ -1508,7 +1753,23 @@ Module to export malicious network activity attributes to Cisco fireSIGHT manage ----- -#### [goamlexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) +#### [defender_endpoint_export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/defender_endpoint_export.py) + + + +Defender for Endpoint KQL hunting query export module +- **features**: +>This module export an event as Defender for Endpoint KQL queries that can then be used in your own python3 or Powershell tool. If you are using Microsoft Sentinel, you can directly connect your MISP instance to Sentinel and then create queries using the `ThreatIntelligenceIndicator` table to match events against imported IOC. +- **input**: +>MISP Event attributes +- **output**: +>Defender for Endpoint KQL queries +- **references**: +>https://docs.microsoft.com/en-us/windows/security/threat-protection/microsoft-defender-atp/advanced-hunting-schema-reference + +----- + +#### [goamlexport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/goamlexport.py) @@ -1539,11 +1800,12 @@ This module is used to export MISP events containing transaction objects into Go - **references**: >http://goaml.unodc.org/ - **requirements**: ->PyMISP, MISP objects +> - PyMISP +> - MISP objects ----- -#### [liteexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) +#### [liteexport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/liteexport.py) Lite export of a MISP event. - **features**: @@ -1555,7 +1817,7 @@ Lite export of a MISP event. ----- -#### [mass_eql_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/mass_eql_export.py) +#### [mass_eql_export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/mass_eql_export.py) @@ -1571,7 +1833,7 @@ Mass EQL query export for a MISP event. ----- -#### [nexthinkexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) +#### [nexthinkexport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/nexthinkexport.py) @@ -1587,7 +1849,7 @@ Nexthink NXQL query export module ----- -#### [osqueryexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) +#### [osqueryexport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/osqueryexport.py) @@ -1601,7 +1863,7 @@ OSQuery export of a MISP event. ----- -#### [pdfexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) +#### [pdfexport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/pdfexport.py) Simple export of a MISP event to PDF. - **features**: @@ -1619,17 +1881,18 @@ Simple export of a MISP event to PDF. - **references**: >https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html - **requirements**: ->PyMISP, reportlab +> - PyMISP +> - reportlab ----- -#### [testexport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/testexport.py) +#### [testexport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/testexport.py) Skeleton export module. ----- -#### [threatStream_misp_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) +#### [threatStream_misp_export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/threatStream_misp_export.py) @@ -1641,13 +1904,14 @@ Module to export a structured CSV file for uploading to threatStream. - **output**: >ThreatStream CSV format file - **references**: ->https://www.anomali.com/platform/threatstream, https://github.com/threatstream +> - https://www.anomali.com/platform/threatstream +> - https://github.com/threatstream - **requirements**: >csv ----- -#### [threat_connect_export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) +#### [threat_connect_export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/threat_connect_export.py) @@ -1666,7 +1930,7 @@ Module to export a structured CSV file for uploading to ThreatConnect. ----- -#### [vt_graph](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/vt_graph.py) +#### [vt_graph](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/vt_graph.py) @@ -1688,7 +1952,7 @@ This module is used to create a VirusTotal Graph from a MISP event. ## Import Modules -#### [csvimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) +#### [csvimport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/csvimport.py) Module to import MISP attributes from a csv file. - **features**: @@ -1702,13 +1966,14 @@ Module to import MISP attributes from a csv file. - **output**: >MISP Event attributes - **references**: ->https://tools.ietf.org/html/rfc4180, https://tools.ietf.org/html/rfc7111 +> - https://tools.ietf.org/html/rfc4180 +> - https://tools.ietf.org/html/rfc7111 - **requirements**: >PyMISP ----- -#### [cuckooimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) +#### [cuckooimport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/cuckooimport.py) @@ -1720,11 +1985,12 @@ Module to import Cuckoo JSON. - **output**: >MISP Event attributes - **references**: ->https://cuckoosandbox.org/, https://github.com/cuckoosandbox/cuckoo +> - https://cuckoosandbox.org/ +> - https://github.com/cuckoosandbox/cuckoo ----- -#### [email_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) +#### [email_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/email_import.py) Module to import emails in MISP. - **features**: @@ -1737,7 +2003,7 @@ Module to import emails in MISP. ----- -#### [goamlimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) +#### [goamlimport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/goamlimport.py) @@ -1755,7 +2021,7 @@ Module to import MISP objects about financial transactions from GoAML files. ----- -#### [joe_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) +#### [joe_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/joe_import.py) @@ -1764,18 +2030,17 @@ A module to import data from a Joe Sandbox analysis json report. >Module using the new format of modules able to return attributes and objects. > >The module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report. -> -> - **input**: >Json report of a Joe Sandbox analysis. - **output**: >MISP attributes & objects parsed from the analysis report. - **references**: ->https://www.joesecurity.org, https://www.joesandbox.com/ +> - https://www.joesecurity.org +> - https://www.joesandbox.com/ ----- -#### [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) +#### [lastline_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/lastline_import.py) @@ -1793,7 +2058,7 @@ Module to import and parse reports from Lastline analysis links. ----- -#### [mispjson](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/mispjson.py) +#### [mispjson](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/mispjson.py) Module to import MISP JSON format for merging MISP events. - **features**: @@ -1805,7 +2070,7 @@ Module to import MISP JSON format for merging MISP events. ----- -#### [ocr](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) +#### [ocr](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP. - **features**: @@ -1817,7 +2082,7 @@ Optical Character Recognition (OCR) module for MISP. ----- -#### [openiocimport](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) +#### [openiocimport](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/openiocimport.py) Module to import OpenIOC packages. - **features**: @@ -1833,7 +2098,7 @@ Module to import OpenIOC packages. ----- -#### [threatanalyzer_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) +#### [threatanalyzer_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/threatanalyzer_import.py) Module to import ThreatAnalyzer archive.zip / analysis.json files. - **features**: @@ -1848,7 +2113,7 @@ Module to import ThreatAnalyzer archive.zip / analysis.json files. ----- -#### [vmray_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) +#### [vmray_import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/vmray_import.py) diff --git a/documentation/generate_documentation.py b/documentation/generate_documentation.py new file mode 100644 index 0000000..4081e50 --- /dev/null +++ b/documentation/generate_documentation.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import os +import json + +module_types = ['expansion', 'export_mod', 'import_mod'] +titles = ['Expansion Modules', 'Export Modules', 'Import Modules'] +githublink = 'https://github.com/MISP/misp-modules/tree/main/misp_modules/modules' + + +def generate_doc(module_type, root_path, logo_path='logos'): + markdown = [] + current_path = os.path.join(root_path, 'website', module_type) + files = sorted(os.listdir(current_path)) + githubpath = f'{githublink}/{module_type}' + for filename in files: + modulename = filename.split('.json')[0] + githubref = f'{githubpath}/{modulename}.py' + markdown.append(f'\n#### [{modulename}]({githubref})\n') + filename = os.path.join(current_path, filename) + with open(filename, 'rt') as f: + definition = json.loads(f.read()) + if 'logo' in definition: + logo = os.path.join(logo_path, definition.pop('logo')) + markdown.append(f"\n\n") + if 'description' in definition: + markdown.append(f"\n{definition.pop('description')}\n") + for field, value in sorted(definition.items()): + if not value: + continue + if isinstance(value, list): + markdown.append(handle_list(field, value)) + continue + markdown.append(get_single_value(field, value.replace('\n', '\n>'))) + markdown.append('\n-----\n') + return markdown + + +def get_single_value(field, value): + return f"- **{field}**:\n>{value}\n" + + +def handle_list(field, values): + if len(values) == 1: + return get_single_value(field, values[0]) + values = '\n> - '.join(values) + return f"- **{field}**:\n> - {values}\n" + + +def write_doc(root_path): + markdown = ["# MISP modules documentation\n"] + for _path, title in zip(module_types, titles): + markdown.append(f'\n## {title}\n') + markdown.extend(generate_doc(_path, root_path)) + with open('README.md', 'w') as w: + w.write(''.join(markdown)) + + +def write_docs_for_mkdocs(root_path): + for _path, title in zip(module_types, titles): + markdown = generate_doc(_path, root_path, logo_path='../logos') + with open(os.path.join(root_path, 'mkdocs', f'{_path}.md'), 'w') as w: + w.write(''.join(markdown)) + + +if __name__ == '__main__': + root_path = os.path.dirname(os.path.realpath(__file__)) + write_doc(root_path) + write_docs_for_mkdocs(root_path) diff --git a/doc/logos/apivoid.png b/documentation/logos/apivoid.png similarity index 100% rename from doc/logos/apivoid.png rename to documentation/logos/apivoid.png diff --git a/doc/logos/assemblyline.png b/documentation/logos/assemblyline.png similarity index 100% rename from doc/logos/assemblyline.png rename to documentation/logos/assemblyline.png diff --git a/doc/logos/backscatter_io.png b/documentation/logos/backscatter_io.png similarity index 100% rename from doc/logos/backscatter_io.png rename to documentation/logos/backscatter_io.png diff --git a/doc/logos/bitcoin.png b/documentation/logos/bitcoin.png similarity index 100% rename from doc/logos/bitcoin.png rename to documentation/logos/bitcoin.png diff --git a/doc/logos/cisco.png b/documentation/logos/cisco.png similarity index 100% rename from doc/logos/cisco.png rename to documentation/logos/cisco.png diff --git a/doc/logos/crowdstrike.png b/documentation/logos/crowdstrike.png similarity index 100% rename from doc/logos/crowdstrike.png rename to documentation/logos/crowdstrike.png diff --git a/doc/logos/cuckoo.png b/documentation/logos/cuckoo.png similarity index 100% rename from doc/logos/cuckoo.png rename to documentation/logos/cuckoo.png diff --git a/doc/logos/cve.png b/documentation/logos/cve.png similarity index 100% rename from doc/logos/cve.png rename to documentation/logos/cve.png diff --git a/doc/logos/cytomic_orion.png b/documentation/logos/cytomic_orion.png similarity index 100% rename from doc/logos/cytomic_orion.png rename to documentation/logos/cytomic_orion.png diff --git a/documentation/logos/defender_endpoing.png b/documentation/logos/defender_endpoing.png new file mode 100644 index 0000000..efc7ace Binary files /dev/null and b/documentation/logos/defender_endpoing.png differ diff --git a/doc/logos/docx.png b/documentation/logos/docx.png similarity index 100% rename from doc/logos/docx.png rename to documentation/logos/docx.png diff --git a/doc/logos/domaintools.png b/documentation/logos/domaintools.png similarity index 100% rename from doc/logos/domaintools.png rename to documentation/logos/domaintools.png diff --git a/doc/logos/eql.png b/documentation/logos/eql.png similarity index 100% rename from doc/logos/eql.png rename to documentation/logos/eql.png diff --git a/doc/logos/eupi.png b/documentation/logos/eupi.png similarity index 100% rename from doc/logos/eupi.png rename to documentation/logos/eupi.png diff --git a/doc/logos/farsight.png b/documentation/logos/farsight.png similarity index 100% rename from doc/logos/farsight.png rename to documentation/logos/farsight.png diff --git a/doc/logos/goAML.jpg b/documentation/logos/goAML.jpg similarity index 100% rename from doc/logos/goAML.jpg rename to documentation/logos/goAML.jpg diff --git a/documentation/logos/google.png b/documentation/logos/google.png new file mode 100644 index 0000000..492f44c Binary files /dev/null and b/documentation/logos/google.png differ diff --git a/doc/logos/greynoise.png b/documentation/logos/greynoise.png similarity index 100% rename from doc/logos/greynoise.png rename to documentation/logos/greynoise.png diff --git a/doc/logos/hibp.png b/documentation/logos/hibp.png similarity index 100% rename from doc/logos/hibp.png rename to documentation/logos/hibp.png diff --git a/documentation/logos/intel471.png b/documentation/logos/intel471.png new file mode 100644 index 0000000..08264e9 Binary files /dev/null and b/documentation/logos/intel471.png differ diff --git a/doc/logos/intelmq.png b/documentation/logos/intelmq.png similarity index 100% rename from doc/logos/intelmq.png rename to documentation/logos/intelmq.png diff --git a/doc/logos/joesandbox.png b/documentation/logos/joesandbox.png similarity index 100% rename from doc/logos/joesandbox.png rename to documentation/logos/joesandbox.png diff --git a/doc/logos/lastline.png b/documentation/logos/lastline.png similarity index 100% rename from doc/logos/lastline.png rename to documentation/logos/lastline.png diff --git a/doc/logos/macaddress_io.png b/documentation/logos/macaddress_io.png similarity index 100% rename from doc/logos/macaddress_io.png rename to documentation/logos/macaddress_io.png diff --git a/doc/logos/macvendors.png b/documentation/logos/macvendors.png similarity index 100% rename from doc/logos/macvendors.png rename to documentation/logos/macvendors.png diff --git a/doc/logos/maxmind.png b/documentation/logos/maxmind.png similarity index 100% rename from doc/logos/maxmind.png rename to documentation/logos/maxmind.png diff --git a/doc/logos/nexthink.svg b/documentation/logos/nexthink.svg similarity index 100% rename from doc/logos/nexthink.svg rename to documentation/logos/nexthink.svg diff --git a/doc/logos/ods.png b/documentation/logos/ods.png similarity index 100% rename from doc/logos/ods.png rename to documentation/logos/ods.png diff --git a/doc/logos/odt.png b/documentation/logos/odt.png similarity index 100% rename from doc/logos/odt.png rename to documentation/logos/odt.png diff --git a/doc/logos/onyphe.jpg b/documentation/logos/onyphe.jpg similarity index 100% rename from doc/logos/onyphe.jpg rename to documentation/logos/onyphe.jpg diff --git a/doc/logos/osquery.png b/documentation/logos/osquery.png similarity index 100% rename from doc/logos/osquery.png rename to documentation/logos/osquery.png diff --git a/doc/logos/otx.png b/documentation/logos/otx.png similarity index 100% rename from doc/logos/otx.png rename to documentation/logos/otx.png diff --git a/doc/logos/passivedns.png b/documentation/logos/passivedns.png similarity index 100% rename from doc/logos/passivedns.png rename to documentation/logos/passivedns.png diff --git a/doc/logos/passivessl.png b/documentation/logos/passivessl.png similarity index 100% rename from doc/logos/passivessl.png rename to documentation/logos/passivessl.png diff --git a/doc/logos/passivetotal.png b/documentation/logos/passivetotal.png similarity index 100% rename from doc/logos/passivetotal.png rename to documentation/logos/passivetotal.png diff --git a/doc/logos/pdf.jpg b/documentation/logos/pdf.jpg similarity index 100% rename from doc/logos/pdf.jpg rename to documentation/logos/pdf.jpg diff --git a/doc/logos/pptx.png b/documentation/logos/pptx.png similarity index 100% rename from doc/logos/pptx.png rename to documentation/logos/pptx.png diff --git a/documentation/logos/recordedfuture.png b/documentation/logos/recordedfuture.png new file mode 100644 index 0000000..a208c04 Binary files /dev/null and b/documentation/logos/recordedfuture.png differ diff --git a/doc/logos/securitytrails.png b/documentation/logos/securitytrails.png similarity index 100% rename from doc/logos/securitytrails.png rename to documentation/logos/securitytrails.png diff --git a/doc/logos/shodan.png b/documentation/logos/shodan.png similarity index 100% rename from doc/logos/shodan.png rename to documentation/logos/shodan.png diff --git a/doc/logos/sigma.png b/documentation/logos/sigma.png similarity index 100% rename from doc/logos/sigma.png rename to documentation/logos/sigma.png diff --git a/documentation/logos/sophoslabs_intelix.svg b/documentation/logos/sophoslabs_intelix.svg new file mode 100644 index 0000000..9fe952f --- /dev/null +++ b/documentation/logos/sophoslabs_intelix.svg @@ -0,0 +1,32 @@ + + + + CC812F0D-F9F0-4D68-9347-3579CDA181A3 + Created with sketchtool. + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/logos/spamhaus.jpg b/documentation/logos/spamhaus.jpg similarity index 100% rename from doc/logos/spamhaus.jpg rename to documentation/logos/spamhaus.jpg diff --git a/doc/logos/stix.png b/documentation/logos/stix.png similarity index 100% rename from doc/logos/stix.png rename to documentation/logos/stix.png diff --git a/doc/logos/threatconnect.png b/documentation/logos/threatconnect.png similarity index 100% rename from doc/logos/threatconnect.png rename to documentation/logos/threatconnect.png diff --git a/doc/logos/threatcrowd.png b/documentation/logos/threatcrowd.png similarity index 100% rename from doc/logos/threatcrowd.png rename to documentation/logos/threatcrowd.png diff --git a/doc/logos/threatminer.png b/documentation/logos/threatminer.png similarity index 100% rename from doc/logos/threatminer.png rename to documentation/logos/threatminer.png diff --git a/doc/logos/threatstream.png b/documentation/logos/threatstream.png similarity index 100% rename from doc/logos/threatstream.png rename to documentation/logos/threatstream.png diff --git a/doc/logos/trustar.png b/documentation/logos/trustar.png similarity index 100% rename from doc/logos/trustar.png rename to documentation/logos/trustar.png diff --git a/doc/logos/urlhaus.png b/documentation/logos/urlhaus.png similarity index 100% rename from doc/logos/urlhaus.png rename to documentation/logos/urlhaus.png diff --git a/doc/logos/urlscan.jpg b/documentation/logos/urlscan.jpg similarity index 100% rename from doc/logos/urlscan.jpg rename to documentation/logos/urlscan.jpg diff --git a/doc/logos/virustotal.png b/documentation/logos/virustotal.png similarity index 100% rename from doc/logos/virustotal.png rename to documentation/logos/virustotal.png diff --git a/doc/logos/vmray.png b/documentation/logos/vmray.png similarity index 100% rename from doc/logos/vmray.png rename to documentation/logos/vmray.png diff --git a/doc/logos/vulndb.png b/documentation/logos/vulndb.png similarity index 100% rename from doc/logos/vulndb.png rename to documentation/logos/vulndb.png diff --git a/doc/logos/vulners.png b/documentation/logos/vulners.png similarity index 100% rename from doc/logos/vulners.png rename to documentation/logos/vulners.png diff --git a/doc/logos/wikidata.png b/documentation/logos/wikidata.png similarity index 100% rename from doc/logos/wikidata.png rename to documentation/logos/wikidata.png diff --git a/doc/logos/xforce.png b/documentation/logos/xforce.png similarity index 100% rename from doc/logos/xforce.png rename to documentation/logos/xforce.png diff --git a/doc/logos/xlsx.png b/documentation/logos/xlsx.png similarity index 100% rename from doc/logos/xlsx.png rename to documentation/logos/xlsx.png diff --git a/doc/logos/yara.png b/documentation/logos/yara.png similarity index 100% rename from doc/logos/yara.png rename to documentation/logos/yara.png diff --git a/docs/REQUIREMENTS.txt b/documentation/mkdocs/REQUIREMENTS.txt similarity index 100% rename from docs/REQUIREMENTS.txt rename to documentation/mkdocs/REQUIREMENTS.txt diff --git a/docs/contribute.md b/documentation/mkdocs/contribute.md similarity index 100% rename from docs/contribute.md rename to documentation/mkdocs/contribute.md diff --git a/documentation/mkdocs/img/favicon.ico b/documentation/mkdocs/img/favicon.ico new file mode 100644 index 0000000..dca12d1 Binary files /dev/null and b/documentation/mkdocs/img/favicon.ico differ diff --git a/documentation/mkdocs/img/misp.png b/documentation/mkdocs/img/misp.png new file mode 100644 index 0000000..5f2d4dd Binary files /dev/null and b/documentation/mkdocs/img/misp.png differ diff --git a/documentation/mkdocs/index.md b/documentation/mkdocs/index.md new file mode 100644 index 0000000..1297a3b --- /dev/null +++ b/documentation/mkdocs/index.md @@ -0,0 +1,120 @@ +# Home + +[![Build Status](https://travis-ci.org/MISP/misp-modules.svg?branch=master)](https://travis-ci.org/MISP/misp-modules) +[![Coverage Status](https://coveralls.io/repos/github/MISP/misp-modules/badge.svg?branch=master)](https://coveralls.io/github/MISP/misp-modules?branch=master) +[![codecov](https://codecov.io/gh/MISP/misp-modules/branch/master/graph/badge.svg)](https://codecov.io/gh/MISP/misp-modules) +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_shield) + +MISP modules are autonomous modules that can be used for expansion and other services in [MISP](https://github.com/MISP/MISP). + +The modules are written in Python 3 following a simple API interface. The objective is to ease the extensions of MISP functionalities +without modifying core components. The API is available via a simple REST API which is independent from MISP installation or configuration. + +MISP modules support is included in MISP starting from version `2.4.28`. + +For more information: [Extending MISP with Python modules](https://www.circl.lu/assets/files/misp-training/switch2016/2-misp-modules.pdf) slides from MISP training. + + +## Existing MISP modules + +### Expansion modules + +* [Backscatter.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/backscatter_io.py) - a hover and expansion module to expand an IP address with mass-scanning observations. +* [BGP Ranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. +* [BTC scam check](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused. +* [BTC transactions](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. +* [CIRCL Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. +* [CIRCL Passive SSL](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. +* [countrycode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to. +* [CrowdStrike Falcon](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/crowdstrike_falcon.py) - an expansion module to expand using CrowdStrike Falcon Intel Indicator API. +* [CVE](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve.py) - a hover module to give more information about a vulnerability (CVE). +* [CVE advanced](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cve_advanced.py) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). +* [Cuckoo submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cuckoo_submit.py) - A hover module to submit malware sample, url, attachment, domain to Cuckoo Sandbox. +* [DBL Spamhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dbl_spamhaus.py) - a hover module to check Spamhaus DBL for a domain name. +* [DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/dns.py) - a simple module to resolve MISP attributes like hostname and domain to expand IP addresses attributes. +* [docx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/docx-enrich.py) - an enrichment module to get text out of Word document into MISP (using free-text parser). +* [DomainTools](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/domaintools.py) - a hover and expansion module to get information from [DomainTools](http://www.domaintools.com/) whois. +* [EUPI](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/eupi.py) - a hover and expansion module to get information about an URL from the [Phishing Initiative project](https://phishing-initiative.eu/?lang=en). +* [EQL](misp_modules/modules/expansion/eql.py) - an expansion module to generate event query language (EQL) from an attribute. [Event Query Language](https://eql.readthedocs.io/en/latest/) +* [Farsight DNSDB Passive DNS](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/farsight_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. +* [GeoIP](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/geoip_country.py) - a hover and expansion module to get GeoIP information from geolite/maxmind. +* [Greynoise](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/greynoise.py) - a hover to get information from greynoise. +* [hashdd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hashdd.py) - a hover module to check file hashes against [hashdd.com](http://www.hashdd.com) including NSLR dataset. +* [hibp](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/hibp.py) - a hover module to lookup against Have I Been Pwned? +* [intel471](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/intel471.py) - an expansion module to get info from [Intel471](https://intel471.com). +* [IPASN](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) - a hover and expansion to get the BGP ASN of an IP address. +* [iprep](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/iprep.py) - an expansion module to get IP reputation from packetmail.net. +* [Joe Sandbox submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py) - Submit files and URLs to Joe Sandbox. +* [Joe Sandbox query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) - Query Joe Sandbox with the link of an analysis and get the parsed data. +* [macaddress.io](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macaddress_io.py) - a hover module to retrieve vendor details and other information regarding a given MAC address or an OUI from [MAC address Vendor Lookup](https://macaddress.io). See [integration tutorial here](https://macaddress.io/integrations/MISP-module). +* [macvendors](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/macvendors.py) - a hover module to retrieve mac vendor information. +* [ocr-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ocr-enrich.py) - an enrichment module to get OCRized data from images into MISP. +* [ods-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ods-enrich.py) - an enrichment module to get text out of OpenOffice spreadsheet document into MISP (using free-text parser). +* [odt-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/odt-enrich.py) - an enrichment module to get text out of OpenOffice document into MISP (using free-text parser). +* [onyphe](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe.py) - a modules to process queries on Onyphe. +* [onyphe_full](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/onyphe_full.py) - a modules to process full queries on Onyphe. +* [OTX](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/otx.py) - an expansion module for [OTX](https://otx.alienvault.com/). +* [passivetotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/passivetotal.py) - a [passivetotal](https://www.passivetotal.org/) module that queries a number of different PassiveTotal datasets. +* [pdf-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pdf-enrich.py) - an enrichment module to extract text from PDF into MISP (using free-text parser). +* [pptx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/pptx-enrich.py) - an enrichment module to get text out of PowerPoint document into MISP (using free-text parser). +* [qrcode](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/qrcode.py) - a module decode QR code, barcode and similar codes from an image and enrich with the decoded values. +* [rbl](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute. +* [reversedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes. +* [securitytrails](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/securitytrails.py) - an expansion module for [securitytrails](https://securitytrails.com/). +* [shodan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module. +* [Sigma queries](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures. +* [Sigma syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator. +* [sourcecache](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance. +* [STIX2 pattern syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax. +* [ThreatCrowd](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/). +* [threatminer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/). +* [urlhaus](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlhaus.py) - Query urlhaus to get additional data about a domain, hash, hostname, ip or url. +* [urlscan](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/urlscan.py) - an expansion module to query [urlscan.io](https://urlscan.io). +* [virustotal](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a high request rate limit required. (More details about the API: [here](https://developers.virustotal.com/reference)) +* [virustotal_public](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/virustotal_public.py) - an expansion module to query the [VirusTotal](https://www.virustotal.com/gui/home) API with a public key and a low request rate limit. (More details about the API: [here](https://developers.virustotal.com/reference)) +* [VMray](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vmray_submit.py) - a module to submit a sample to VMray. +* [VulnDB](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulndb.py) - a module to query [VulnDB](https://www.riskbasedsecurity.com/). +* [Vulners](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/vulners.py) - an expansion module to expand information about CVEs using Vulners API. +* [whois](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/whois.py) - a module to query a local instance of [uwhois](https://github.com/rafiot/uwhoisd). +* [wikidata](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/wiki.py) - a [wikidata](https://www.wikidata.org) expansion module. +* [xforce](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xforceexchange.py) - an IBM X-Force Exchange expansion module. +* [xlsx-enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/xlsx-enrich.py) - an enrichment module to get text out of an Excel document into MISP (using free-text parser). +* [YARA query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_query.py) - a module to create YARA rules from single hash attributes. +* [YARA syntax validator](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/yara_syntax_validator.py) - YARA syntax validator. + +### Export modules + +* [CEF](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). +* [Cisco FireSight Manager ACL rule](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/cisco_firesight_manager_ACL_rule_export.py) module to export as rule for the Cisco FireSight manager ACL. +* [GoAML export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). +* [Lite Export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/liteexport.py) module to export a lite event. +* [Mass EQL Export](misp_modules/modules/export_mod/mass_eql_export.py) module to export applicable attributes from an event to a mass EQL query. +* [PDF export](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. +* [Nexthink query format](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format. +* [osquery](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format. +* [ThreatConnect](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format. +* [ThreatStream](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/export_mod/threatStream_misp_export.py) module to export in ThreatStream format. + +### Import modules + +* [CSV import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/csvimport.py) Customizable CSV import module. +* [Cuckoo JSON](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/cuckooimport.py) Cuckoo JSON import. +* [Email Import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/email_import.py) Email import module for MISP to import basic metadata. +* [GoAML import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/goamlimport.py) Module to import [GoAML](http://goaml.unodc.org/goaml/en/index.html) XML format. +* [Joe Sandbox import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) Parse data from a Joe Sandbox json report. +* [OCR](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/ocr.py) Optical Character Recognition (OCR) module for MISP to import attributes from images, scan or faxes. +* [OpenIOC](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/openiocimport.py) OpenIOC import based on PyMISP library. +* [ThreatAnalyzer](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. +* [VMRay](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. + + +## How to contribute your own module? + +Fork the project, add your module, test it and make a pull-request. Modules can be also private as you can add a module in your own MISP installation. +For further information please see [Contribute](contribute/). + + +## Licenses +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%MISP%2Fmisp-modules.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2FMISP%2Fmisp-modules?ref=badge_large) + +For further Information see also the [license file](license/). \ No newline at end of file diff --git a/docs/install.md b/documentation/mkdocs/install.md similarity index 100% rename from docs/install.md rename to documentation/mkdocs/install.md diff --git a/docs/license.md b/documentation/mkdocs/license.md similarity index 100% rename from docs/license.md rename to documentation/mkdocs/license.md diff --git a/doc/expansion/apiosintds.json b/documentation/website/expansion/apiosintds.json similarity index 76% rename from doc/expansion/apiosintds.json rename to documentation/website/expansion/apiosintds.json index 81a1eec..8bdaf39 100644 --- a/doc/expansion/apiosintds.json +++ b/documentation/website/expansion/apiosintds.json @@ -1,8 +1,12 @@ { "description": "On demand query API for OSINT.digitalside.it project.", - "requirements": ["The apiosintDS python library to query the OSINT.digitalside.it API."], + "requirements": [ + "The apiosintDS python library to query the OSINT.digitalside.it API." + ], "input": "A domain, ip, url or hash attribute.", "output": "Hashes and urls resulting from the query to OSINT.digitalside.it", - "references": ["https://osint.digitalside.it/#About"], + "references": [ + "https://osint.digitalside.it/#About" + ], "features": "The module simply queries the API of OSINT.digitalside.it with a domain, ip, url or hash attribute.\n\nThe result of the query is then parsed to extract additional hashes or urls. A module parameters also allows to parse the hashes related to the urls.\n\nFurthermore, it is possible to cache the urls and hashes collected over the last 7 days by OSINT.digitalside.it" -} +} \ No newline at end of file diff --git a/doc/expansion/apivoid.json b/documentation/website/expansion/apivoid.json similarity index 72% rename from doc/expansion/apivoid.json rename to documentation/website/expansion/apivoid.json index 2173d5b..5962f57 100644 --- a/doc/expansion/apivoid.json +++ b/documentation/website/expansion/apivoid.json @@ -1,9 +1,13 @@ { "description": "Module to query APIVoid with some domain attributes.", - "logo": "logos/apivoid.png", - "requirements": ["A valid APIVoid API key with enough credits to proceed 2 queries"], + "logo": "apivoid.png", + "requirements": [ + "A valid APIVoid API key with enough credits to proceed 2 queries" + ], "input": "A domain attribute.", "output": "DNS records and SSL certificates related to the domain.", "features": "This module takes a domain name and queries API Void to get the related DNS records and the SSL certificates. It returns then those pieces of data as MISP objects that can be added to the event.\n\nTo make it work, a valid API key and enough credits to proceed 2 queries (0.06 + 0.07 credits) are required.", - "references": ["https://www.apivoid.com/"] -} + "references": [ + "https://www.apivoid.com/" + ] +} \ No newline at end of file diff --git a/doc/expansion/assemblyline_query.json b/documentation/website/expansion/assemblyline_query.json similarity index 78% rename from doc/expansion/assemblyline_query.json rename to documentation/website/expansion/assemblyline_query.json index 700bde0..4d54176 100644 --- a/doc/expansion/assemblyline_query.json +++ b/documentation/website/expansion/assemblyline_query.json @@ -1,9 +1,13 @@ { "description": "A module tu query the AssemblyLine API with a submission ID to get the submission report and parse it.", - "logo": "logos/assemblyline.png", - "requirements": ["assemblyline_client: Python library to query the AssemblyLine rest API."], + "logo": "assemblyline.png", + "requirements": [ + "assemblyline_client: Python library to query the AssemblyLine rest API." + ], "input": "Link of an AssemblyLine submission report.", "output": "MISP attributes & objects parsed from the AssemblyLine submission.", - "references": ["https://www.cyber.cg.ca/en/assemblyline"], + "references": [ + "https://www.cyber.cg.ca/en/assemblyline" + ], "features": "The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the used-ID and an API key or the password associated to the user-ID.\n\nThe submission ID extracted from the submission link is then used to query AssemblyLine and get the full submission report. This report is parsed to extract file objects and the associated IPs, domains or URLs the files are connecting to.\n\nSome more data may be parsed in the future." -} +} \ No newline at end of file diff --git a/doc/expansion/assemblyline_submit.json b/documentation/website/expansion/assemblyline_submit.json similarity index 72% rename from doc/expansion/assemblyline_submit.json rename to documentation/website/expansion/assemblyline_submit.json index 9fe9af6..8f147ca 100644 --- a/doc/expansion/assemblyline_submit.json +++ b/documentation/website/expansion/assemblyline_submit.json @@ -1,9 +1,13 @@ { "description": "A module to submit samples and URLs to AssemblyLine for advanced analysis, and return the link of the submission.", - "logo": "logos/assemblyline.png", - "requirements": ["assemblyline_client: Python library to query the AssemblyLine rest API."], + "logo": "assemblyline.png", + "requirements": [ + "assemblyline_client: Python library to query the AssemblyLine rest API." + ], "input": "Sample, or url to submit to AssemblyLine.", "output": "Link of the report generated in AssemblyLine.", - "references": ["https://www.cyber.gc.ca/en/assemblyline"], + "references": [ + "https://www.cyber.gc.ca/en/assemblyline" + ], "features": "The module requires the address of the AssemblyLine server you want to query as well as your credentials used for this instance. Credentials include the user-ID and an API key or the password associated to the user-ID.\n\nIf the sample or url is correctly submitted, you get then the link of the submission." -} +} \ No newline at end of file diff --git a/doc/expansion/backscatter_io.json b/documentation/website/expansion/backscatter_io.json similarity index 66% rename from doc/expansion/backscatter_io.json rename to documentation/website/expansion/backscatter_io.json index a8475c5..146e41c 100644 --- a/doc/expansion/backscatter_io.json +++ b/documentation/website/expansion/backscatter_io.json @@ -1,9 +1,13 @@ { "description": "Query backscatter.io (https://backscatter.io/).", - "requirements": ["backscatter python library"], - "features": "The module takes a source or destination IP address as input and displays the information known by backscatter.io.\n\n", - "logo": "logos/backscatter_io.png", - "references": ["https://pypi.org/project/backscatter/"], + "requirements": [ + "backscatter python library" + ], + "features": "The module takes a source or destination IP address as input and displays the information known by backscatter.io.", + "logo": "backscatter_io.png", + "references": [ + "https://pypi.org/project/backscatter/" + ], "input": "IP addresses.", "output": "Text containing a history of the IP addresses especially on scanning based on backscatter.io information ." } diff --git a/documentation/website/expansion/bgpranking.json b/documentation/website/expansion/bgpranking.json new file mode 100644 index 0000000..5b0383e --- /dev/null +++ b/documentation/website/expansion/bgpranking.json @@ -0,0 +1,12 @@ +{ + "description": "Query BGP Ranking (https://bgpranking-ng.circl.lu/).", + "requirements": [ + "pybgpranking python library" + ], + "features": "The module takes an AS number attribute as input and displays its description as well as its ranking position in BGP Ranking for a given day.", + "references": [ + "https://github.com/D4-project/BGP-Ranking/" + ], + "input": "Autonomous system number.", + "output": "An asn object with its related bgp-ranking object." +} diff --git a/doc/expansion/btc_scam_check.json b/documentation/website/expansion/btc_scam_check.json similarity index 57% rename from doc/expansion/btc_scam_check.json rename to documentation/website/expansion/btc_scam_check.json index 44fce03..01fe8ff 100644 --- a/doc/expansion/btc_scam_check.json +++ b/documentation/website/expansion/btc_scam_check.json @@ -1,9 +1,13 @@ { "description": "An expansion hover module to query a special dns blacklist to check if a bitcoin address has been abused.", - "requirements": ["dnspython3: dns python library"], + "requirements": [ + "dnspython3: dns python library" + ], "features": "The module queries a dns blacklist directly with the bitcoin address and get a response if the address has been abused.", - "logo": "logos/bitcoin.png", + "logo": "bitcoin.png", "input": "btc address attribute.", - "output" : "Text to indicate if the BTC address has been abused.", - "references": ["https://btcblack.it/"] -} + "output": "Text to indicate if the BTC address has been abused.", + "references": [ + "https://btcblack.it/" + ] +} \ No newline at end of file diff --git a/doc/expansion/btc_steroids.json b/documentation/website/expansion/btc_steroids.json similarity index 88% rename from doc/expansion/btc_steroids.json rename to documentation/website/expansion/btc_steroids.json index fd264d8..b365d44 100644 --- a/doc/expansion/btc_steroids.json +++ b/documentation/website/expansion/btc_steroids.json @@ -1,6 +1,6 @@ { "description": "An expansion hover module to get a blockchain balance from a BTC address in MISP.", - "logo": "logos/bitcoin.png", + "logo": "bitcoin.png", "input": "btc address attribute.", "output": "Text to describe the blockchain balance and the transactions related to the btc address in input." -} +} \ No newline at end of file diff --git a/doc/expansion/censys_enrich.json b/documentation/website/expansion/censys_enrich.json similarity index 76% rename from doc/expansion/censys_enrich.json rename to documentation/website/expansion/censys_enrich.json index 83e6d5f..9f3a6f0 100644 --- a/doc/expansion/censys_enrich.json +++ b/documentation/website/expansion/censys_enrich.json @@ -1,8 +1,12 @@ { "description": "An expansion module to enrich attributes in MISP by quering the censys.io API", - "requirements": ["API credentials to censys.io"], + "requirements": [ + "API credentials to censys.io" + ], "input": "IP, domain or certificate fingerprint (md5, sha1 or sha256)", "output": "MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details", - "references": ["https://www.censys.io"], + "references": [ + "https://www.censys.io" + ], "features": "This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API." -} +} \ No newline at end of file diff --git a/doc/expansion/circl_passivedns.json b/documentation/website/expansion/circl_passivedns.json similarity index 61% rename from doc/expansion/circl_passivedns.json rename to documentation/website/expansion/circl_passivedns.json index 024437c..b50136b 100644 --- a/doc/expansion/circl_passivedns.json +++ b/documentation/website/expansion/circl_passivedns.json @@ -1,9 +1,15 @@ { "description": "Module to access CIRCL Passive DNS.", - "logo": "logos/passivedns.png", - "requirements": ["pypdns: Passive DNS python library", "A CIRCL passive DNS account with username & password"], + "logo": "passivedns.png", + "requirements": [ + "pypdns: Passive DNS python library", + "A CIRCL passive DNS account with username & password" + ], "input": "Hostname, domain, or ip-address attribute.", "ouput": "Passive DNS objects related to the input attribute.", "features": "This module takes a hostname, domain or ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive DNS REST API to get the asssociated passive dns entries and return them as MISP objects.\n\nTo make it work a username and a password are thus required to authenticate to the CIRCL Passive DNS API.", - "references": ["https://www.circl.lu/services/passive-dns/", "https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/"] -} + "references": [ + "https://www.circl.lu/services/passive-dns/", + "https://datatracker.ietf.org/doc/draft-dulaunoy-dnsop-passive-dns-cof/" + ] +} \ No newline at end of file diff --git a/doc/expansion/circl_passivessl.json b/documentation/website/expansion/circl_passivessl.json similarity index 66% rename from doc/expansion/circl_passivessl.json rename to documentation/website/expansion/circl_passivessl.json index f9792e1..4010297 100644 --- a/doc/expansion/circl_passivessl.json +++ b/documentation/website/expansion/circl_passivessl.json @@ -1,9 +1,14 @@ { "description": "Modules to access CIRCL Passive SSL.", - "logo": "logos/passivessl.png", - "requirements": ["pypssl: Passive SSL python library", "A CIRCL passive SSL account with username & password"], + "logo": "passivessl.png", + "requirements": [ + "pypssl: Passive SSL python library", + "A CIRCL passive SSL account with username & password" + ], "input": "IP address attribute.", "output": "x509 certificate objects seen by the IP address(es).", "features": "This module takes an ip-address (ip-src or ip-dst) attribute as input, and queries the CIRCL Passive SSL REST API to gather the related certificates and return the corresponding MISP objects.\n\nTo make it work a username and a password are required to authenticate to the CIRCL Passive SSL API.", - "references": ["https://www.circl.lu/services/passive-ssl/"] -} + "references": [ + "https://www.circl.lu/services/passive-ssl/" + ] +} \ No newline at end of file diff --git a/doc/expansion/countrycode.json b/documentation/website/expansion/countrycode.json similarity index 99% rename from doc/expansion/countrycode.json rename to documentation/website/expansion/countrycode.json index c6214e5..110bdf7 100644 --- a/doc/expansion/countrycode.json +++ b/documentation/website/expansion/countrycode.json @@ -3,4 +3,4 @@ "input": "Hostname or domain attribute.", "output": "Text with the country code the input belongs to.", "features": "The module takes a domain or a hostname as input, and returns the country it belongs to.\n\nFor non country domains, a list of the most common possible extensions is used." -} +} \ No newline at end of file diff --git a/documentation/website/expansion/cpe.json b/documentation/website/expansion/cpe.json new file mode 100644 index 0000000..0160d1c --- /dev/null +++ b/documentation/website/expansion/cpe.json @@ -0,0 +1,10 @@ +{ + "description": "An expansion module to query the CVE search API with a cpe code to get its related vulnerabilities.", + "logo": "cve.png", + "input": "CPE attribute.", + "output": "The vulnerabilities related to the CPE.", + "references": [ + "https://cve.circl.lu/api/" + ], + "features": "The module takes a cpe attribute as input and queries the CVE search API to get its related vulnerabilities. \nThe list of vulnerabilities is then parsed and returned as vulnerability objects.\n\nUsers can use their own CVE search API url by defining a value to the custom_API_URL parameter. If no custom API url is given, the default cve.circl.lu api url is used.\n\nIn order to limit the amount of data returned by CVE serach, users can also the limit parameter. With the limit set, the API returns only the requested number of vulnerabilities, sorted from the highest cvss score to the lowest one." +} \ No newline at end of file diff --git a/doc/expansion/crowdstrike_falcon.json b/documentation/website/expansion/crowdstrike_falcon.json similarity index 83% rename from doc/expansion/crowdstrike_falcon.json rename to documentation/website/expansion/crowdstrike_falcon.json index 07e9dbd..a2408b9 100644 --- a/doc/expansion/crowdstrike_falcon.json +++ b/documentation/website/expansion/crowdstrike_falcon.json @@ -1,9 +1,13 @@ { "description": "Module to query Crowdstrike Falcon.", - "logo": "logos/crowdstrike.png", - "requirements": ["A CrowdStrike API access (API id & key)"], + "logo": "crowdstrike.png", + "requirements": [ + "A CrowdStrike API access (API id & key)" + ], "input": "A MISP attribute included in the following list:\n- domain\n- email-attachment\n- email-dst\n- email-reply-to\n- email-src\n- email-subject\n- filename\n- hostname\n- ip-src\n- ip-dst\n- md5\n- mutex\n- regkey\n- sha1\n- sha256\n- uri\n- url\n- user-agent\n- whois-registrant-email\n- x509-fingerprint-md5", "output": "MISP attributes mapped after the CrowdStrike API has been queried, included in the following list:\n- hostname\n- email-src\n- email-subject\n- filename\n- md5\n- sha1\n- sha256\n- ip-dst\n- ip-dst\n- mutex\n- regkey\n- url\n- user-agent\n- x509-fingerprint-md5", - "references": ["https://www.crowdstrike.com/products/crowdstrike-falcon-faq/"], + "references": [ + "https://www.crowdstrike.com/products/crowdstrike-falcon-faq/" + ], "features": "This module takes a MISP attribute as input to query a CrowdStrike Falcon API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes.\n\nPlease note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported." -} +} \ No newline at end of file diff --git a/doc/expansion/cuckoo_submit.json b/documentation/website/expansion/cuckoo_submit.json similarity index 62% rename from doc/expansion/cuckoo_submit.json rename to documentation/website/expansion/cuckoo_submit.json index 7fe8067..5c23218 100644 --- a/doc/expansion/cuckoo_submit.json +++ b/documentation/website/expansion/cuckoo_submit.json @@ -1,9 +1,14 @@ { "description": "An expansion module to submit files and URLs to Cuckoo Sandbox.", - "logo": "logos/cuckoo.png", - "requirements": ["Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)"], + "logo": "cuckoo.png", + "requirements": [ + "Access to a Cuckoo Sandbox API and an API key if the API requires it. (api_url and api_key)" + ], "input": "A malware-sample or attachment for files. A url or domain for URLs.", "output": "A text field containing 'Cuckoo task id: '", - "references": ["https://cuckoosandbox.org/", "https://cuckoo.sh/docs/"], + "references": [ + "https://cuckoosandbox.org/", + "https://cuckoo.sh/docs/" + ], "features": "The module takes a malware-sample, attachment, url or domain and submits it to Cuckoo Sandbox.\n The returned task id can be used to retrieve results when the analysis completed." -} +} \ No newline at end of file diff --git a/doc/expansion/cve.json b/documentation/website/expansion/cve.json similarity index 77% rename from doc/expansion/cve.json rename to documentation/website/expansion/cve.json index 04f131f..04f5733 100644 --- a/doc/expansion/cve.json +++ b/documentation/website/expansion/cve.json @@ -1,8 +1,11 @@ { "description": "An expansion hover module to expand information about CVE id.", - "logo": "logos/cve.png", + "logo": "cve.png", "input": "Vulnerability attribute.", "output": "Text giving information about the CVE related to the Vulnerability.", - "references": ["https://cve.circl.lu/", "https://cve.mitre.org/"], + "references": [ + "https://cve.circl.lu/", + "https://cve.mitre.org/" + ], "features": "The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to get information about the vulnerability as it is described in the list of CVEs." -} +} \ No newline at end of file diff --git a/documentation/website/expansion/cve_advanced.json b/documentation/website/expansion/cve_advanced.json new file mode 100644 index 0000000..364fb32 --- /dev/null +++ b/documentation/website/expansion/cve_advanced.json @@ -0,0 +1,11 @@ +{ + "description": "An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE).", + "logo": "cve.png", + "input": "Vulnerability attribute.", + "output": "Additional information about the vulnerability, such as its cvss score, some references, or the related weaknesses and attack patterns.", + "references": [ + "https://cve.circl.lu", + "https://cve/mitre.org/" + ], + "features": "The module takes a vulnerability attribute as input and queries the CIRCL CVE search API to gather additional information.\n\nThe result of the query is then parsed to return additional information about the vulnerability, like its cvss score or some references, as well as the potential related weaknesses and attack patterns.\n\nThe vulnerability additional data is returned in a vulnerability MISP object, and the related additional information are put into weakness and attack-pattern MISP objects." +} \ No newline at end of file diff --git a/doc/expansion/cytomic_orion.json b/documentation/website/expansion/cytomic_orion.json similarity index 62% rename from doc/expansion/cytomic_orion.json rename to documentation/website/expansion/cytomic_orion.json index 6f87657..8623670 100644 --- a/doc/expansion/cytomic_orion.json +++ b/documentation/website/expansion/cytomic_orion.json @@ -1,9 +1,14 @@ { "description": "An expansion module to enrich attributes in MISP by quering the Cytomic Orion API", - "logo": "logos/cytomic_orion.png", - "requirements": ["Access (license) to Cytomic Orion"], + "logo": "cytomic_orion.png", + "requirements": [ + "Access (license) to Cytomic Orion" + ], "input": "MD5, hash of the sample / malware to search for.", "output": "MISP objects with sightings of the hash in Cytomic Orion. Includes files and machines.", - "references": ["https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/", "https://www.cytomicmodel.com/solutions/"], + "references": [ + "https://www.vanimpe.eu/2020/03/10/integrating-misp-and-cytomic-orion/", + "https://www.cytomicmodel.com/solutions/" + ], "features": "This module takes an MD5 hash and searches for occurrences of this hash in the Cytomic Orion database. Returns observed files and machines." -} +} \ No newline at end of file diff --git a/doc/expansion/dbl_spamhaus.json b/documentation/website/expansion/dbl_spamhaus.json similarity index 76% rename from doc/expansion/dbl_spamhaus.json rename to documentation/website/expansion/dbl_spamhaus.json index ea73dcb..6a33c8e 100644 --- a/doc/expansion/dbl_spamhaus.json +++ b/documentation/website/expansion/dbl_spamhaus.json @@ -1,9 +1,13 @@ { "description": "Module to check Spamhaus DBL for a domain name.", - "logo": "logos/spamhaus.jpg", - "requirements": ["dnspython3: DNS python3 library"], + "logo": "spamhaus.jpg", + "requirements": [ + "dnspython3: DNS python3 library" + ], "input": "Domain or hostname attribute.", "output": "Information about the nature of the input.", - "references": ["https://www.spamhaus.org/faq/section/Spamhaus%20DBL"], + "references": [ + "https://www.spamhaus.org/faq/section/Spamhaus%20DBL" + ], "features": "This modules takes a domain or a hostname in input and queries the Domain Block List provided by Spamhaus to determine what kind of domain it is.\n\nDBL then returns a response code corresponding to a certain classification of the domain we display. If the queried domain is not in the list, it is also mentionned.\n\nPlease note that composite MISP attributes containing domain or hostname are supported as well." -} +} \ No newline at end of file diff --git a/doc/expansion/dns.json b/documentation/website/expansion/dns.json similarity index 90% rename from doc/expansion/dns.json rename to documentation/website/expansion/dns.json index dc43b64..a0fb4dd 100644 --- a/doc/expansion/dns.json +++ b/documentation/website/expansion/dns.json @@ -1,7 +1,9 @@ { "description": "A simple DNS expansion service to resolve IP address from domain MISP attributes.", - "requirements": ["dnspython3: DNS python3 library"], + "requirements": [ + "dnspython3: DNS python3 library" + ], "input": "Domain or hostname attribute.", "output": "IP address resolving the input.", "features": "The module takes a domain of hostname attribute as input, and tries to resolve it. If no error is encountered, the IP address that resolves the domain is returned, otherwise the origin of the error is displayed.\n\nThe address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8).\n\nPlease note that composite MISP attributes containing domain or hostname are supported as well." -} +} \ No newline at end of file diff --git a/doc/expansion/docx-enrich.json b/documentation/website/expansion/docx_enrich.json similarity index 82% rename from doc/expansion/docx-enrich.json rename to documentation/website/expansion/docx_enrich.json index fccba57..55bd955 100644 --- a/doc/expansion/docx-enrich.json +++ b/documentation/website/expansion/docx_enrich.json @@ -1,9 +1,11 @@ { "description": "Module to extract freetext from a .docx document.", - "logo": "logos/docx.png", - "requirements": ["docx python library"], + "logo": "docx.png", + "requirements": [ + "docx python library" + ], "input": "Attachment attribute containing a .docx document.", "output": "Text and freetext parsed from the document.", "references": [], "features": "The module reads the text contained in a .docx document. The result is passed to the freetext import parser so IoCs can be extracted out of it." -} +} \ No newline at end of file diff --git a/doc/expansion/domaintools.json b/documentation/website/expansion/domaintools.json similarity index 80% rename from doc/expansion/domaintools.json rename to documentation/website/expansion/domaintools.json index 849028c..99c916b 100644 --- a/doc/expansion/domaintools.json +++ b/documentation/website/expansion/domaintools.json @@ -1,9 +1,14 @@ { "description": "DomainTools MISP expansion module.", - "logo": "logos/domaintools.png", - "requirements": ["Domaintools python library", "A Domaintools API access (username & apikey)"], + "logo": "domaintools.png", + "requirements": [ + "Domaintools python library", + "A Domaintools API access (username & apikey)" + ], "input": "A MISP attribute included in the following list:\n- domain\n- hostname\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-name\n- whois-registrant-phone\n- ip-src\n- ip-dst", "output": "MISP attributes mapped after the Domaintools API has been queried, included in the following list:\n- whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- text\n- domain", - "references": ["https://www.domaintools.com/"], + "references": [ + "https://www.domaintools.com/" + ], "features": "This module takes a MISP attribute as input to query the Domaintools API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes.\n\nPlease note that composite attributes composed by at least one of the input types mentionned below (domains, IPs, hostnames) are also supported." -} +} \ No newline at end of file diff --git a/doc/expansion/eql.json b/documentation/website/expansion/eql.json similarity index 77% rename from doc/expansion/eql.json rename to documentation/website/expansion/eql.json index 1a32adf..4af9df4 100644 --- a/doc/expansion/eql.json +++ b/documentation/website/expansion/eql.json @@ -1,9 +1,11 @@ { "description": "EQL query generation for a MISP attribute.", - "logo": "logos/eql.png", + "logo": "eql.png", "requirements": [], "input": "A filename or ip attribute.", "output": "Attribute containing EQL for a network or file attribute.", - "references": ["https://eql.readthedocs.io/en/latest/"], + "references": [ + "https://eql.readthedocs.io/en/latest/" + ], "features": "This module adds a new attribute to a MISP event containing an EQL query for a network or file attribute." -} +} \ No newline at end of file diff --git a/doc/expansion/eupi.json b/documentation/website/expansion/eupi.json similarity index 71% rename from doc/expansion/eupi.json rename to documentation/website/expansion/eupi.json index 02a16fb..07eb59e 100644 --- a/doc/expansion/eupi.json +++ b/documentation/website/expansion/eupi.json @@ -1,9 +1,14 @@ { "description": "A module to query the Phishing Initiative service (https://phishing-initiative.lu).", - "logo": "logos/eupi.png", - "requirements": ["pyeupi: eupi python library", "An access to the Phishing Initiative API (apikey & url)"], + "logo": "eupi.png", + "requirements": [ + "pyeupi: eupi python library", + "An access to the Phishing Initiative API (apikey & url)" + ], "input": "A domain, hostname or url MISP attribute.", "output": "Text containing information about the input, resulting from the query on Phishing Initiative.", - "references": ["https://phishing-initiative.eu/?lang=en"], + "references": [ + "https://phishing-initiative.eu/?lang=en" + ], "features": "This module takes a domain, hostname or url MISP attribute as input to query the Phishing Initiative API. The API returns then the result of the query with some information about the value queried.\n\nPlease note that composite attributes containing domain or hostname are also supported." -} +} \ No newline at end of file diff --git a/documentation/website/expansion/farsight_passivedns.json b/documentation/website/expansion/farsight_passivedns.json new file mode 100644 index 0000000..93183ce --- /dev/null +++ b/documentation/website/expansion/farsight_passivedns.json @@ -0,0 +1,14 @@ +{ + "description": "Module to access Farsight DNSDB Passive DNS.", + "logo": "farsight.png", + "requirements": [ + "An access to the Farsight Passive DNS API (apikey)" + ], + "input": "A domain, hostname or IP address MISP attribute.", + "output": "Passive-dns objects, resulting from the query on the Farsight Passive DNS API.", + "references": [ + "https://www.farsightsecurity.com/", + "https://docs.dnsdb.info/dnsdb-api/" + ], + "features": "This module takes a domain, hostname or IP address MISP attribute as input to query the Farsight Passive DNS API.\n The results of rdata and rrset lookups are then returned and parsed into passive-dns objects.\n\nAn API key is required to submit queries to the API.\n It is also possible to define a custom server URL, and to set a limit of results to get.\n This limit is set for each lookup, which means we can have an up to the limit number of passive-dns objects resulting from an rdata query about an IP address, but an up to the limit number of passive-dns objects for each lookup queries about a domain or a hostname (== twice the limit)." +} diff --git a/documentation/website/expansion/geoip_asn.json b/documentation/website/expansion/geoip_asn.json new file mode 100644 index 0000000..9a7b1dd --- /dev/null +++ b/documentation/website/expansion/geoip_asn.json @@ -0,0 +1,13 @@ +{ + "descrption": "An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get information about its related AS number.", + "logo": "maxmind.png", + "requirements": [ + "A local copy of Maxmind's Geolite database" + ], + "input": "An IP address MISP attribute.", + "output": "Text containing information about the AS number of the IP address.", + "references": [ + "https://www.maxmind.com/en/home" + ], + "features": "The module takes an IP address attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the related AS number." +} \ No newline at end of file diff --git a/documentation/website/expansion/geoip_city.json b/documentation/website/expansion/geoip_city.json new file mode 100644 index 0000000..24d286b --- /dev/null +++ b/documentation/website/expansion/geoip_city.json @@ -0,0 +1,13 @@ +{ + "description": "An expansion module to query a local copy of Maxmind's Geolite database with an IP address, in order to get information about the city where it is located.", + "logo": "maxmind.png", + "requirements": [ + "A local copy of Maxmind's Geolite database" + ], + "input": "An IP address MISP attribute.", + "output": "Text containing information about the city where the IP address is located.", + "references": [ + "https://www.maxmind.com/en/home" + ], + "features": "The module takes an IP address attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the city where this IP address is located." +} \ No newline at end of file diff --git a/doc/expansion/geoip_country.json b/documentation/website/expansion/geoip_country.json similarity index 72% rename from doc/expansion/geoip_country.json rename to documentation/website/expansion/geoip_country.json index 9db49a2..ec84282 100644 --- a/doc/expansion/geoip_country.json +++ b/documentation/website/expansion/geoip_country.json @@ -1,9 +1,13 @@ { "description": "Module to query a local copy of Maxmind's Geolite database.", - "logo": "logos/maxmind.png", - "requirements": ["A local copy of Maxmind's Geolite database"], + "logo": "maxmind.png", + "requirements": [ + "A local copy of Maxmind's Geolite database" + ], "input": "An IP address MISP Attribute.", "output": "Text containing information about the location of the IP address.", - "references": ["https://www.maxmind.com/en/home"], + "references": [ + "https://www.maxmind.com/en/home" + ], "features": "This module takes an IP address MISP attribute as input and queries a local copy of the Maxmind's Geolite database to get information about the location of this IP address.\n\nPlease note that composite attributes domain|ip are also supported." -} +} \ No newline at end of file diff --git a/documentation/website/expansion/google_search.json b/documentation/website/expansion/google_search.json new file mode 100644 index 0000000..8772d21 --- /dev/null +++ b/documentation/website/expansion/google_search.json @@ -0,0 +1,13 @@ +{ + "descrption": "A hover module to get information about an url using a Google search.", + "logo": "google.png", + "requirements": [ + "The python Google Search API library" + ], + "input": "An url attribute.", + "output": "Text containing the result of a Google search on the input url.", + "references": [ + "https://github.com/abenassi/Google-Search-API" + ], + "features": "The module takes an url as input to query the Google search API. The result of the query is then return as raw text." +} \ No newline at end of file diff --git a/documentation/website/expansion/greynoise.json b/documentation/website/expansion/greynoise.json new file mode 100644 index 0000000..4c61727 --- /dev/null +++ b/documentation/website/expansion/greynoise.json @@ -0,0 +1,14 @@ +{ + "description": "Module to access GreyNoise.io API", + "logo": "greynoise.png", + "requirements": [ + "A Greynoise API key." + ], + "input": "An IP address.", + "output": "Additional information about the IP fetched from Greynoise API.", + "references": [ + "https://greynoise.io/", + "https://github.com/GreyNoise-Intelligence/api.greynoise.io" + ], + "features": "The module takes an IP address as input and queries Greynoise for some additional information about it: basically it checks whether a given IP address is \u201cInternet background noise\u201d, or has been observed scanning or attacking devices across the Internet. The result is returned as text." +} \ No newline at end of file diff --git a/doc/expansion/hashdd.json b/documentation/website/expansion/hashdd.json similarity index 86% rename from doc/expansion/hashdd.json rename to documentation/website/expansion/hashdd.json index d963820..2edc1d1 100644 --- a/doc/expansion/hashdd.json +++ b/documentation/website/expansion/hashdd.json @@ -2,6 +2,8 @@ "description": "A hover module to check hashes against hashdd.com including NSLR dataset.", "input": "A hash MISP attribute (md5).", "output": "Text describing the known level of the hash in the hashdd databases.", - "references": ["https://hashdd.com/"], + "references": [ + "https://hashdd.com/" + ], "features": "This module takes a hash attribute as input to check its known level, using the hashdd API. This information is then displayed." -} +} \ No newline at end of file diff --git a/doc/expansion/hibp.json b/documentation/website/expansion/hibp.json similarity index 83% rename from doc/expansion/hibp.json rename to documentation/website/expansion/hibp.json index 3c3ee54..a2b7b09 100644 --- a/doc/expansion/hibp.json +++ b/documentation/website/expansion/hibp.json @@ -1,9 +1,11 @@ { "description": "Module to access haveibeenpwned.com API.", - "logo": "logos/hibp.png", + "logo": "hibp.png", "requirements": [], "input": "An email address", "output": "Additional information about the email address.", - "references": ["https://haveibeenpwned.com/"], + "references": [ + "https://haveibeenpwned.com/" + ], "features": "The module takes an email address as input and queries haveibeenpwned.com API to find additional information about it. This additional information actually tells if any account using the email address has already been compromised in a data breach." -} +} \ No newline at end of file diff --git a/documentation/website/expansion/html_to_markdown.json b/documentation/website/expansion/html_to_markdown.json new file mode 100644 index 0000000..0864431 --- /dev/null +++ b/documentation/website/expansion/html_to_markdown.json @@ -0,0 +1,9 @@ +{ + "description": "Expansion module to fetch the html content from an url and convert it into markdown.", + "input": "URL attribute.", + "output": "Markdown content converted from the HTML fetched from the url.", + "requirements": [ + "The markdownify python library" + ], + "features": "The module take an URL as input and the HTML content is fetched from it. This content is then converted into markdown that is returned as text." +} \ No newline at end of file diff --git a/documentation/website/expansion/intel471.json b/documentation/website/expansion/intel471.json new file mode 100644 index 0000000..8935276 --- /dev/null +++ b/documentation/website/expansion/intel471.json @@ -0,0 +1,13 @@ +{ + "descrption": "An expansion module to query Intel471 in order to get additional information about a domain, ip address, email address, url or hash.", + "logo": "intel471.png", + "requirements": [ + "The intel471 python library" + ], + "input": "A MISP attribute whose type is included in the following list:\n- hostname\n- domain\n- url\n- ip-src\n- ip-dst\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-name\n- md5\n- sha1\n- sha256", + "output": "Freetext", + "references": [ + "https://public.intel471.com/" + ], + "features": "The module uses the Intel471 python library to query the Intel471 API with the value of the input attribute. The result of the query is then returned as freetext so the Freetext import parses it." +} \ No newline at end of file diff --git a/doc/expansion/intelmq_eventdb.json b/documentation/website/expansion/intelmq_eventdb.json similarity index 57% rename from doc/expansion/intelmq_eventdb.json rename to documentation/website/expansion/intelmq_eventdb.json index bc48414..ce2b12a 100644 --- a/doc/expansion/intelmq_eventdb.json +++ b/documentation/website/expansion/intelmq_eventdb.json @@ -1,9 +1,15 @@ { "description": "Module to access intelmqs eventdb.", - "logo": "logos/intelmq.png", - "requirements": ["psycopg2: Python library to support PostgreSQL", "An access to the IntelMQ database (username, password, hostname and database reference)"], + "logo": "intelmq.png", + "requirements": [ + "psycopg2: Python library to support PostgreSQL", + "An access to the IntelMQ database (username, password, hostname and database reference)" + ], "input": "A hostname, domain, IP address or AS attribute.", "output": "Text giving information about the input using IntelMQ database.", - "references": ["https://github.com/certtools/intelmq", "https://intelmq.readthedocs.io/en/latest/Developers-Guide/"], + "references": [ + "https://github.com/certtools/intelmq", + "https://intelmq.readthedocs.io/en/latest/Developers-Guide/" + ], "features": "/!\\ EXPERIMENTAL MODULE, some features may not work /!\\\n\nThis module takes a domain, hostname, IP address or Autonomous system MISP attribute as input to query the IntelMQ database. The result of the query gives then additional information about the input." -} +} \ No newline at end of file diff --git a/doc/expansion/ipasn.json b/documentation/website/expansion/ipasn.json similarity index 70% rename from doc/expansion/ipasn.json rename to documentation/website/expansion/ipasn.json index 8caed92..5f30608 100644 --- a/doc/expansion/ipasn.json +++ b/documentation/website/expansion/ipasn.json @@ -1,8 +1,12 @@ { "description": "Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).", - "requirements": ["pyipasnhistory: Python library to access IPASN-history instance"], + "requirements": [ + "pyipasnhistory: Python library to access IPASN-history instance" + ], "input": "An IP address MISP attribute.", "output": "Asn object(s) objects related to the IP address used as input.", - "references": ["https://github.com/D4-project/IPASN-History"], + "references": [ + "https://github.com/D4-project/IPASN-History" + ], "features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service. The result of the query is the latest asn related to the IP address, that is returned as a MISP object." -} +} \ No newline at end of file diff --git a/doc/expansion/iprep.json b/documentation/website/expansion/iprep.json similarity index 71% rename from doc/expansion/iprep.json rename to documentation/website/expansion/iprep.json index 95250e0..2e27304 100644 --- a/doc/expansion/iprep.json +++ b/documentation/website/expansion/iprep.json @@ -1,8 +1,12 @@ { "description": "Module to query IPRep data for IP addresses.", - "requirements": ["An access to the packetmail API (apikey)"], + "requirements": [ + "An access to the packetmail API (apikey)" + ], "input": "An IP address MISP attribute.", "output": "Text describing additional information about the input after a query on the IPRep API.", - "references": ["https://github.com/mahesh557/packetmail"], + "references": [ + "https://github.com/mahesh557/packetmail" + ], "features": "This module takes an IP address attribute as input and queries the database from packetmail.net to get some information about the reputation of the IP." -} +} \ No newline at end of file diff --git a/doc/expansion/joesandbox_query.json b/documentation/website/expansion/joesandbox_query.json similarity index 84% rename from doc/expansion/joesandbox_query.json rename to documentation/website/expansion/joesandbox_query.json index 1a94edb..12f2853 100644 --- a/doc/expansion/joesandbox_query.json +++ b/documentation/website/expansion/joesandbox_query.json @@ -1,9 +1,14 @@ { "description": "Query Joe Sandbox API with a submission url to get the json report and extract its data that is parsed and converted into MISP attributes and objects.\n\nThis url can by the way come from the result of the [joesandbox_submit expansion module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_submit.py).", - "logo": "logos/joesandbox.png", - "requirements": ["jbxapi: Joe Sandbox API python3 library"], + "logo": "joesandbox.png", + "requirements": [ + "jbxapi: Joe Sandbox API python3 library" + ], "input": "Link of a Joe Sandbox sample or url submission.", "output": "MISP attributes & objects parsed from the analysis report.", - "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], + "references": [ + "https://www.joesecurity.org", + "https://www.joesandbox.com/" + ], "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the import module [joe_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/joe_import.py) taking directly the json report as input.\n\nEven if the introspection will allow all kinds of links to call this module, obviously only the ones presenting a sample or url submission in the Joe Sandbox API will return results.\n\nTo make it work you will need to fill the 'apikey' configuration with your Joe Sandbox API key and provide a valid link as input." -} +} \ No newline at end of file diff --git a/doc/expansion/joesandbox_submit.json b/documentation/website/expansion/joesandbox_submit.json similarity index 77% rename from doc/expansion/joesandbox_submit.json rename to documentation/website/expansion/joesandbox_submit.json index ad59239..0ac454f 100644 --- a/doc/expansion/joesandbox_submit.json +++ b/documentation/website/expansion/joesandbox_submit.json @@ -1,9 +1,14 @@ { "description": "A module to submit files or URLs to Joe Sandbox for an advanced analysis, and return the link of the submission.", - "logo": "logos/joesandbox.png", - "requirements": ["jbxapi: Joe Sandbox API python3 library"], + "logo": "joesandbox.png", + "requirements": [ + "jbxapi: Joe Sandbox API python3 library" + ], "input": "Sample, url (or domain) to submit to Joe Sandbox for an advanced analysis.", "output": "Link of the report generated in Joe Sandbox.", - "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], + "references": [ + "https://www.joesecurity.org", + "https://www.joesandbox.com/" + ], "features": "The module requires a Joe Sandbox API key to submit files or URL, and returns the link of the submitted analysis.\n\nIt is then possible, when the analysis is completed, to query the Joe Sandbox API to get the data related to the analysis, using the [joesandbox_query module](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) directly on this submission link." -} +} \ No newline at end of file diff --git a/doc/expansion/lastline_query.json b/documentation/website/expansion/lastline_query.json similarity index 89% rename from doc/expansion/lastline_query.json rename to documentation/website/expansion/lastline_query.json index 6165890..611b514 100644 --- a/doc/expansion/lastline_query.json +++ b/documentation/website/expansion/lastline_query.json @@ -1,9 +1,11 @@ { "description": "Query Lastline with an analysis link and parse the report into MISP attributes and objects.\nThe analysis link can also be retrieved from the output of the [lastline_submit](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_submit.py) expansion module.", - "logo": "logos/lastline.png", + "logo": "lastline.png", "requirements": [], "input": "Link to a Lastline analysis.", "output": "MISP attributes and objects parsed from the analysis report.", - "references": ["https://www.lastline.com"], + "references": [ + "https://www.lastline.com" + ], "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_import](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/import_mod/lastline_import.py) import module." -} +} \ No newline at end of file diff --git a/doc/expansion/lastline_submit.json b/documentation/website/expansion/lastline_submit.json similarity index 85% rename from doc/expansion/lastline_submit.json rename to documentation/website/expansion/lastline_submit.json index d053f55..7c4387f 100644 --- a/doc/expansion/lastline_submit.json +++ b/documentation/website/expansion/lastline_submit.json @@ -1,9 +1,11 @@ { "description": "Module to submit a file or URL to Lastline.", - "logo": "logos/lastline.png", + "logo": "lastline.png", "requirements": [], "input": "File or URL to submit to Lastline.", "output": "Link to the report generated by Lastline.", - "references": ["https://www.lastline.com"], + "references": [ + "https://www.lastline.com" + ], "features": "The module requires a Lastline Analysis `api_token` and `key`.\nWhen the analysis is completed, it is possible to import the generated report by feeding the analysis link to the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) module." -} +} \ No newline at end of file diff --git a/doc/expansion/macaddress_io.json b/documentation/website/expansion/macaddress_io.json similarity index 60% rename from doc/expansion/macaddress_io.json rename to documentation/website/expansion/macaddress_io.json index 6bd2658..013564a 100644 --- a/doc/expansion/macaddress_io.json +++ b/documentation/website/expansion/macaddress_io.json @@ -1,9 +1,15 @@ { "description": "MISP hover module for macaddress.io", - "logo": "logos/macaddress_io.png", - "requirements": ["maclookup: macaddress.io python library", "An access to the macaddress.io API (apikey)"], + "logo": "macaddress_io.png", + "requirements": [ + "maclookup: macaddress.io python library", + "An access to the macaddress.io API (apikey)" + ], "input": "MAC address MISP attribute.", "output": "Text containing information on the MAC address fetched from a query on macaddress.io.", - "references": ["https://macaddress.io/", "https://github.com/CodeLineFi/maclookup-python"], + "references": [ + "https://macaddress.io/", + "https://github.com/CodeLineFi/maclookup-python" + ], "features": "This module takes a MAC address attribute as input and queries macaddress.io for additional information.\n\nThis information contains data about:\n- MAC address details\n- Vendor details\n- Block details" -} +} \ No newline at end of file diff --git a/doc/expansion/macvendors.json b/documentation/website/expansion/macvendors.json similarity index 73% rename from doc/expansion/macvendors.json rename to documentation/website/expansion/macvendors.json index cc10475..38c3588 100644 --- a/doc/expansion/macvendors.json +++ b/documentation/website/expansion/macvendors.json @@ -1,9 +1,12 @@ { "description": "Module to access Macvendors API.", - "logo": "logos/macvendors.png", + "logo": "macvendors.png", "requirements": [], "input": "A MAC address.", "output": "Additional information about the MAC address.", - "references": ["https://macvendors.com/", "https://macvendors.com/api"], + "references": [ + "https://macvendors.com/", + "https://macvendors.com/api" + ], "features": "The module takes a MAC address as input and queries macvendors.com for some information about it. The API returns the name of the vendor related to the address." -} +} \ No newline at end of file diff --git a/doc/expansion/malwarebazaar.json b/documentation/website/expansion/malwarebazaar.json similarity index 91% rename from doc/expansion/malwarebazaar.json rename to documentation/website/expansion/malwarebazaar.json index 2db6ad5..8c8228c 100644 --- a/doc/expansion/malwarebazaar.json +++ b/documentation/website/expansion/malwarebazaar.json @@ -3,6 +3,8 @@ "requirements": [], "input": "A hash attribute (md5, sha1 or sha256).", "output": "File object(s) related to the input attribute found on MALWAREbazaar databases.", - "references": ["https://bazaar.abuse.ch/"], + "references": [ + "https://bazaar.abuse.ch/" + ], "features": "The module takes a hash attribute as input and queries MALWAREbazaar's API to fetch additional data about it. The result, if the payload is known on the databases, is at least one file object describing the file the input hash is related to.\n\nThe module is using the new format of modules able to return object since the result is one or multiple MISP object(s)." -} +} \ No newline at end of file diff --git a/doc/expansion/ocr-enrich.json b/documentation/website/expansion/ocr_enrich.json similarity index 85% rename from doc/expansion/ocr-enrich.json rename to documentation/website/expansion/ocr_enrich.json index 8765b22..0e8f627 100644 --- a/doc/expansion/ocr-enrich.json +++ b/documentation/website/expansion/ocr_enrich.json @@ -1,8 +1,10 @@ { "description": "Module to process some optical character recognition on pictures.", - "requirements": ["cv2: The OpenCV python library."], + "requirements": [ + "cv2: The OpenCV python library." + ], "input": "A picture attachment.", "output": "Text and freetext fetched from the input picture.", "references": [], "features": "The module takes an attachment attributes as input and process some optical character recognition on it. The text found is then passed to the Freetext importer to extract potential IoCs." -} +} \ No newline at end of file diff --git a/doc/expansion/ods-enrich.json b/documentation/website/expansion/ods_enrich.json similarity index 65% rename from doc/expansion/ods-enrich.json rename to documentation/website/expansion/ods_enrich.json index dda4281..ade4105 100644 --- a/doc/expansion/ods-enrich.json +++ b/documentation/website/expansion/ods_enrich.json @@ -1,10 +1,12 @@ { "description": "Module to extract freetext from a .ods document.", - "logo": "logos/ods.png", - "requirements": ["ezodf: Python package to create/manipulate OpenDocumentFormat files.", - "pandas_ods_reader: Python library to read in ODS files."], + "logo": "ods.png", + "requirements": [ + "ezodf: Python package to create/manipulate OpenDocumentFormat files.", + "pandas_ods_reader: Python library to read in ODS files." + ], "input": "Attachment attribute containing a .ods document.", "output": "Text and freetext parsed from the document.", "references": [], "features": "The module reads the text contained in a .ods document. The result is passed to the freetext import parser so IoCs can be extracted out of it." -} +} \ No newline at end of file diff --git a/doc/expansion/odt-enrich.json b/documentation/website/expansion/odt_enrich.json similarity index 80% rename from doc/expansion/odt-enrich.json rename to documentation/website/expansion/odt_enrich.json index e201c77..8922a9b 100644 --- a/doc/expansion/odt-enrich.json +++ b/documentation/website/expansion/odt_enrich.json @@ -1,9 +1,11 @@ { "description": "Module to extract freetext from a .odt document.", - "logo": "logos/odt.png", - "requirements": ["ODT reader python library."], + "logo": "odt.png", + "requirements": [ + "ODT reader python library." + ], "input": "Attachment attribute containing a .odt document.", "output": "Text and freetext parsed from the document.", "references": [], "features": "The module reads the text contained in a .odt document. The result is passed to the freetext import parser so IoCs can be extracted out of it." -} +} \ No newline at end of file diff --git a/doc/expansion/onyphe.json b/documentation/website/expansion/onyphe.json similarity index 61% rename from doc/expansion/onyphe.json rename to documentation/website/expansion/onyphe.json index 04ebdd3..f38ea25 100644 --- a/doc/expansion/onyphe.json +++ b/documentation/website/expansion/onyphe.json @@ -1,9 +1,15 @@ { "description": "Module to process a query on Onyphe.", - "logo": "logos/onyphe.jpg", - "requirements": ["onyphe python library", "An access to the Onyphe API (apikey)"], + "logo": "onyphe.jpg", + "requirements": [ + "onyphe python library", + "An access to the Onyphe API (apikey)" + ], "input": "A domain, hostname or IP address MISP attribute.", "output": "MISP attributes fetched from the Onyphe query.", - "references": ["https://www.onyphe.io/", "https://github.com/sebdraven/pyonyphe"], + "references": [ + "https://www.onyphe.io/", + "https://github.com/sebdraven/pyonyphe" + ], "features": "This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted." -} +} \ No newline at end of file diff --git a/doc/expansion/onyphe_full.json b/documentation/website/expansion/onyphe_full.json similarity index 69% rename from doc/expansion/onyphe_full.json rename to documentation/website/expansion/onyphe_full.json index 4b722fa..e1a040a 100644 --- a/doc/expansion/onyphe_full.json +++ b/documentation/website/expansion/onyphe_full.json @@ -1,9 +1,15 @@ { "description": "Module to process a full query on Onyphe.", - "logo": "logos/onyphe.jpg", - "requirements": ["onyphe python library", "An access to the Onyphe API (apikey)"], + "logo": "onyphe.jpg", + "requirements": [ + "onyphe python library", + "An access to the Onyphe API (apikey)" + ], "input": "A domain, hostname or IP address MISP attribute.", "output": "MISP attributes fetched from the Onyphe query.", - "references": ["https://www.onyphe.io/", "https://github.com/sebdraven/pyonyphe"], + "references": [ + "https://www.onyphe.io/", + "https://github.com/sebdraven/pyonyphe" + ], "features": "This module takes a domain, hostname, or IP address attribute as input in order to query the Onyphe API. Data fetched from the query is then parsed and MISP attributes are extracted.\n\nThe parsing is here more advanced than the one on onyphe module, and is returning more attributes, since more fields of the query result are watched and parsed." -} +} \ No newline at end of file diff --git a/doc/expansion/otx.json b/documentation/website/expansion/otx.json similarity index 77% rename from doc/expansion/otx.json rename to documentation/website/expansion/otx.json index c6032cc..a17e2ff 100644 --- a/doc/expansion/otx.json +++ b/documentation/website/expansion/otx.json @@ -1,9 +1,13 @@ { "description": "Module to get information from AlienVault OTX.", - "logo": "logos/otx.png", - "requirements": ["An access to the OTX API (apikey)"], + "logo": "otx.png", + "requirements": [ + "An access to the OTX API (apikey)" + ], "input": "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512", "output": "MISP attributes mapped from the result of the query on OTX, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- email", - "references": ["https://www.alienvault.com/open-threat-exchange"], + "references": [ + "https://www.alienvault.com/open-threat-exchange" + ], "features": "This module takes a MISP attribute as input to query the OTX Alienvault API. The API returns then the result of the query with some types we map into compatible types we add as MISP attributes." -} +} \ No newline at end of file diff --git a/doc/expansion/passivetotal.json b/documentation/website/expansion/passivetotal.json similarity index 82% rename from doc/expansion/passivetotal.json rename to documentation/website/expansion/passivetotal.json index ef8b044..26835d5 100644 --- a/doc/expansion/passivetotal.json +++ b/documentation/website/expansion/passivetotal.json @@ -1,9 +1,14 @@ { "description": "", - "logo": "logos/passivetotal.png", - "requirements": ["Passivetotal python library", "An access to the PassiveTotal API (apikey)"], + "logo": "passivetotal.png", + "requirements": [ + "Passivetotal python library", + "An access to the PassiveTotal API (apikey)" + ], "input": "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- x509-fingerprint-sha1\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-phone\n- text\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date", "output": "MISP attributes mapped from the result of the query on PassiveTotal, included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- x509-fingerprint-sha1\n- email-src\n- email-dst\n- target-email\n- whois-registrant-email\n- whois-registrant-phone\n- text\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- md5\n- sha1\n- sha256\n- link", - "references": ["https://www.passivetotal.org/register"], + "references": [ + "https://www.passivetotal.org/register" + ], "features": "The PassiveTotal MISP expansion module brings the datasets derived from Internet scanning directly into your MISP instance. This module supports passive DNS, historic SSL, WHOIS, and host attributes. In order to use the module, you must have a valid PassiveTotal account username and API key. Registration is free and can be done by visiting https://www.passivetotal.org/register" -} +} \ No newline at end of file diff --git a/doc/expansion/pdf-enrich.json b/documentation/website/expansion/pdf_enrich.json similarity index 76% rename from doc/expansion/pdf-enrich.json rename to documentation/website/expansion/pdf_enrich.json index 5b3f0a8..a17ef51 100644 --- a/doc/expansion/pdf-enrich.json +++ b/documentation/website/expansion/pdf_enrich.json @@ -1,9 +1,11 @@ { "description": "Module to extract freetext from a PDF document.", - "logo": "logos/pdf.jpg", - "requirements": ["pdftotext: Python library to extract text from PDF."], + "logo": "pdf.jpg", + "requirements": [ + "pdftotext: Python library to extract text from PDF." + ], "input": "Attachment attribute containing a PDF document.", "output": "Text and freetext parsed from the document.", "references": [], "features": "The module reads the text contained in a PDF document. The result is passed to the freetext import parser so IoCs can be extracted out of it." -} +} \ No newline at end of file diff --git a/doc/expansion/pptx-enrich.json b/documentation/website/expansion/pptx_enrich.json similarity index 77% rename from doc/expansion/pptx-enrich.json rename to documentation/website/expansion/pptx_enrich.json index aff0d8d..664c70a 100644 --- a/doc/expansion/pptx-enrich.json +++ b/documentation/website/expansion/pptx_enrich.json @@ -1,9 +1,11 @@ { "description": "Module to extract freetext from a .pptx document.", - "logo": "logos/pptx.png", - "requirements": ["pptx: Python library to read PowerPoint files."], + "logo": "pptx.png", + "requirements": [ + "pptx: Python library to read PowerPoint files." + ], "input": "Attachment attribute containing a .pptx document.", "output": "Text and freetext parsed from the document.", "references": [], "features": "The module reads the text contained in a .pptx document. The result is passed to the freetext import parser so IoCs can be extracted out of it." -} +} \ No newline at end of file diff --git a/doc/expansion/qrcode.json b/documentation/website/expansion/qrcode.json similarity index 72% rename from doc/expansion/qrcode.json rename to documentation/website/expansion/qrcode.json index 38ed77c..f585511 100644 --- a/doc/expansion/qrcode.json +++ b/documentation/website/expansion/qrcode.json @@ -1,9 +1,11 @@ { "description": "Module to decode QR codes.", - "requirements": ["cv2: The OpenCV python library.", - "pyzbar: Python library to read QR codes."], + "requirements": [ + "cv2: The OpenCV python library.", + "pyzbar: Python library to read QR codes." + ], "input": "A QR code stored as attachment attribute.", "output": "The URL or bitcoin address the QR code is pointing to.", "references": [], "features": "The module reads the QR code and returns the related address, which can be an URL or a bitcoin address." -} +} \ No newline at end of file diff --git a/documentation/website/expansion/ransomcoindb.json b/documentation/website/expansion/ransomcoindb.json new file mode 100644 index 0000000..26c3c55 --- /dev/null +++ b/documentation/website/expansion/ransomcoindb.json @@ -0,0 +1,12 @@ +{ + "descrption": "Module to access the ransomcoinDB with a hash or btc address attribute and get the associated btc address of hashes.", + "requirements": [ + "A ransomcoinDB API key." + ], + "input": "A hash (md5, sha1 or sha256) or btc attribute.", + "output": "Hashes associated to a btc address or btc addresses associated to a hash.", + "references": [ + "https://ransomcoindb.concinnity-risks.com" + ], + "features": "The module takes either a hash attribute or a btc attribute as input to query the ransomcoinDB API for some additional data.\n\nIf the input is a btc address, we will get the associated hashes returned in a file MISP object. If we query ransomcoinDB with a hash, the response contains the associated btc addresses returned as single MISP btc attributes." +} \ No newline at end of file diff --git a/doc/expansion/rbl.json b/documentation/website/expansion/rbl.json similarity index 65% rename from doc/expansion/rbl.json rename to documentation/website/expansion/rbl.json index 9700eca..942daa7 100644 --- a/doc/expansion/rbl.json +++ b/documentation/website/expansion/rbl.json @@ -1,8 +1,12 @@ { "description": "Module to check an IPv4 address against known RBLs.", - "requirements": ["dnspython3: DNS python3 library"], + "requirements": [ + "dnspython3: DNS python3 library" + ], "input": "IP address attribute.", "output": "Text with additional data from Real-time Blackhost Lists about the IP address.", - "references": ["[RBLs list](https://github.com/MISP/misp-modules/blob/8817de476572a10a9c9d03258ec81ca70f3d926d/misp_modules/modules/expansion/rbl.py#L20)"], + "references": [ + "[RBLs list](https://github.com/MISP/misp-modules/blob/8817de476572a10a9c9d03258ec81ca70f3d926d/misp_modules/modules/expansion/rbl.py#L20)" + ], "features": "This module takes an IP address attribute as input and queries multiple know Real-time Blackhost Lists to check if they have already seen this IP address.\n\nWe display then all the information we get from those different sources." -} +} \ No newline at end of file diff --git a/documentation/website/expansion/recordedfuture.json b/documentation/website/expansion/recordedfuture.json new file mode 100644 index 0000000..91cf23e --- /dev/null +++ b/documentation/website/expansion/recordedfuture.json @@ -0,0 +1,13 @@ +{ + "description": "Module to enrich attributes with threat intelligence from Recorded Future.", + "logo": "recordedfuture.png", + "requirements": [ + "A Recorded Future API token." + ], + "input": "A MISP attribute of one of the following types: ip, ip-src, ip-dst, domain, hostname, md5, sha1, sha256, uri, url, vulnerability, weakness.", + "output": "A MISP object containing a copy of the enriched attribute with added tags from Recorded Future and a list of new attributes related to the enriched attribute.", + "references": [ + "https://www.recordedfuture.com/" + ], + "features": "Enrich an attribute to add a custom enrichment object to the event. The object contains a copy of the enriched attribute with added tags presenting risk score and triggered risk rules from Recorded Future. Malware and Threat Actors related to the enriched indicator in Recorded Future is matched against MISP's galaxy clusters and applied as galaxy tags. The custom enrichment object also includes a list of related indicators from Recorded Future (IP's, domains, hashes, URL's and vulnerabilities) added as additional attributes." +} \ No newline at end of file diff --git a/doc/expansion/reversedns.json b/documentation/website/expansion/reversedns.json similarity index 90% rename from doc/expansion/reversedns.json rename to documentation/website/expansion/reversedns.json index 6934462..cdd3419 100644 --- a/doc/expansion/reversedns.json +++ b/documentation/website/expansion/reversedns.json @@ -1,7 +1,9 @@ { "description": "Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.", - "requirements": ["DNS python library"], + "requirements": [ + "DNS python library" + ], "input": "An IP address attribute.", "output": "Hostname attribute the input is resolved into.", "features": "The module takes an IP address as input and tries to find the hostname this IP address is resolved into.\n\nThe address of the DNS resolver to use is also configurable, but if no configuration is set, we use the Google public DNS address (8.8.8.8).\n\nPlease note that composite MISP attributes containing IP addresses are supported as well." -} +} \ No newline at end of file diff --git a/doc/expansion/securitytrails.json b/documentation/website/expansion/securitytrails.json similarity index 77% rename from doc/expansion/securitytrails.json rename to documentation/website/expansion/securitytrails.json index 8541e4e..97f81b4 100644 --- a/doc/expansion/securitytrails.json +++ b/documentation/website/expansion/securitytrails.json @@ -1,9 +1,14 @@ { "description": "An expansion modules for SecurityTrails.", - "logo": "logos/securitytrails.png", - "requirements": ["dnstrails python library", "An access to the SecurityTrails API (apikey)"], + "logo": "securitytrails.png", + "requirements": [ + "dnstrails python library", + "An access to the SecurityTrails API (apikey)" + ], "input": "A domain, hostname or IP address attribute.", "output": "MISP attributes resulting from the query on SecurityTrails API, included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- dns-soa-email\n- whois-registrant-email\n- whois-registrant-phone\n- whois-registrant-name\n- whois-registrar\n- whois-creation-date\n- domain", - "references": ["https://securitytrails.com/"], + "references": [ + "https://securitytrails.com/" + ], "features": "The module takes a domain, hostname or IP address attribute as input and queries the SecurityTrails API with it.\n\nMultiple parsing operations are then processed on the result of the query to extract a much information as possible.\n\nFrom this data extracted are then mapped MISP attributes." -} +} \ No newline at end of file diff --git a/doc/expansion/shodan.json b/documentation/website/expansion/shodan.json similarity index 61% rename from doc/expansion/shodan.json rename to documentation/website/expansion/shodan.json index 57241f0..703a084 100644 --- a/doc/expansion/shodan.json +++ b/documentation/website/expansion/shodan.json @@ -1,9 +1,14 @@ { "description": "Module to query on Shodan.", - "logo": "logos/shodan.png", - "requirements": ["shodan python library", "An access to the Shodan API (apikey)"], + "logo": "shodan.png", + "requirements": [ + "shodan python library", + "An access to the Shodan API (apikey)" + ], "input": "An IP address MISP attribute.", "output": "Text with additional data about the input, resulting from the query on Shodan.", - "references": ["https://www.shodan.io/"], + "references": [ + "https://www.shodan.io/" + ], "features": "The module takes an IP address as input and queries the Shodan API to get some additional data about it." -} +} \ No newline at end of file diff --git a/doc/expansion/sigma_queries.json b/documentation/website/expansion/sigma_queries.json similarity index 69% rename from doc/expansion/sigma_queries.json rename to documentation/website/expansion/sigma_queries.json index f127ba4..c967112 100644 --- a/doc/expansion/sigma_queries.json +++ b/documentation/website/expansion/sigma_queries.json @@ -1,9 +1,13 @@ { "description": "An expansion hover module to display the result of sigma queries.", - "logo": "logos/sigma.png", - "requirements": ["Sigma python library"], + "logo": "sigma.png", + "requirements": [ + "Sigma python library" + ], "input": "A Sigma attribute.", "output": "Text displaying results of queries on the Sigma attribute.", - "references": ["https://github.com/Neo23x0/sigma/wiki"], + "references": [ + "https://github.com/Neo23x0/sigma/wiki" + ], "features": "This module takes a Sigma rule attribute as input and tries all the different queries available to convert it into different formats recognized by SIEMs." -} +} \ No newline at end of file diff --git a/doc/expansion/sigma_syntax_validator.json b/documentation/website/expansion/sigma_syntax_validator.json similarity index 67% rename from doc/expansion/sigma_syntax_validator.json rename to documentation/website/expansion/sigma_syntax_validator.json index 8e17ae0..b90c931 100644 --- a/doc/expansion/sigma_syntax_validator.json +++ b/documentation/website/expansion/sigma_syntax_validator.json @@ -1,9 +1,14 @@ { "description": "An expansion hover module to perform a syntax check on sigma rules.", - "logo": "logos/sigma.png", - "requirements": ["Sigma python library", "Yaml python library"], + "logo": "sigma.png", + "requirements": [ + "Sigma python library", + "Yaml python library" + ], "input": "A Sigma attribute.", "output": "Text describing the validity of the Sigma rule.", - "references": ["https://github.com/Neo23x0/sigma/wiki"], + "references": [ + "https://github.com/Neo23x0/sigma/wiki" + ], "features": "This module takes a Sigma rule attribute as input and performs a syntax check on it.\n\nIt displays then that the rule is valid if it is the case, and the error related to the rule otherwise." -} +} \ No newline at end of file diff --git a/documentation/website/expansion/socialscan.json b/documentation/website/expansion/socialscan.json new file mode 100644 index 0000000..a1cf359 --- /dev/null +++ b/documentation/website/expansion/socialscan.json @@ -0,0 +1,8 @@ +{ + "description": "A hover module to get information on the availability of an email address or username on some online platforms.", + "requirements": ["The socialscan python library"], + "input": "An email address or usename attribute.", + "output": "Text containing information about the availability of an email address or a username in some online platforms.", + "references": ["https://github.com/iojw/socialscan"], + "features": "The module takes an email address or username as input and check its availability on some online platforms. The results for each platform are then returned to see if the email address or the username is used, available or if there is an issue with it." +} diff --git a/documentation/website/expansion/sophoslabs_intelix.json b/documentation/website/expansion/sophoslabs_intelix.json new file mode 100644 index 0000000..8871192 --- /dev/null +++ b/documentation/website/expansion/sophoslabs_intelix.json @@ -0,0 +1,13 @@ +{ + "description": "An expansion module to query the Sophoslabs intelix API to get additional information about an ip address, url, domain or sha256 attribute.", + "logo": "sophoslabs_intelix.svg", + "requirements": [ + "A client_id and client_secret pair to authenticate to the SophosLabs Intelix API" + ], + "input": "An ip address, url, domain or sha256 attribute.", + "output": "SophosLabs Intelix report and lookup objects", + "references": [ + "https://aws.amazon.com/marketplace/pp/B07SLZPMCS" + ], + "features": "The module takes an ip address, url, domain or sha256 attribute and queries the SophosLabs Intelix API with the attribute value. The result of this query is a SophosLabs Intelix hash report, or an ip or url lookup, that is then parsed and returned in a MISP object." +} \ No newline at end of file diff --git a/doc/expansion/sourcecache.json b/documentation/website/expansion/sourcecache.json similarity index 67% rename from doc/expansion/sourcecache.json rename to documentation/website/expansion/sourcecache.json index ab4669c..4340f2c 100644 --- a/doc/expansion/sourcecache.json +++ b/documentation/website/expansion/sourcecache.json @@ -1,8 +1,12 @@ { "description": "Module to cache web pages of analysis reports, OSINT sources. The module returns a link of the cached page.", - "requirements": ["urlarchiver: python library to fetch and archive URL on the file-system"], + "requirements": [ + "urlarchiver: python library to fetch and archive URL on the file-system" + ], "input": "A link or url attribute.", "output": "A malware-sample attribute describing the cached page.", - "references": ["https://github.com/adulau/url_archiver"], + "references": [ + "https://github.com/adulau/url_archiver" + ], "features": "This module takes a link or url attribute as input and caches the related web page. It returns then a link of the cached page." -} +} \ No newline at end of file diff --git a/doc/expansion/stix2_pattern_syntax_validator.json b/documentation/website/expansion/stix2_pattern_syntax_validator.json similarity index 60% rename from doc/expansion/stix2_pattern_syntax_validator.json rename to documentation/website/expansion/stix2_pattern_syntax_validator.json index 2ea43b5..0ac079d 100644 --- a/doc/expansion/stix2_pattern_syntax_validator.json +++ b/documentation/website/expansion/stix2_pattern_syntax_validator.json @@ -1,9 +1,13 @@ { "description": "An expansion hover module to perform a syntax check on stix2 patterns.", - "logo": "logos/stix.png", - "requirements": ["stix2patterns python library"], + "logo": "stix.png", + "requirements": [ + "stix2patterns python library" + ], "input": "A STIX2 pattern attribute.", "output": "Text describing the validity of the STIX2 pattern.", - "references": ["[STIX2.0 patterning specifications](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html)"], + "references": [ + "[STIX2.0 patterning specifications](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html)" + ], "features": "This module takes a STIX2 pattern attribute as input and performs a syntax check on it.\n\nIt displays then that the rule is valid if it is the case, and the error related to the rule otherwise." -} +} \ No newline at end of file diff --git a/doc/expansion/threatcrowd.json b/documentation/website/expansion/threatcrowd.json similarity index 87% rename from doc/expansion/threatcrowd.json rename to documentation/website/expansion/threatcrowd.json index 99725b8..e279ece 100644 --- a/doc/expansion/threatcrowd.json +++ b/documentation/website/expansion/threatcrowd.json @@ -1,8 +1,10 @@ { "description": "Module to get information from ThreatCrowd.", - "logo": "logos/threatcrowd.png", + "logo": "threatcrowd.png", "input": "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512\n- whois-registrant-email", "output": "MISP attributes mapped from the result of the query on ThreatCrowd, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- hostname\n- whois-registrant-email", - "references": ["https://www.threatcrowd.org/"], + "references": [ + "https://www.threatcrowd.org/" + ], "features": "This module takes a MISP attribute as input and queries ThreatCrowd with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." -} +} \ No newline at end of file diff --git a/doc/expansion/threatminer.json b/documentation/website/expansion/threatminer.json similarity index 87% rename from doc/expansion/threatminer.json rename to documentation/website/expansion/threatminer.json index d2f26bd..0b0d641 100644 --- a/doc/expansion/threatminer.json +++ b/documentation/website/expansion/threatminer.json @@ -1,8 +1,10 @@ { "description": "Module to get information from ThreatMiner.", - "logo": "logos/threatminer.png", + "logo": "threatminer.png", "input": "A MISP attribute included in the following list:\n- hostname\n- domain\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- sha512", "output": "MISP attributes mapped from the result of the query on ThreatMiner, included in the following list:\n- domain\n- ip-src\n- ip-dst\n- text\n- md5\n- sha1\n- sha256\n- sha512\n- ssdeep\n- authentihash\n- filename\n- whois-registrant-email\n- url\n- link", - "references": ["https://www.threatminer.org/"], + "references": [ + "https://www.threatminer.org/" + ], "features": "This module takes a MISP attribute as input and queries ThreatMiner with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." -} +} \ No newline at end of file diff --git a/doc/expansion/trustar_enrich.json b/documentation/website/expansion/trustar_enrich.json similarity index 81% rename from doc/expansion/trustar_enrich.json rename to documentation/website/expansion/trustar_enrich.json index 294419d..415f52d 100644 --- a/doc/expansion/trustar_enrich.json +++ b/documentation/website/expansion/trustar_enrich.json @@ -1,8 +1,10 @@ { "description": "Module to get enrich indicators with TruSTAR.", - "logo": "logos/trustar.png", + "logo": "trustar.png", "input": "Any of the following MISP attributes:\n- btc\n- domain\n- email-src\n- filename\n- hostname\n- ip-src\n- ip-dst\n- md5\n- sha1\n- sha256\n- url", "output": "MISP attributes enriched with indicator summary data from the TruSTAR API. Data includes a severity level score and additional source and scoring info.", - "references": ["https://docs.trustar.co/api/v13/indicators/get_indicator_summaries.html"], + "references": [ + "https://docs.trustar.co/api/v13/indicators/get_indicator_summaries.html" + ], "features": "This module enriches MISP attributes with scoring and metadata from TruSTAR.\n\nThe TruSTAR indicator summary is appended to the attributes along with links to any associated reports." -} +} \ No newline at end of file diff --git a/doc/expansion/urlhaus.json b/documentation/website/expansion/urlhaus.json similarity index 86% rename from doc/expansion/urlhaus.json rename to documentation/website/expansion/urlhaus.json index 8e5cef3..cd59661 100644 --- a/doc/expansion/urlhaus.json +++ b/documentation/website/expansion/urlhaus.json @@ -1,9 +1,11 @@ { "description": "Query of the URLhaus API to get additional information about the input attribute.", - "logo": "logos/urlhaus.png", + "logo": "urlhaus.png", "requirements": [], "input": "A domain, hostname, url, ip, md5 or sha256 attribute.", "output": "MISP attributes & objects fetched from the result of the URLhaus API query.", - "references": ["https://urlhaus.abuse.ch/"], + "references": [ + "https://urlhaus.abuse.ch/" + ], "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module takes one of the attribute type specified as input, and query the URLhaus API with it. If any result is returned by the API, attributes and objects are created accordingly." -} +} \ No newline at end of file diff --git a/doc/expansion/urlscan.json b/documentation/website/expansion/urlscan.json similarity index 73% rename from doc/expansion/urlscan.json rename to documentation/website/expansion/urlscan.json index d847761..3aab2ab 100644 --- a/doc/expansion/urlscan.json +++ b/documentation/website/expansion/urlscan.json @@ -1,9 +1,13 @@ { "description": "An expansion module to query urlscan.io.", - "logo": "logos/urlscan.jpg", - "requirements": ["An access to the urlscan.io API"], + "logo": "urlscan.jpg", + "requirements": [ + "An access to the urlscan.io API" + ], "input": "A domain, hostname or url attribute.", "output": "MISP attributes mapped from the result of the query on urlscan.io.", - "references": ["https://urlscan.io/"], + "references": [ + "https://urlscan.io/" + ], "features": "This module takes a MISP attribute as input and queries urlscan.io with it.\n\nThe result of this query is then parsed and some data is mapped into MISP attributes in order to enrich the input attribute." -} +} \ No newline at end of file diff --git a/doc/expansion/virustotal.json b/documentation/website/expansion/virustotal.json similarity index 80% rename from doc/expansion/virustotal.json rename to documentation/website/expansion/virustotal.json index 31fd6ac..85c036f 100644 --- a/doc/expansion/virustotal.json +++ b/documentation/website/expansion/virustotal.json @@ -1,9 +1,14 @@ { "description": "Module to get advanced information from virustotal.", - "logo": "logos/virustotal.png", - "requirements": ["An access to the VirusTotal API (apikey), with a high request rate limit."], + "logo": "virustotal.png", + "requirements": [ + "An access to the VirusTotal API (apikey), with a high request rate limit." + ], "input": "A domain, hash (md5, sha1, sha256 or sha512), hostname or IP address attribute.", "output": "MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute.", - "references": ["https://www.virustotal.com/", "https://developers.virustotal.com/reference"], + "references": [ + "https://www.virustotal.com/", + "https://developers.virustotal.com/reference" + ], "features": "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [standard VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal_public.py), this module is made for advanced parsing of VirusTotal report, with a recursive analysis of the elements found after the first request.\n\nThus, it requires a higher request rate limit to avoid the API to return a 204 error (Request rate limit exceeded), and the data parsed from the different requests are returned as MISP attributes and objects, with the corresponding relations between each one of them." -} +} \ No newline at end of file diff --git a/doc/expansion/virustotal_public.json b/documentation/website/expansion/virustotal_public.json similarity index 78% rename from doc/expansion/virustotal_public.json rename to documentation/website/expansion/virustotal_public.json index 242c734..2b9df12 100644 --- a/doc/expansion/virustotal_public.json +++ b/documentation/website/expansion/virustotal_public.json @@ -1,9 +1,14 @@ { "description": "Module to get information from VirusTotal.", - "logo": "logos/virustotal.png", - "requirements": ["An access to the VirusTotal API (apikey)"], + "logo": "virustotal.png", + "requirements": [ + "An access to the VirusTotal API (apikey)" + ], "input": "A domain, hostname, ip, url or hash (md5, sha1, sha256 or sha512) attribute.", "output": "MISP attributes and objects resulting from the parsing of the VirusTotal report concerning the input attribute.", - "references": ["https://www.virustotal.com", "https://developers.virustotal.com/reference"], + "references": [ + "https://www.virustotal.com", + "https://developers.virustotal.com/reference" + ], "features": "New format of modules able to return attributes and objects.\n\nA module to take a MISP attribute as input and query the VirusTotal API to get additional data about it.\n\nCompared to the [more advanced VirusTotal expansion module](https://github.com/MISP/misp-modules/blob/master/misp_modules/modules/expansion/virustotal.py), this module is made for VirusTotal users who have a low request rate limit.\n\nThus, it only queries the API once and returns the results that is parsed into MISP attributes and objects." -} +} \ No newline at end of file diff --git a/doc/expansion/vmray_submit.json b/documentation/website/expansion/vmray_submit.json similarity index 74% rename from doc/expansion/vmray_submit.json rename to documentation/website/expansion/vmray_submit.json index ea6cf3f..2b38792 100644 --- a/doc/expansion/vmray_submit.json +++ b/documentation/website/expansion/vmray_submit.json @@ -1,9 +1,13 @@ { "description": "Module to submit a sample to VMRay.", - "logo": "logos/vmray.png", - "requirements": ["An access to the VMRay API (apikey & url)"], + "logo": "vmray.png", + "requirements": [ + "An access to the VMRay API (apikey & url)" + ], "input": "An attachment or malware-sample attribute.", "output": "MISP attributes mapped from the result of the query on VMRay API, included in the following list:\n- text\n- sha1\n- sha256\n- md5\n- link", - "references": ["https://www.vmray.com/"], + "references": [ + "https://www.vmray.com/" + ], "features": "This module takes an attachment or malware-sample attribute as input to query the VMRay API.\n\nThe sample contained within the attribute in then enriched with data from VMRay mapped into MISP attributes." -} +} \ No newline at end of file diff --git a/doc/expansion/vulndb.json b/documentation/website/expansion/vulndb.json similarity index 71% rename from doc/expansion/vulndb.json rename to documentation/website/expansion/vulndb.json index 330a3eb..e1dd869 100644 --- a/doc/expansion/vulndb.json +++ b/documentation/website/expansion/vulndb.json @@ -1,9 +1,13 @@ { "description": "Module to query VulnDB (RiskBasedSecurity.com).", - "logo": "logos/vulndb.png", - "requirements": ["An access to the VulnDB API (apikey, apisecret)"], + "logo": "vulndb.png", + "requirements": [ + "An access to the VulnDB API (apikey, apisecret)" + ], "input": "A vulnerability attribute.", "output": "Additional data enriching the CVE input, fetched from VulnDB.", - "references": ["https://vulndb.cyberriskanalytics.com/"], + "references": [ + "https://vulndb.cyberriskanalytics.com/" + ], "features": "This module takes a vulnerability attribute as input and queries VulnDB in order to get some additional data about it.\n\nThe API gives the result of the query which can be displayed in the screen, and/or mapped into MISP attributes to add in the event." -} +} \ No newline at end of file diff --git a/doc/expansion/vulners.json b/documentation/website/expansion/vulners.json similarity index 69% rename from doc/expansion/vulners.json rename to documentation/website/expansion/vulners.json index f3f3026..ab5a778 100644 --- a/doc/expansion/vulners.json +++ b/documentation/website/expansion/vulners.json @@ -1,9 +1,14 @@ { "description": "An expansion hover module to expand information about CVE id using Vulners API.", - "logo": "logos/vulners.png", - "requirements": ["Vulners python library", "An access to the Vulners API"], + "logo": "vulners.png", + "requirements": [ + "Vulners python library", + "An access to the Vulners API" + ], "input": "A vulnerability attribute.", "output": "Text giving additional information about the CVE in input.", - "references": ["https://vulners.com/"], + "references": [ + "https://vulners.com/" + ], "features": "This module takes a vulnerability attribute as input and queries the Vulners API in order to get some additional data about it.\n\nThe API then returns details about the vulnerability." -} +} \ No newline at end of file diff --git a/doc/expansion/whois.json b/documentation/website/expansion/whois.json similarity index 77% rename from doc/expansion/whois.json rename to documentation/website/expansion/whois.json index 938bad5..bba0828 100644 --- a/doc/expansion/whois.json +++ b/documentation/website/expansion/whois.json @@ -1,8 +1,12 @@ { "description": "Module to query a local instance of uwhois (https://github.com/rafiot/uwhoisd).", - "requirements": ["uwhois: A whois python library"], + "requirements": [ + "uwhois: A whois python library" + ], "input": "A domain or IP address attribute.", "output": "Text describing the result of a whois request for the input value.", - "references": ["https://github.com/rafiot/uwhoisd"], + "references": [ + "https://github.com/rafiot/uwhoisd" + ], "features": "This module takes a domain or IP address attribute as input and queries a 'Univseral Whois proxy server' to get the correct details of the Whois query on the input value (check the references for more details about this whois server)." -} +} \ No newline at end of file diff --git a/doc/expansion/wiki.json b/documentation/website/expansion/wiki.json similarity index 72% rename from doc/expansion/wiki.json rename to documentation/website/expansion/wiki.json index d6de62b..36bb009 100644 --- a/doc/expansion/wiki.json +++ b/documentation/website/expansion/wiki.json @@ -1,9 +1,13 @@ { "description": "An expansion hover module to extract information from Wikidata to have additional information about particular term for analysis.", - "logo": "logos/wikidata.png", - "requirements": ["SPARQLWrapper python library"], + "logo": "wikidata.png", + "requirements": [ + "SPARQLWrapper python library" + ], "input": "Text attribute.", "output": "Text attribute.", - "references": ["https://www.wikidata.org"], + "references": [ + "https://www.wikidata.org" + ], "features": "This module takes a text attribute as input and queries the Wikidata API. If the text attribute is clear enough to define a specific term, the API returns a wikidata link in response." -} +} \ No newline at end of file diff --git a/doc/expansion/xforceexchange.json b/documentation/website/expansion/xforceexchange.json similarity index 73% rename from doc/expansion/xforceexchange.json rename to documentation/website/expansion/xforceexchange.json index bbe3c86..fe6fcbb 100644 --- a/doc/expansion/xforceexchange.json +++ b/documentation/website/expansion/xforceexchange.json @@ -1,9 +1,13 @@ { "description": "An expansion module for IBM X-Force Exchange.", - "logo": "logos/xforce.png", - "requirements": ["An access to the X-Force API (apikey)"], + "logo": "xforce.png", + "requirements": [ + "An access to the X-Force API (apikey)" + ], "input": "A MISP attribute included in the following list:\n- ip-src\n- ip-dst\n- vulnerability\n- md5\n- sha1\n- sha256", "output": "MISP attributes mapped from the result of the query on X-Force Exchange.", - "references": ["https://exchange.xforce.ibmcloud.com/"], + "references": [ + "https://exchange.xforce.ibmcloud.com/" + ], "features": "This module takes a MISP attribute as input to query the X-Force API. The API returns then additional information known in their threats data, that is mapped into MISP attributes." -} +} \ No newline at end of file diff --git a/doc/expansion/xlsx-enrich.json b/documentation/website/expansion/xlsx_enrich.json similarity index 73% rename from doc/expansion/xlsx-enrich.json rename to documentation/website/expansion/xlsx_enrich.json index c41f17c..dff623d 100644 --- a/doc/expansion/xlsx-enrich.json +++ b/documentation/website/expansion/xlsx_enrich.json @@ -1,9 +1,11 @@ { "description": "Module to extract freetext from a .xlsx document.", - "logo": "logos/xlsx.png", - "requirements": ["pandas: Python library to perform data analysis, time series and statistics."], + "logo": "xlsx.png", + "requirements": [ + "pandas: Python library to perform data analysis, time series and statistics." + ], "input": "Attachment attribute containing a .xlsx document.", "output": "Text and freetext parsed from the document.", "references": [], "features": "The module reads the text contained in a .xlsx document. The result is passed to the freetext import parser so IoCs can be extracted out of it." -} +} \ No newline at end of file diff --git a/doc/expansion/yara_query.json b/documentation/website/expansion/yara_query.json similarity index 77% rename from doc/expansion/yara_query.json rename to documentation/website/expansion/yara_query.json index 408353d..453e599 100644 --- a/doc/expansion/yara_query.json +++ b/documentation/website/expansion/yara_query.json @@ -1,9 +1,14 @@ { "description": "An expansion & hover module to translate any hash attribute into a yara rule.", - "logo": "logos/yara.png", - "requirements": ["yara-python python library"], + "logo": "yara.png", + "requirements": [ + "yara-python python library" + ], "features": "The module takes a hash attribute (md5, sha1, sha256, imphash) as input, and is returning a YARA rule from it. This YARA rule is also validated using the same method as in 'yara_syntax_validator' module.\nBoth hover and expansion functionalities are supported with this module, where the hover part is displaying the resulting YARA rule and the expansion part allows you to add the rule as a new attribute, as usual with expansion modules.", "input": "MISP Hash attribute (md5, sha1, sha256, imphash, or any of the composite attribute with filename and one of the previous hash type).", "output": "YARA rule.", - "references": ["https://virustotal.github.io/yara/", "https://github.com/virustotal/yara-python"] -} + "references": [ + "https://virustotal.github.io/yara/", + "https://github.com/virustotal/yara-python" + ] +} \ No newline at end of file diff --git a/doc/expansion/yara_syntax_validator.json b/documentation/website/expansion/yara_syntax_validator.json similarity index 70% rename from doc/expansion/yara_syntax_validator.json rename to documentation/website/expansion/yara_syntax_validator.json index 93a96ee..72550b2 100644 --- a/doc/expansion/yara_syntax_validator.json +++ b/documentation/website/expansion/yara_syntax_validator.json @@ -1,9 +1,13 @@ { "description": "An expansion hover module to perform a syntax check on if yara rules are valid or not.", - "logo": "logos/yara.png", - "requirements": ["yara_python python library"], + "logo": "yara.png", + "requirements": [ + "yara_python python library" + ], "input": "YARA rule attribute.", "output": "Text to inform users if their rule is valid.", - "references": ["http://virustotal.github.io/yara/"], + "references": [ + "http://virustotal.github.io/yara/" + ], "features": "This modules simply takes a YARA rule as input, and checks its syntax. It returns then a confirmation if the syntax is valid, otherwise the syntax error is displayed." -} +} \ No newline at end of file diff --git a/doc/expansion/yeti.json b/documentation/website/expansion/yeti.json similarity index 100% rename from doc/expansion/yeti.json rename to documentation/website/expansion/yeti.json diff --git a/documentation/website/export_mod/cef_export.json b/documentation/website/export_mod/cef_export.json new file mode 100644 index 0000000..cd247a7 --- /dev/null +++ b/documentation/website/export_mod/cef_export.json @@ -0,0 +1,10 @@ +{ + "description": "Module to export a MISP event in CEF format.", + "requirements": [], + "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in Common Event Format.\nThus, there is no particular feature concerning MISP Events since any event can be exported. However, 4 configuration parameters recognized by CEF format are required and should be provided by users before exporting data: the device vendor, product and version, as well as the default severity of data.", + "references": [ + "https://community.softwaregrp.com/t5/ArcSight-Connectors/ArcSight-Common-Event-Format-CEF-Guide/ta-p/1589306?attachment-id=65537" + ], + "input": "MISP Event attributes", + "output": "Common Event Format file" +} \ No newline at end of file diff --git a/doc/export_mod/cisco_firesight_manager_ACL_rule_export.json b/documentation/website/export_mod/cisco_firesight_manager_ACL_rule_export.json similarity index 79% rename from doc/export_mod/cisco_firesight_manager_ACL_rule_export.json rename to documentation/website/export_mod/cisco_firesight_manager_ACL_rule_export.json index 6d1d0dd..b9c72f9 100644 --- a/doc/export_mod/cisco_firesight_manager_ACL_rule_export.json +++ b/documentation/website/export_mod/cisco_firesight_manager_ACL_rule_export.json @@ -1,9 +1,11 @@ { "description": "Module to export malicious network activity attributes to Cisco fireSIGHT manager block rules.", - "logo": "logos/cisco.png", - "requirements": ["Firesight manager console credentials"], + "logo": "cisco.png", + "requirements": [ + "Firesight manager console credentials" + ], "input": "Network activity attributes (IPs, URLs).", "output": "Cisco fireSIGHT manager block rules.", "references": [], "features": "The module goes through the attributes to find all the network activity ones in order to create block rules for the Cisco fireSIGHT manager." -} +} \ No newline at end of file diff --git a/documentation/website/export_mod/defender_endpoint_export.json b/documentation/website/export_mod/defender_endpoint_export.json new file mode 100644 index 0000000..ee45766 --- /dev/null +++ b/documentation/website/export_mod/defender_endpoint_export.json @@ -0,0 +1,11 @@ +{ + "description": "Defender for Endpoint KQL hunting query export module", + "requirements": [], + "features": "This module export an event as Defender for Endpoint KQL queries that can then be used in your own python3 or Powershell tool. If you are using Microsoft Sentinel, you can directly connect your MISP instance to Sentinel and then create queries using the `ThreatIntelligenceIndicator` table to match events against imported IOC.", + "references": [ + "https://docs.microsoft.com/en-us/windows/security/threat-protection/microsoft-defender-atp/advanced-hunting-schema-reference" + ], + "input": "MISP Event attributes", + "output": "Defender for Endpoint KQL queries", + "logo": "defender_endpoint.png" +} \ No newline at end of file diff --git a/documentation/website/export_mod/goamlexport.json b/documentation/website/export_mod/goamlexport.json new file mode 100644 index 0000000..aaab295 --- /dev/null +++ b/documentation/website/export_mod/goamlexport.json @@ -0,0 +1,14 @@ +{ + "description": "This module is used to export MISP events containing transaction objects into GoAML format.", + "logo": "goAML.jpg", + "requirements": [ + "PyMISP", + "MISP objects" + ], + "features": "The module works as long as there is at least one transaction object in the Event.\n\nThen in order to have a valid GoAML document, please follow these guidelines:\n- For each transaction object, use either a bank-account, person, or legal-entity object to describe the origin of the transaction, and again one of them to describe the target of the transaction.\n- Create an object reference for both origin and target objects of the transaction.\n- A bank-account object needs a signatory, which is a person object, put as object reference of the bank-account.\n- A person can have an address, which is a geolocation object, put as object reference of the person.\n\nSupported relation types for object references that are recommended for each object are the folowing:\n- transaction:\n\t- 'from', 'from_my_client': Origin of the transaction - at least one of them is required.\n\t- 'to', 'to_my_client': Target of the transaction - at least one of them is required.\n\t- 'address': Location of the transaction - optional.\n- bank-account:\n\t- 'signatory': Signatory of a bank-account - the reference from bank-account to a signatory is required, but the relation-type is optional at the moment since this reference will always describe a signatory.\n\t- 'entity': Entity owning the bank account - optional.\n- person:\n\t- 'address': Address of a person - optional.", + "references": [ + "http://goaml.unodc.org/" + ], + "input": "MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target.", + "output": "GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities)." +} \ No newline at end of file diff --git a/documentation/website/export_mod/liteexport.json b/documentation/website/export_mod/liteexport.json new file mode 100644 index 0000000..1f91039 --- /dev/null +++ b/documentation/website/export_mod/liteexport.json @@ -0,0 +1,8 @@ +{ + "description": "Lite export of a MISP event.", + "requirements": [], + "features": "This module is simply producing a json MISP event format file, but exporting only Attributes from the Event. Thus, MISP Events exported with this module should have attributes that are not internal references, otherwise the resulting event would be empty.", + "references": [], + "input": "MISP Event attributes", + "output": "Lite MISP Event" +} \ No newline at end of file diff --git a/doc/export_mod/mass_eql_export.json b/documentation/website/export_mod/mass_eql_export.json similarity index 74% rename from doc/export_mod/mass_eql_export.json rename to documentation/website/export_mod/mass_eql_export.json index 5eadd23..30b12a9 100644 --- a/doc/export_mod/mass_eql_export.json +++ b/documentation/website/export_mod/mass_eql_export.json @@ -1,9 +1,11 @@ { "description": "Mass EQL query export for a MISP event.", - "logo": "logos/eql.png", + "logo": "eql.png", "requirements": [], "features": "This module produces EQL queries for all relevant attributes in a MISP event.", - "references": ["https://eql.readthedocs.io/en/latest/"], + "references": [ + "https://eql.readthedocs.io/en/latest/" + ], "input": "MISP Event attributes", "output": "Text file containing one or more EQL queries" - } +} \ No newline at end of file diff --git a/documentation/website/export_mod/nexthinkexport.json b/documentation/website/export_mod/nexthinkexport.json new file mode 100644 index 0000000..0c06f9e --- /dev/null +++ b/documentation/website/export_mod/nexthinkexport.json @@ -0,0 +1,11 @@ +{ + "description": "Nexthink NXQL query export module", + "requirements": [], + "features": "This module export an event as Nexthink NXQL queries that can then be used in your own python3 tool or from wget/powershell", + "references": [ + "https://doc.nexthink.com/Documentation/Nexthink/latest/APIAndIntegrations/IntroducingtheWebAPIV2" + ], + "input": "MISP Event attributes", + "output": "Nexthink NXQL queries", + "logo": "nexthink.svg" +} \ No newline at end of file diff --git a/documentation/website/export_mod/osqueryexport.json b/documentation/website/export_mod/osqueryexport.json new file mode 100644 index 0000000..5b563c0 --- /dev/null +++ b/documentation/website/export_mod/osqueryexport.json @@ -0,0 +1,9 @@ +{ + "description": "OSQuery export of a MISP event.", + "requirements": [], + "features": "This module export an event as osquery queries that can be used in packs or in fleet management solution like Kolide.", + "references": [], + "input": "MISP Event attributes", + "output": "osquery SQL queries", + "logo": "osquery.png" +} \ No newline at end of file diff --git a/documentation/website/export_mod/pdfexport.json b/documentation/website/export_mod/pdfexport.json new file mode 100644 index 0000000..b23c681 --- /dev/null +++ b/documentation/website/export_mod/pdfexport.json @@ -0,0 +1,13 @@ +{ + "description": "Simple export of a MISP event to PDF.", + "requirements": [ + "PyMISP", + "reportlab" + ], + "features": "The module takes care of the PDF file building, and work with any MISP Event. Except the requirement of reportlab, used to create the file, there is no special feature concerning the Event. Some parameters can be given through the config dict. 'MISP_base_url_for_dynamic_link' is your MISP URL, to attach an hyperlink to your event on your MISP instance from the PDF. Keep it clear to avoid hyperlinks in the generated pdf.\n 'MISP_name_for_metadata' is your CERT or MISP instance name. Used as text in the PDF' metadata\n 'Activate_textual_description' is a boolean (True or void) to activate the textual description/header abstract of an event\n 'Activate_galaxy_description' is a boolean (True or void) to activate the description of event related galaxies.\n 'Activate_related_events' is a boolean (True or void) to activate the description of related event. Be aware this might leak information on confidential events linked to the current event !\n 'Activate_internationalization_fonts' is a boolean (True or void) to activate Noto fonts instead of default fonts (Helvetica). This allows the support of CJK alphabet. Be sure to have followed the procedure to download Noto fonts (~70Mo) in the right place (/tools/pdf_fonts/Noto_TTF), to allow PyMisp to find and use them during PDF generation.\n 'Custom_fonts_path' is a text (path or void) to the TTF file of your choice, to create the PDF with it. Be aware the PDF won't support bold/italic/special style anymore with this option ", + "references": [ + "https://acrobat.adobe.com/us/en/acrobat/about-adobe-pdf.html" + ], + "input": "MISP Event", + "output": "MISP Event in a PDF file." +} \ No newline at end of file diff --git a/doc/export_mod/testexport.json b/documentation/website/export_mod/testexport.json similarity index 95% rename from doc/export_mod/testexport.json rename to documentation/website/export_mod/testexport.json index 213ea92..884ccbe 100644 --- a/doc/export_mod/testexport.json +++ b/documentation/website/export_mod/testexport.json @@ -1,3 +1,3 @@ { "description": "Skeleton export module." -} +} \ No newline at end of file diff --git a/documentation/website/export_mod/threatStream_misp_export.json b/documentation/website/export_mod/threatStream_misp_export.json new file mode 100644 index 0000000..b096f41 --- /dev/null +++ b/documentation/website/export_mod/threatStream_misp_export.json @@ -0,0 +1,14 @@ +{ + "description": "Module to export a structured CSV file for uploading to threatStream.", + "logo": "threatstream.png", + "requirements": [ + "csv" + ], + "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatStream.", + "references": [ + "https://www.anomali.com/platform/threatstream", + "https://github.com/threatstream" + ], + "input": "MISP Event attributes", + "output": "ThreatStream CSV format file" +} \ No newline at end of file diff --git a/documentation/website/export_mod/threat_connect_export.json b/documentation/website/export_mod/threat_connect_export.json new file mode 100644 index 0000000..23708dd --- /dev/null +++ b/documentation/website/export_mod/threat_connect_export.json @@ -0,0 +1,13 @@ +{ + "description": "Module to export a structured CSV file for uploading to ThreatConnect.", + "logo": "threatconnect.png", + "requirements": [ + "csv" + ], + "features": "The module takes a MISP event in input, to look every attribute. Each attribute matching with some predefined types is then exported in a CSV format recognized by ThreatConnect.\nUsers should then provide, as module configuration, the source of data they export, because it is required by the output format.", + "references": [ + "https://www.threatconnect.com" + ], + "input": "MISP Event attributes", + "output": "ThreatConnect CSV format file" +} \ No newline at end of file diff --git a/doc/export_mod/vt_graph.json b/documentation/website/export_mod/vt_graph.json similarity index 66% rename from doc/export_mod/vt_graph.json rename to documentation/website/export_mod/vt_graph.json index e317730..993c791 100644 --- a/doc/export_mod/vt_graph.json +++ b/documentation/website/export_mod/vt_graph.json @@ -1,9 +1,13 @@ { "description": "This module is used to create a VirusTotal Graph from a MISP event.", - "logo": "logos/virustotal.png", - "requirements": ["vt_graph_api, the python library to query the VirusTotal graph API"], + "logo": "virustotal.png", + "requirements": [ + "vt_graph_api, the python library to query the VirusTotal graph API" + ], "features": "The module takes the MISP event as input and queries the VirusTotal Graph API to create a new graph out of the event.\n\nOnce the graph is ready, we get the url of it, which is returned so we can view it on VirusTotal.", - "references": ["https://www.virustotal.com/gui/graph-overview"], + "references": [ + "https://www.virustotal.com/gui/graph-overview" + ], "input": "A MISP event.", "output": "Link of the VirusTotal Graph created for the event." -} +} \ No newline at end of file diff --git a/documentation/website/import_mod/csvimport.json b/documentation/website/import_mod/csvimport.json new file mode 100644 index 0000000..61bc6cc --- /dev/null +++ b/documentation/website/import_mod/csvimport.json @@ -0,0 +1,13 @@ +{ + "description": "Module to import MISP attributes from a csv file.", + "requirements": [ + "PyMISP" + ], + "features": "In order to parse data from a csv file, a header is required to let the module know which column is matching with known attribute fields / MISP types.\n\nThis header either comes from the csv file itself or is part of the configuration of the module and should be filled out in MISP plugin settings, each field separated by COMMAS. Fields that do not match with any type known in MISP or are not MISP attribute fields should be ignored in import, using a space or simply nothing between two separators (example: 'ip-src, , comment, ').\n\nIf the csv file already contains a header that does not start by a '#', you should tick the checkbox 'has_header' to avoid importing it and have potential issues. You can also redefine the header even if it is already contained in the file, by following the rules for headers explained earlier. One reason why you would redefine a header is for instance when you want to skip some fields, or some fields are not valid types.", + "references": [ + "https://tools.ietf.org/html/rfc4180", + "https://tools.ietf.org/html/rfc7111" + ], + "input": "CSV format file.", + "output": "MISP Event attributes" +} \ No newline at end of file diff --git a/documentation/website/import_mod/cuckooimport.json b/documentation/website/import_mod/cuckooimport.json new file mode 100644 index 0000000..2e51ea8 --- /dev/null +++ b/documentation/website/import_mod/cuckooimport.json @@ -0,0 +1,12 @@ +{ + "description": "Module to import Cuckoo JSON.", + "logo": "cuckoo.png", + "requirements": [], + "features": "The module simply imports MISP Attributes from a Cuckoo JSON format file. There is thus no special feature to make it work.", + "references": [ + "https://cuckoosandbox.org/", + "https://github.com/cuckoosandbox/cuckoo" + ], + "input": "Cuckoo JSON file", + "output": "MISP Event attributes" +} \ No newline at end of file diff --git a/documentation/website/import_mod/email_import.json b/documentation/website/import_mod/email_import.json new file mode 100644 index 0000000..95ec3c7 --- /dev/null +++ b/documentation/website/import_mod/email_import.json @@ -0,0 +1,8 @@ +{ + "description": "Module to import emails in MISP.", + "requirements": [], + "features": "This module can be used to import e-mail text as well as attachments and urls.\n3 configuration parameters are then used to unzip attachments, guess zip attachment passwords, and extract urls: set each one of them to True or False to process or not the respective corresponding actions.", + "references": [], + "input": "E-mail file", + "output": "MISP Event attributes" +} \ No newline at end of file diff --git a/documentation/website/import_mod/goamlimport.json b/documentation/website/import_mod/goamlimport.json new file mode 100644 index 0000000..e8f12cf --- /dev/null +++ b/documentation/website/import_mod/goamlimport.json @@ -0,0 +1,11 @@ +{ + "description": "Module to import MISP objects about financial transactions from GoAML files.", + "logo": "goAML.jpg", + "requirements": [ + "PyMISP" + ], + "features": "Unlike the GoAML export module, there is here no special feature to import data from GoAML external files, since the module will import MISP Objects with their References on its own, as it is required for the export module to rebuild a valid GoAML document.", + "references": "http://goaml.unodc.org/", + "input": "GoAML format file, describing financial transactions, with their origin and target (bank accounts, persons or entities).", + "output": "MISP objects (transaction, bank-account, person, legal-entity, geolocation), with references, describing financial transactions and their origin and target." +} \ No newline at end of file diff --git a/doc/import_mod/joe_import.json b/documentation/website/import_mod/joe_import.json similarity index 78% rename from doc/import_mod/joe_import.json rename to documentation/website/import_mod/joe_import.json index ceba4ab..f60d1dd 100644 --- a/doc/import_mod/joe_import.json +++ b/documentation/website/import_mod/joe_import.json @@ -1,9 +1,12 @@ { "description": "A module to import data from a Joe Sandbox analysis json report.", - "logo": "logos/joesandbox.png", + "logo": "joesandbox.png", "requirements": [], "input": "Json report of a Joe Sandbox analysis.", "output": "MISP attributes & objects parsed from the analysis report.", - "references": ["https://www.joesecurity.org", "https://www.joesandbox.com/"], - "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report.\n\n" + "references": [ + "https://www.joesecurity.org", + "https://www.joesandbox.com/" + ], + "features": "Module using the new format of modules able to return attributes and objects.\n\nThe module returns the same results as the expansion module [joesandbox_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/joesandbox_query.py) using the submission link of the analysis to get the json report." } diff --git a/doc/import_mod/lastline_import.json b/documentation/website/import_mod/lastline_import.json similarity index 86% rename from doc/import_mod/lastline_import.json rename to documentation/website/import_mod/lastline_import.json index 99414e0..d89a433 100644 --- a/doc/import_mod/lastline_import.json +++ b/documentation/website/import_mod/lastline_import.json @@ -1,9 +1,11 @@ { "description": "Module to import and parse reports from Lastline analysis links.", - "logo": "logos/lastline.png", + "logo": "lastline.png", "requirements": [], "input": "Link to a Lastline analysis.", "output": "MISP attributes and objects parsed from the analysis report.", - "references": ["https://www.lastline.com"], + "references": [ + "https://www.lastline.com" + ], "features": "The module requires a Lastline Portal `username` and `password`.\nThe module uses the new format and it is able to return MISP attributes and objects.\nThe module returns the same results as the [lastline_query](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/lastline_query.py) expansion module." -} +} \ No newline at end of file diff --git a/documentation/website/import_mod/mispjson.json b/documentation/website/import_mod/mispjson.json new file mode 100644 index 0000000..7ba47bd --- /dev/null +++ b/documentation/website/import_mod/mispjson.json @@ -0,0 +1,8 @@ +{ + "description": "Module to import MISP JSON format for merging MISP events.", + "requirements": [], + "features": "The module simply imports MISP Attributes from an other MISP Event in order to merge events together. There is thus no special feature to make it work.", + "references": [], + "input": "MISP Event", + "output": "MISP Event attributes" +} \ No newline at end of file diff --git a/documentation/website/import_mod/ocr.json b/documentation/website/import_mod/ocr.json new file mode 100644 index 0000000..a33c7e2 --- /dev/null +++ b/documentation/website/import_mod/ocr.json @@ -0,0 +1,8 @@ +{ + "description": "Optical Character Recognition (OCR) module for MISP.", + "requirements": [], + "features": "The module tries to recognize some text from an image and import the result as a freetext attribute, there is then no special feature asked to users to make it work.", + "references": [], + "input": "Image", + "output": "freetext MISP attribute" +} \ No newline at end of file diff --git a/documentation/website/import_mod/openiocimport.json b/documentation/website/import_mod/openiocimport.json new file mode 100644 index 0000000..3e00baf --- /dev/null +++ b/documentation/website/import_mod/openiocimport.json @@ -0,0 +1,12 @@ +{ + "description": "Module to import OpenIOC packages.", + "requirements": [ + "PyMISP" + ], + "features": "The module imports MISP Attributes from OpenIOC packages, there is then no special feature for users to make it work.", + "references": [ + "https://www.fireeye.com/blog/threat-research/2013/10/openioc-basics.html" + ], + "input": "OpenIOC packages", + "output": "MISP Event attributes" +} \ No newline at end of file diff --git a/documentation/website/import_mod/threatanalyzer_import.json b/documentation/website/import_mod/threatanalyzer_import.json new file mode 100644 index 0000000..5866e09 --- /dev/null +++ b/documentation/website/import_mod/threatanalyzer_import.json @@ -0,0 +1,10 @@ +{ + "description": "Module to import ThreatAnalyzer archive.zip / analysis.json files.", + "requirements": [], + "features": "The module imports MISP Attributes from a ThreatAnalyzer format file. This file can be either ZIP, or JSON format.\nThere is by the way no special feature for users to make the module work.", + "references": [ + "https://www.threattrack.com/malware-analysis.aspx" + ], + "input": "ThreatAnalyzer format file", + "output": "MISP Event attributes" +} \ No newline at end of file diff --git a/documentation/website/import_mod/vmray_import.json b/documentation/website/import_mod/vmray_import.json new file mode 100644 index 0000000..c80b237 --- /dev/null +++ b/documentation/website/import_mod/vmray_import.json @@ -0,0 +1,13 @@ +{ + "description": "Module to import VMRay (VTI) results.", + "logo": "vmray.png", + "requirements": [ + "vmray_rest_api" + ], + "features": "The module imports MISP Attributes from VMRay format, using the VMRay api.\nUsers should then provide as the module configuration the API Key as well as the server url in order to fetch their data to import.", + "references": [ + "https://www.vmray.com/" + ], + "input": "VMRay format", + "output": "MISP Event attributes" +} \ No newline at end of file diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 440ad3f..21e3db3 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -183,10 +183,9 @@ class QueryModule(tornado.web.RequestHandler): executor = ThreadPoolExecutor(nb_threads) @run_on_executor - def run_request(self, jsonpayload): - x = json.loads(jsonpayload) + def run_request(self, module, jsonpayload): log.debug('MISP QueryModule request {0}'.format(jsonpayload)) - response = mhandlers[x['module']].handler(q=jsonpayload) + response = mhandlers[module].handler(q=jsonpayload) return json.dumps(response) @tornado.gen.coroutine @@ -198,7 +197,7 @@ class QueryModule(tornado.web.RequestHandler): timeout = datetime.timedelta(seconds=int(dict_payload.get('timeout'))) else: timeout = datetime.timedelta(seconds=300) - response = yield tornado.gen.with_timeout(timeout, self.run_request(jsonpayload)) + response = yield tornado.gen.with_timeout(timeout, self.run_request(dict_payload['module'], jsonpayload)) self.write(response) except tornado.gen.TimeoutError: log.warning('Timeout on {} '.format(dict_payload['module'])) diff --git a/misp_modules/modules/expansion/_vmray/__init__.py b/misp_modules/lib/_vmray/__init__.py similarity index 100% rename from misp_modules/modules/expansion/_vmray/__init__.py rename to misp_modules/lib/_vmray/__init__.py diff --git a/misp_modules/lib/_vmray/parser.py b/misp_modules/lib/_vmray/parser.py new file mode 100644 index 0000000..6e8d375 --- /dev/null +++ b/misp_modules/lib/_vmray/parser.py @@ -0,0 +1,1411 @@ +import base64 +import json +import re + +from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass, field +from enum import Enum +from pathlib import PureWindowsPath +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union + +from pymisp import MISPAttribute, MISPEvent, MISPObject + +from .rest_api import VMRayRESTAPI, VMRayRESTAPIError + + +USER_RE = re.compile(r".:.Users\\(.*?)\\", re.IGNORECASE) +DOC_RE = re.compile(r".:.DOCUME~1.\\(.*?)\\", re.IGNORECASE) +DOC_AND_SETTINGS_RE = re.compile(r".:.Documents and Settings\\(.*?)\\", re.IGNORECASE) +USERPROFILES = [USER_RE, DOC_RE, DOC_AND_SETTINGS_RE] + + +def classifications_to_str(classifications: List[str]) -> Optional[str]: + if classifications: + return "Classifications: " + ", ".join(classifications) + return None + + +def merge_lists(target: List[Any], source: List[Any]): + return list({*target, *source}) + + +@dataclass +class Attribute: + type: str + value: str + category: Optional[str] = None + comment: Optional[str] = None + to_ids: bool = False + + def __eq__(self, other: Dict[str, Any]) -> bool: + return asdict(self) == other + + +@dataclass +class Artifact: + is_ioc: bool + verdict: Optional[str] + + @abstractmethod + def to_attributes(self) -> Iterator[Attribute]: + raise NotImplementedError() + + @abstractmethod + def to_misp_object(self, tag: bool) -> MISPObject: + raise NotImplementedError() + + @abstractmethod + def merge(self, other: "Artifact") -> None: + raise NotImplementedError() + + @abstractmethod + def __eq__(self, other: "Artifact") -> bool: + raise NotImplementedError() + + def tag_artifact_attribute(self, attribute: MISPAttribute) -> None: + if self.is_ioc: + attribute.add_tag('vmray:artifact="IOC"') + + if self.verdict: + attribute.add_tag(f'vmray:verdict="{self.verdict}"') + + +@dataclass +class DomainArtifact(Artifact): + domain: str + sources: List[str] + ips: List[str] = field(default_factory=list) + classifications: List[str] = field(default_factory=list) + + def to_attributes(self) -> Iterator[Attribute]: + value = self.domain + comment = ", ".join(self.sources) if self.sources else None + + attr = Attribute(type="domain", value=value, comment=comment) + yield attr + + def to_misp_object(self, tag: bool) -> MISPObject: + obj = MISPObject(name="domain-ip") + + classifications = classifications_to_str(self.classifications) + attr = obj.add_attribute( + "domain", value=self.domain, to_ids=self.is_ioc, comment=classifications + ) + if tag: + self.tag_artifact_attribute(attr) + + for ip in self.ips: + obj.add_attribute("ip", value=ip, to_ids=self.is_ioc) + + return obj + + def merge(self, other: Artifact) -> None: + if not isinstance(other, DomainArtifact): + return + + self.ips = merge_lists(self.ips, other.ips) + self.classifications = merge_lists(self.classifications, other.classifications) + + def __eq__(self, other: Artifact) -> bool: + if not isinstance(other, DomainArtifact): + return NotImplemented + + return self.domain == other.domain + + +@dataclass +class EmailArtifact(Artifact): + sender: Optional[str] + subject: Optional[str] + recipients: List[str] = field(default_factory=list) + classifications: List[str] = field(default_factory=list) + + def to_attributes(self) -> Iterator[Attribute]: + if self.sender: + classifications = classifications_to_str(self.classifications) + yield Attribute( + type="email-src", value=self.sender, comment=classifications + ) + + if self.subject: + yield Attribute(type="email-subject", value=self.subject, to_ids=False) + + for recipient in self.recipients: + yield Attribute(type="email-dst", value=recipient, to_ids=False) + + def to_misp_object(self, tag: bool) -> MISPObject: + obj = MISPObject(name="email") + + if self.sender: + classifications = classifications_to_str(self.classifications) + attr = obj.add_attribute( + "from", value=self.sender, to_ids=self.is_ioc, comment=classifications + ) + if tag: + self.tag_artifact_attribute(attr) + + if self.subject: + obj.add_attribute("subject", value=self.subject, to_ids=False) + + for recipient in self.recipients: + obj.add_attribute("to", value=recipient, to_ids=False) + + return obj + + def merge(self, other: Artifact) -> None: + if not isinstance(other, EmailArtifact): + return + + self.recipients = merge_lists(self.recipients, other.recipients) + self.classifications = merge_lists(self.classifications, other.classifications) + + def __eq__(self, other: Artifact) -> bool: + if not isinstance(other, EmailArtifact): + return NotImplemented + + return self.sender == other.sender and self.subject == other.subject + + +@dataclass +class FileArtifact(Artifact): + filenames: List[str] + operations: List[str] + md5: str + sha1: str + sha256: str + ssdeep: str + imphash: Optional[str] + classifications: List[str] + size: Optional[int] + mimetype: Optional[str] = None + + def to_attributes(self) -> Iterator[Attribute]: + operations = ", ".join(self.operations) + comment = f"File operations: {operations}" + + for filename in self.filenames: + attr = Attribute(type="filename", value=filename, comment=comment) + yield attr + + for hash_type in ("md5", "sha1", "sha256", "ssdeep", "imphash"): + for filename in self.filenames: + value = getattr(self, hash_type) + if value is not None: + attr = Attribute( + type=f"filename|{hash_type}", + value=f"{filename}|{value}", + category="Payload delivery", + to_ids=True, + ) + yield attr + + def to_misp_object(self, tag: bool) -> MISPObject: + obj = MISPObject(name="file") + + if self.size: + obj.add_attribute("size-in-bytes", value=self.size) + + classifications = classifications_to_str(self.classifications) + hashes = [ + ("md5", self.md5), + ("sha1", self.sha1), + ("sha256", self.sha256), + ("ssdeep", self.ssdeep), + ] + for (key, value) in hashes: + if not value: + continue + + attr = obj.add_attribute( + key, value=value, to_ids=self.is_ioc, comment=classifications + ) + + if tag: + self.tag_artifact_attribute(attr) + + if self.mimetype: + obj.add_attribute("mimetype", value=self.mimetype, to_ids=False) + + operations = None + if self.operations: + operations = "Operations: " + ", ".join(self.operations) + + for filename in self.filenames: + filename = PureWindowsPath(filename) + obj.add_attribute("filename", value=filename.name, comment=operations) + + fullpath = str(filename) + for regex in USERPROFILES: + fullpath = regex.sub(r"%USERPROFILE%\\", fullpath) + + obj.add_attribute("fullpath", fullpath) + + return obj + + def merge(self, other: Artifact) -> None: + if not isinstance(other, FileArtifact): + return + + self.filenames = merge_lists(self.filenames, other.filenames) + self.operations = merge_lists(self.operations, other.operations) + self.classifications = merge_lists(self.classifications, other.classifications) + + def __eq__(self, other: Artifact) -> bool: + if not isinstance(other, FileArtifact): + return NotImplemented + + return self.sha256 == other.sha256 + + +@dataclass +class IpArtifact(Artifact): + ip: str + sources: List[str] + classifications: List[str] = field(default_factory=list) + + def to_attributes(self) -> Iterator[Attribute]: + sources = ", ".join(self.sources) + comment = f"Found in: {sources}" + + attr = Attribute(type="ip-dst", value=self.ip, comment=comment) + yield attr + + def to_misp_object(self, tag: bool) -> MISPObject: + obj = MISPObject(name="ip-port") + + classifications = classifications_to_str(self.classifications) + attr = obj.add_attribute( + "ip", value=self.ip, comment=classifications, to_ids=self.is_ioc + ) + if tag: + self.tag_artifact_attribute(attr) + + return obj + + def merge(self, other: Artifact) -> None: + if not isinstance(other, IpArtifact): + return + + self.sources = merge_lists(self.sources, other.sources) + self.classifications = merge_lists(self.classifications, other.classifications) + + def __eq__(self, other: Artifact) -> bool: + if not isinstance(other, IpArtifact): + return NotImplemented + + return self.ip == other.ip + + +@dataclass +class MutexArtifact(Artifact): + name: str + operations: List[str] + classifications: List[str] = field(default_factory=list) + + def to_attributes(self) -> Iterator[Attribute]: + operations = ", ".join(self.operations) + comment = f"Operations: {operations}" + + attr = Attribute(type="mutex", value=self.name, comment=comment) + yield attr + + def to_misp_object(self, tag: bool) -> MISPObject: + obj = MISPObject(name="mutex") + + classifications = classifications_to_str(self.classifications) + attr = obj.add_attribute( + "name", + value=self.name, + category="External analysis", + to_ids=False, + comment=classifications, + ) + if tag: + self.tag_artifact_attribute(attr) + + operations = None + if self.operations: + operations = "Operations: " + ", ".join(self.operations) + obj.add_attribute("description", value=operations, to_ids=False) + + return obj + + def merge(self, other: Artifact) -> None: + if not isinstance(other, MutexArtifact): + return + + self.operations = merge_lists(self.operations, other.operations) + self.classifications = merge_lists(self.classifications, other.classifications) + + def __eq__(self, other: Artifact) -> bool: + if not isinstance(other, MutexArtifact): + return NotImplemented + + return self.name == other.name + + +@dataclass +class ProcessArtifact(Artifact): + filename: str + pid: Optional[int] = None + parent_pid: Optional[int] = None + cmd_line: Optional[str] = None + operations: List[str] = field(default_factory=list) + classifications: List[str] = field(default_factory=list) + + def to_attributes(self) -> Iterator[Attribute]: + process_desc = f"Process created: {self.filename}\nPID: {self.pid}" + classifications = classifications_to_str(self.classifications) + yield Attribute(type="text", value=process_desc, comment=classifications) + + def to_misp_object(self, tag: bool) -> MISPObject: + obj = MISPObject(name="process") + + if self.pid: + obj.add_attribute("pid", value=self.pid, category="External analysis") + + if self.parent_pid: + obj.add_attribute( + "parent-pid", value=self.parent_pid, category="External analysis" + ) + + classifications = classifications_to_str(self.classifications) + name_attr = obj.add_attribute( + "name", self.filename, category="External analysis", comment=classifications + ) + + cmd_attr = obj.add_attribute("command-line", value=self.cmd_line) + + if tag: + self.tag_artifact_attribute(name_attr) + self.tag_artifact_attribute(cmd_attr) + + return obj + + def merge(self, other: Artifact) -> None: + if not isinstance(other, ProcessArtifact): + return + + self.operations = merge_lists(self.operations, other.operations) + self.classifications = merge_lists(self.classifications, other.classifications) + + def __eq__(self, other: Artifact) -> bool: + if not isinstance(other, ProcessArtifact): + return NotImplemented + + return self.filename == other.filename and self.cmd_line == other.cmd_line + + +@dataclass +class RegistryArtifact(Artifact): + key: str + operations: List[str] + + def to_attributes(self) -> Iterator[Attribute]: + operations = ", ".join(self.operations) + comment = f"Operations: {operations}" + + attr = Attribute(type="regkey", value=self.key, comment=comment) + yield attr + + def to_misp_object(self, tag: bool) -> MISPObject: + obj = MISPObject(name="registry-key") + + operations = None + if self.operations: + operations = "Operations: " + ", ".join(self.operations) + + attr = obj.add_attribute( + "key", value=self.key, to_ids=self.is_ioc, comment=operations + ) + if tag: + self.tag_artifact_attribute(attr) + + return obj + + def merge(self, other: Artifact) -> None: + if not isinstance(other, RegistryArtifact): + return + + self.operations = merge_lists(self.operations, other.operations) + + def __eq__(self, other: Artifact) -> bool: + if not isinstance(other, RegistryArtifact): + return NotImplemented + + return self.key == other.key + + +@dataclass +class UrlArtifact(Artifact): + url: str + operations: List[str] + domain: Optional[str] = None + ips: List[str] = field(default_factory=list) + + def to_attributes(self) -> Iterator[Attribute]: + operations = ", ".join(self.operations) + comment = f"Operations: {operations}" + + attr = Attribute(type="url", value=self.url, comment=comment) + yield attr + + def to_misp_object(self, tag: bool) -> MISPObject: + obj = MISPObject(name="url") + + operations = None + if self.operations: + operations = "Operations: " + ", ".join(self.operations) + + attr = obj.add_attribute( + "url", + value=self.url, + comment=operations, + category="External analysis", + to_ids=False, + ) + if tag: + self.tag_artifact_attribute(attr) + + if self.domain: + obj.add_attribute( + "domain", self.domain, category="External analysis", to_ids=False + ) + + for ip in self.ips: + obj.add_attribute("ip", ip, category="External analysis", to_ids=False) + + return obj + + def merge(self, other: Artifact) -> None: + if not isinstance(other, UrlArtifact): + return + + self.ips = merge_lists(self.ips, other.ips) + self.operations = merge_lists(self.operations, other.operations) + + def __eq__(self, other: Artifact) -> bool: + if not isinstance(other, UrlArtifact): + return NotImplemented + + return self.url == other.url and self.domain == other.domain + + +@dataclass +class MitreAttack: + description: str + id: str + + def to_misp_galaxy(self) -> str: + return f'misp-galaxy:mitre-attack-pattern="{self.description} - {self.id}"' + + +@dataclass +class VTI: + category: str + operation: str + technique: str + score: int + + +class ReportVersion(Enum): + v1 = "v1" + v2 = "v2" + + +class VMRayParseError(Exception): + pass + + +class ReportParser(ABC): + @abstractmethod + def __init__(self, api: VMRayRESTAPI, analysis_id: int): + raise NotImplementedError() + + @abstractmethod + def is_static_report(self) -> bool: + raise NotImplementedError() + + @abstractmethod + def artifacts(self) -> Iterator[Artifact]: + raise NotImplementedError() + + @abstractmethod + def classifications(self) -> Optional[str]: + raise NotImplementedError() + + @abstractmethod + def details(self) -> Iterator[str]: + raise NotImplementedError() + + @abstractmethod + def mitre_attacks(self) -> Iterator[MitreAttack]: + raise NotImplementedError() + + @abstractmethod + def sandbox_type(self) -> str: + raise NotImplementedError() + + @abstractmethod + def score(self) -> str: + raise NotImplementedError() + + @abstractmethod + def vtis(self) -> Iterator[VTI]: + raise NotImplementedError() + + +class Summary(ReportParser): + def __init__( + self, analysis_id: int, api: VMRayRESTAPI = None, report: Dict[str, Any] = None + ): + self.analysis_id = analysis_id + + if report: + self.report = report + else: + data = api.call( + "GET", + f"/rest/analysis/{analysis_id}/archive/logs/summary.json", + raw_data=True, + ) + self.report = json.load(data) + + @staticmethod + def to_verdict(score: Union[int, str]) -> Optional[str]: + if isinstance(score, int): + if 0 <= score <= 24: + return "clean" + if 25 <= score <= 74: + return "suspicious" + if 75 <= score <= 100: + return "malicious" + return "n/a" + if isinstance(score, str): + score = score.lower() + if score in ("not_suspicious", "whitelisted"): + return "clean" + if score == "blacklisted": + return "malicious" + if score in ("not_available", "unknown"): + return "n/a" + return score + return None + + def is_static_report(self) -> bool: + return self.report["vti"]["vti_rule_type"] == "Static" + + def artifacts(self) -> Iterator[Artifact]: + artifacts = self.report["artifacts"] + domains = artifacts.get("domains", []) + for domain in domains: + classifications = domain.get("classifications", []) + is_ioc = domain.get("ioc", False) + verdict = self.to_verdict(domain.get("severity")) + ips = domain.get("ip_addresses", []) + artifact = DomainArtifact( + domain=domain["domain"], + sources=domain["sources"], + ips=ips, + classifications=classifications, + is_ioc=is_ioc, + verdict=verdict, + ) + yield artifact + + emails = artifacts.get("emails", []) + for email in emails: + sender = email.get("sender") + subject = email.get("subject") + verdict = self.to_verdict(email.get("severity")) + recipients = email.get("recipients", []) + classifications = email.get("classifications", []) + is_ioc = email.get("ioc", False) + + artifact = EmailArtifact( + sender=sender, + subject=subject, + verdict=verdict, + recipients=recipients, + classifications=classifications, + is_ioc=is_ioc, + ) + yield artifact + + files = artifacts.get("files", []) + for file_ in files: + if file_["filename"] is None: + continue + + filenames = [file_["filename"]] + if "filenames" in file_: + filenames += file_["filenames"] + + hashes = file_["hashes"] + classifications = file_.get("classifications", []) + operations = file_.get("operations", []) + is_ioc = file_.get("ioc", False) + mimetype = file_.get("mime_type") + verdict = self.to_verdict(file_.get("severity")) + + for hash_dict in hashes: + imp = hash_dict.get("imp_hash") + + artifact = FileArtifact( + filenames=filenames, + imphash=imp, + md5=hash_dict["md5_hash"], + ssdeep=hash_dict["ssdeep_hash"], + sha256=hash_dict["sha256_hash"], + sha1=hash_dict["sha1_hash"], + operations=operations, + classifications=classifications, + size=file_.get("file_size"), + is_ioc=is_ioc, + mimetype=mimetype, + verdict=verdict, + ) + yield artifact + + ips = artifacts.get("ips", []) + for ip in ips: + is_ioc = ip.get("ioc", False) + verdict = self.to_verdict(ip.get("severity")) + classifications = ip.get("classifications", []) + artifact = IpArtifact( + ip=ip["ip_address"], + sources=ip["sources"], + classifications=classifications, + verdict=verdict, + is_ioc=is_ioc, + ) + yield artifact + + mutexes = artifacts.get("mutexes", []) + for mutex in mutexes: + verdict = self.to_verdict(mutex.get("severity")) + is_ioc = mutex.get("ioc", False) + artifact = MutexArtifact( + name=mutex["mutex_name"], + operations=mutex["operations"], + classifications=[], + verdict=verdict, + is_ioc=is_ioc, + ) + yield artifact + + processes = artifacts.get("processes", []) + for process in processes: + classifications = process.get("classifications", []) + cmd_line = process.get("cmd_line") + name = process["image_name"] + verdict = self.to_verdict(process.get("severity")) + is_ioc = process.get("ioc", False) + + artifact = ProcessArtifact( + filename=name, + classifications=classifications, + cmd_line=cmd_line, + verdict=verdict, + is_ioc=is_ioc, + ) + yield artifact + + registry = artifacts.get("registry", []) + for reg in registry: + is_ioc = reg.get("ioc", False) + verdict = self.to_verdict(reg.get("severity")) + artifact = RegistryArtifact( + key=reg["reg_key_name"], + operations=reg["operations"], + verdict=verdict, + is_ioc=is_ioc, + ) + yield artifact + + urls = artifacts.get("urls", []) + for url in urls: + ips = url.get("ip_addresses", []) + is_ioc = url.get("ioc", False) + verdict = self.to_verdict(url.get("severity")) + + artifact = UrlArtifact( + url=url["url"], + operations=url["operations"], + ips=ips, + is_ioc=is_ioc, + verdict=verdict, + ) + yield artifact + + def classifications(self) -> Optional[str]: + classifications = self.report["classifications"] + if classifications: + str_classifications = ", ".join(classifications) + return f"Classifications: {str_classifications}" + return None + + def details(self) -> Iterator[str]: + details = self.report["analysis_details"] + execution_successful = details["execution_successful"] + termination_reason = details["termination_reason"] + result = details["result_str"] + + if self.analysis_id == 0: + analysis = "" + else: + analysis = f" {self.analysis_id}" + + yield f"Analysis{analysis}: execution_successful: {execution_successful}" + yield f"Analysis{analysis}: termination_reason: {termination_reason}" + yield f"Analysis{analysis}: result: {result}" + + def mitre_attacks(self) -> Iterator[MitreAttack]: + mitre_attack = self.report["mitre_attack"] + techniques = mitre_attack.get("techniques", []) + + for technique in techniques: + mitre_attack = MitreAttack( + description=technique["description"], id=technique["id"] + ) + yield mitre_attack + + def sandbox_type(self) -> str: + vm_name = self.report["vm_and_analyzer_details"]["vm_name"] + sample_type = self.report["sample_details"]["sample_type"] + return f"{vm_name} | {sample_type}" + + def score(self) -> str: + vti_score = self.report["vti"]["vti_score"] + return self.to_verdict(vti_score) + + def vtis(self) -> Iterator[VTI]: + try: + vtis = self.report["vti"]["vti_rule_matches"] + except KeyError: + vtis = [] + + for vti in vtis: + new_vti = VTI( + category=vti["category_desc"], + operation=vti["operation_desc"], + technique=vti["technique_desc"], + score=vti["rule_score"], + ) + + yield new_vti + + +class SummaryV2(ReportParser): + def __init__( + self, analysis_id: int, api: VMRayRESTAPI = None, report: Dict[str, Any] = None + ): + self.analysis_id = analysis_id + + if report: + self.report = report + else: + self.api = api + data = api.call( + "GET", + f"/rest/analysis/{analysis_id}/archive/logs/summary_v2.json", + raw_data=True, + ) + self.report = json.load(data) + + def _resolve_refs( + self, data: Union[List[Dict[str, Any]], Dict[str, Any]] + ) -> Iterator[Dict[str, Any]]: + if not data: + return [] + + if isinstance(data, dict): + data = [data] + + for ref in data: + yield self._resolve_ref(ref) + + def _resolve_ref(self, data: Dict[str, Any]) -> Dict[str, Any]: + if data == {}: + return {} + + if data["_type"] != "reference" or data["source"] != "logs/summary_v2.json": + return {} + + resolved_ref = self.report + paths = data["path"] + for path_part in paths: + try: + resolved_ref = resolved_ref[path_part] + except KeyError: + return {} + + return resolved_ref + + @staticmethod + def convert_verdict(verdict: Optional[str]) -> str: + if verdict == "not_available" or not verdict: + return "n/a" + + return verdict + + def is_static_report(self) -> bool: + return self.report["vti"]["score_type"] == "static" + + def artifacts(self) -> Iterator[Artifact]: + artifacts = self.report["artifacts"] + + ref_domains = artifacts.get("ref_domains", []) + for domain in self._resolve_refs(ref_domains): + classifications = domain.get("classifications", []) + artifact = DomainArtifact( + domain=domain["domain"], + sources=domain["sources"], + classifications=classifications, + is_ioc=domain["is_ioc"], + verdict=domain["verdict"], + ) + + ref_ip_addresses = domain.get("ref_ip_addresses", []) + if not ref_ip_addresses: + continue + + for ip_address in self._resolve_refs(ref_ip_addresses): + artifact.ips.append(ip_address["ip_address"]) + + yield artifact + + ref_emails = artifacts.get("ref_emails", []) + for email in self._resolve_refs(ref_emails): + sender = email.get("sender") + subject = email.get("subject") + recipients = email.get("recipients", []) + verdict = email["verdict"] + is_ioc = email["is_ioc"] + classifications = email.get("classifications", []) + + artifact = EmailArtifact( + sender=sender, + subject=subject, + recipients=recipients, + classifications=classifications, + verdict=verdict, + is_ioc=is_ioc, + ) + + yield artifact + + ref_files = artifacts.get("ref_files", []) + for file_ in self._resolve_refs(ref_files): + filenames = [] + + if "ref_filenames" in file_: + for filename in self._resolve_refs(file_["ref_filenames"]): + if not filename: + continue + filenames.append(filename["filename"]) + + artifact = FileArtifact( + operations=file_.get("operations", []), + md5=file_["hash_values"]["md5"], + sha1=file_["hash_values"]["sha1"], + sha256=file_["hash_values"]["sha256"], + ssdeep=file_["hash_values"]["ssdeep"], + imphash=None, + mimetype=file_.get("mime_type"), + filenames=filenames, + is_ioc=file_["is_ioc"], + classifications=file_.get("classifications", []), + size=file_["size"], + verdict=file_["verdict"], + ) + yield artifact + + ref_ip_addresses = artifacts.get("ref_ip_addresses", []) + for ip in self._resolve_refs(ref_ip_addresses): + classifications = ip.get("classifications", []) + verdict = ip["verdict"] + is_ioc = ip["is_ioc"] + artifact = IpArtifact( + ip=ip["ip_address"], + sources=ip["sources"], + classifications=classifications, + verdict=verdict, + is_ioc=is_ioc, + ) + yield artifact + + ref_mutexes = artifacts.get("ref_mutexes", []) + for mutex in self._resolve_refs(ref_mutexes): + is_ioc = mutex["is_ioc"] + classifications = mutex.get("classifications", []) + artifact = MutexArtifact( + name=mutex["name"], + operations=mutex["operations"], + verdict=mutex["verdict"], + classifications=classifications, + is_ioc=is_ioc, + ) + yield artifact + + ref_processes = artifacts.get("ref_processes", []) + for process in self._resolve_refs(ref_processes): + cmd_line = process.get("cmd_line") + classifications = process.get("classifications", []) + verdict = process.get("verdict") + artifact = ProcessArtifact( + pid=process["os_pid"], + parent_pid=process["origin_monitor_id"], + filename=process["filename"], + is_ioc=process["is_ioc"], + cmd_line=cmd_line, + classifications=classifications, + verdict=verdict, + ) + yield artifact + + ref_registry_records = artifacts.get("ref_registry_records", []) + for reg in self._resolve_refs(ref_registry_records): + artifact = RegistryArtifact( + key=reg["reg_key_name"], + operations=reg["operations"], + is_ioc=reg["is_ioc"], + verdict=reg["verdict"], + ) + yield artifact + + url_refs = artifacts.get("ref_urls", []) + for url in self._resolve_refs(url_refs): + domain = None + ref_domain = url.get("ref_domain", {}) + if ref_domain: + domain = self._resolve_ref(ref_domain)["domain"] + + ips = [] + ref_ip_addresses = url.get("ref_ip_addresses", []) + for ip_address in self._resolve_refs(ref_ip_addresses): + ips.append(ip_address["ip_address"]) + + artifact = UrlArtifact( + url=url["url"], + operations=url["operations"], + is_ioc=url["is_ioc"], + domain=domain, + ips=ips, + verdict=url["verdict"], + ) + yield artifact + + def classifications(self) -> Optional[str]: + try: + classifications = ", ".join(self.report["classifications"]) + return f"Classifications: {classifications}" + except KeyError: + return None + + def details(self) -> Iterator[str]: + details = self.report["analysis_metadata"] + is_execution_successful = details["is_execution_successful"] + termination_reason = details["termination_reason"] + result = details["result_str"] + + yield f"Analysis {self.analysis_id}: execution_successful: {is_execution_successful}" + yield f"Analysis {self.analysis_id}: termination_reason: {termination_reason}" + yield f"Analysis {self.analysis_id}: result: {result}" + + def mitre_attacks(self) -> Iterator[MitreAttack]: + mitre_attack = self.report["mitre_attack"] + techniques = mitre_attack["v4"]["techniques"] + + for technique_id, technique in techniques.items(): + mitre_attack = MitreAttack( + description=technique["description"], + id=technique_id.replace("technique_", ""), + ) + yield mitre_attack + + def sandbox_type(self) -> str: + vm_information = self.report["virtual_machine"]["description"] + sample_type = self.report["analysis_metadata"]["sample_type"] + return f"{vm_information} | {sample_type}" + + def score(self) -> str: + verdict = self.report["analysis_metadata"]["verdict"] + return self.convert_verdict(verdict) + + def vtis(self) -> Iterator[VTI]: + if "matches" not in self.report["vti"]: + return + + vti_matches = self.report["vti"]["matches"] + for vti in vti_matches.values(): + new_vti = VTI( + category=vti["category_desc"], + operation=vti["operation_desc"], + technique=vti["technique_desc"], + score=vti["analysis_score"], + ) + + yield new_vti + + +class VMRayParser: + def __init__(self) -> None: + # required for api import + self.api: Optional[VMRayRESTAPI] = None + self.sample_id: Optional[int] = None + + # required for file import + self.report: Optional[Dict[str, Any]] = None + self.report_name: Optional[str] = None + self.include_report = False + + # required by API import and file import + self.report_version = ReportVersion.v2 + + self.use_misp_object = True + self.ignore_analysis_finished = False + self.tag_objects = True + + self.include_analysis_id = True + self.include_vti_details = True + self.include_iocs = True + self.include_all_artifacts = False + self.include_analysis_details = True + + # a new event if we use misp objects + self.event = MISPEvent() + + # new attributes if we don't use misp objects + self.attributes: List[Attribute] = [] + + def from_api(self, config: Dict[str, Any]) -> None: + url = self._read_config_key(config, "url") + api_key = self._read_config_key(config, "apikey") + + try: + self.sample_id = int(self._read_config_key(config, "Sample ID")) + except ValueError: + raise VMRayParseError("Could not convert sample id to integer.") + + self.api = VMRayRESTAPI(url, api_key, False) + + self.ignore_analysis_finished = self._config_from_string(config.get("ignore_analysis_finished")) + self._setup_optional_config(config) + self.report_version = self._get_report_version() + + def from_base64_string( + self, config: Dict[str, Any], data: str, filename: str + ) -> None: + """ read base64 encoded summary json """ + + buffer = base64.b64decode(data) + self.report = json.loads(buffer) + self.report_name = filename + + if "analysis_details" in self.report: + self.report_version = ReportVersion.v1 + elif "analysis_metadata" in self.report: + self.report_version = ReportVersion.v2 + else: + raise VMRayParseError("Uploaded file is not a summary.json") + + self._setup_optional_config(config) + self.include_report = bool(int(config.get("Attach Report", "0"))) + + def _setup_optional_config(self, config: Dict[str, Any]) -> None: + self.include_analysis_id = bool(int(config.get("Analysis ID", "1"))) + self.include_vti_details = bool(int(config.get("VTI", "1"))) + self.include_iocs = bool(int(config.get("IOCs", "1"))) + self.include_all_artifacts = bool(int(config.get("Artifacts", "0"))) + self.include_analysis_details = bool(int(config.get("Analysis Details", "1"))) + + self.use_misp_object = not self._config_from_string( + config.get("disable_misp_objects") + ) + self.tag_objects = not self._config_from_string(config.get("disable_tags")) + + @staticmethod + def _config_from_string(text: Optional[str]) -> bool: + if not text: + return False + + text = text.lower() + return text in ("yes", "true") + + @staticmethod + def _read_config_key(config: Dict[str, Any], key: str) -> str: + try: + value = config[key] + return value + except KeyError: + raise VMRayParseError(f"VMRay config is missing a value for `{key}`.") + + @staticmethod + def _analysis_score_to_taxonomies(analysis_score: int) -> Optional[str]: + mapping = { + -1: "-1", + 1: "1/5", + 2: "2/5", + 3: "3/5", + 4: "4/5", + 5: "5/5", + } + + try: + return mapping[analysis_score] + except KeyError: + return None + + def _get_report_version(self) -> ReportVersion: + info = self._vmary_api_call("/rest/system_info") + if info["version_major"] >= 4: + return ReportVersion.v2 + + # version 3.2 an less do not tag artifacts as ICOs + # so we extract all artifacts + if info["version_major"] == 3 and info["version_minor"] < 3: + self.include_all_artifacts = True + return ReportVersion.v1 + + def _vmary_api_call( + self, api_path: str, params: Dict[str, Any] = None, raw_data: bool = False + ) -> Union[Dict[str, Any], bytes]: + try: + return self.api.call("GET", api_path, params=params, raw_data=raw_data) + except (VMRayRESTAPIError, ValueError) as exc: + raise VMRayParseError(str(exc)) + + def _get_analysis(self) -> Dict[str, Any]: + return self._vmary_api_call(f"/rest/analysis/sample/{self.sample_id}") + + def _analysis_finished(self) -> bool: + result = self._vmary_api_call(f"/rest/submission/sample/{self.sample_id}") + + all_finished = [] + for submission in result: + finished = submission["submission_finished"] + all_finished.append(finished) + + return all(all_finished) + + def _online_reports(self) -> Iterator[Tuple[ReportParser, str]]: + # check if sample id exists + try: + self._vmary_api_call(f"/rest/sample/{self.sample_id}") + except VMRayRESTAPIError: + raise VMRayParseError( + f"Could not find sample id `{self.sample_id}` on server." + ) + + # check if all submission are finished + if not self.ignore_analysis_finished and not self._analysis_finished(): + raise VMRayParseError( + f"Not all analysis for `{self.sample_id}` are finished. " + "Try it again in a few minutes." + ) + + analysis_results = self._get_analysis() + for analysis in analysis_results: + analysis_id = analysis["analysis_id"] + permalink = analysis["analysis_webif_url"] + + # the summary json could not exist, due to a VM error + try: + if self.report_version == ReportVersion.v1: + report_parser = Summary(api=self.api, analysis_id=analysis_id) + else: + report_parser = SummaryV2(api=self.api, analysis_id=analysis_id) + except VMRayRESTAPIError: + continue + + yield report_parser, permalink + + def _offline_report(self) -> ReportParser: + if self.report_version == ReportVersion.v1: + analysis_id = 0 + return Summary(report=self.report, analysis_id=analysis_id) + else: + analysis_id = self.report["analysis_metadata"]["analysis_id"] + return SummaryV2(report=self.report, analysis_id=analysis_id) + + def _reports(self) -> Iterator[Tuple[ReportParser, Optional[str]]]: + if self.report: + yield self._offline_report(), None + else: + yield from self._online_reports() + + def _get_sample_verdict(self) -> Optional[str]: + if self.report: + if self.report_version == ReportVersion.v2: + verdict = SummaryV2.convert_verdict( + self.report["analysis_metadata"]["verdict"] + ) + return verdict + return None + + data = self._vmary_api_call(f"/rest/sample/{self.sample_id}") + if "sample_verdict" in data: + verdict = SummaryV2.convert_verdict(data["sample_verdict"]) + return verdict + + if "sample_severity" in data: + verdict = Summary.to_verdict(data["sample_severity"]) + return verdict + + return None + + def parse(self) -> None: + """ Convert analysis results to MISP Objects """ + + if self.use_misp_object: + self.parse_as_misp_object() + else: + self.parse_as_attributes() + + def parse_as_attributes(self) -> None: + """ + Parse report as attributes + This method is compatible with the implementation provided + by Koen Van Impe + """ + + for report, permalink in self._reports(): + if report.is_static_report(): + continue + + if self.include_analysis_details: + for detail in report.details(): + attr = Attribute(type="text", value=detail) + self.attributes.append(attr) + + classifications = report.classifications() + if classifications: + attr = Attribute(type="text", value=classifications) + self.attributes.append(attr) + + if self.include_vti_details: + for vti in report.vtis(): + attr = Attribute(type="text", value=vti.operation) + self.attributes.append(attr) + + for artifact in report.artifacts(): + if self.include_all_artifacts or ( + self.include_iocs and artifact.is_ioc + ): + for attr in artifact.to_attributes(): + self.attributes.append(attr) + + if self.include_analysis_id and permalink: + attr = Attribute(type="link", value=permalink) + self.attributes.append(attr) + + def parse_as_misp_object(self): + mitre_attacks = [] + vtis = [] + artifacts = [] + + # add sandbox signature + sb_sig = MISPObject(name="sb-signature") + sb_sig.add_attribute("software", "VMRay Platform") + + for report, permalink in self._reports(): + if report.is_static_report(): + continue + + # create sandbox object + obj = MISPObject(name="sandbox-report") + obj.add_attribute("on-premise-sandbox", "vmray") + + if permalink: + obj.add_attribute("permalink", permalink) + + if self.include_report and self.report: + report_data = base64.b64encode( + json.dumps(self.report, indent=2).encode("utf-8") + ).decode("utf-8") + obj.add_attribute( + "sandbox-file", value=self.report_name, data=report_data + ) + + score = report.score() + attr_score = obj.add_attribute("score", score) + + if self.tag_objects: + attr_score.add_tag(f'vmray:verdict="{score}"') + + sandbox_type = report.sandbox_type() + obj.add_attribute("sandbox-type", sandbox_type) + + classifications = report.classifications() + if classifications: + obj.add_attribute("results", classifications) + + self.event.add_object(obj) + + if self.include_vti_details: + for vti in report.vtis(): + if vti not in vtis: + vtis.append(vti) + + for artifact in report.artifacts(): + if self.include_all_artifacts or ( + self.include_iocs and artifact.is_ioc + ): + if artifact not in artifacts: + artifacts.append(artifact) + else: + idx = artifacts.index(artifact) + dup = artifacts[idx] + dup.merge(artifact) + + for mitre_attack in report.mitre_attacks(): + if mitre_attack not in mitre_attacks: + mitre_attacks.append(mitre_attack) + + # process VTI's + for vti in vtis: + vti_text = f"{vti.category}: {vti.operation}. {vti.technique}" + vti_attr = sb_sig.add_attribute("signature", value=vti_text) + + if self.tag_objects: + value = self._analysis_score_to_taxonomies(vti.score) + if value: + vti_attr.add_tag(f'vmray:vti_analysis_score="{value}"') + + self.event.add_object(sb_sig) + + # process artifacts + for artifact in artifacts: + artifact_obj = artifact.to_misp_object(self.tag_objects) + self.event.add_object(artifact_obj) + + # tag event with Mitre Att&ck + for mitre_attack in mitre_attacks: + self.event.add_tag(mitre_attack.to_misp_galaxy()) + + # tag event + if self.tag_objects: + verdict = self._get_sample_verdict() + if verdict: + self.event.add_tag(f'vmray:verdict="{verdict}"') + + def to_json(self) -> Dict[str, Any]: + """ Convert parsed results into JSON """ + + if not self.use_misp_object: + results = [] + + # remove duplicates + for attribute in self.attributes: + if attribute not in results: + results.append(asdict(attribute)) + + # add attributes to event + for attribute in results: + self.event.add_attribute(**attribute) + + self.event.run_expansions() + event = json.loads(self.event.to_json()) + + return {"results": event} diff --git a/misp_modules/modules/import_mod/_vmray/vmray_rest_api.py b/misp_modules/lib/_vmray/rest_api.py similarity index 100% rename from misp_modules/modules/import_mod/_vmray/vmray_rest_api.py rename to misp_modules/lib/_vmray/rest_api.py diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 14d5499..2723736 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,12 +1,11 @@ -from . import _vmray # noqa import os import sys sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', - 'countrycode', 'cve', 'cve_advanced', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'eql', - 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', + 'countrycode', 'cve', 'cve_advanced', 'cpe', 'dns', 'btc_steroids', 'domaintools', 'eupi', + 'eql', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_asn', 'geoip_city', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', @@ -18,4 +17,14 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid', 'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar', 'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich', - 'trustar_enrich', 'recordedfuture'] + 'trustar_enrich', 'recordedfuture', 'html_to_markdown', 'socialscan'] + + +minimum_required_fields = ('type', 'uuid', 'value') + +checking_error = 'containing at least a "type" field and a "value" field' +standard_error_message = 'This module requires an "attribute" field as input' + + +def check_input_attribute(attribute, requirements=minimum_required_fields): + return all(feature in attribute for feature in requirements) diff --git a/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py index af3f204..5df1207 100755 --- a/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py +++ b/misp_modules/modules/expansion/_dnsdb_query/dnsdb_query.py @@ -119,7 +119,10 @@ class DnsdbClient(object): break yield json.loads(line.decode('ascii')) except (HTTPError, URLError) as e: - raise QueryError(str(e), sys.exc_traceback) + try: + raise QueryError(str(e), sys.exc_traceback) + except AttributeError: + raise QueryError(str(e), sys.exc_info) def quote(path): diff --git a/misp_modules/modules/expansion/_vmray/vmray_rest_api.py b/misp_modules/modules/expansion/_vmray/vmray_rest_api.py deleted file mode 100644 index 4d5245b..0000000 --- a/misp_modules/modules/expansion/_vmray/vmray_rest_api.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python3 -"""Python client library for VMRay REST API""" - -import base64 -import datetime -import os.path -import requests -import urllib.parse - -# disable nasty certification warning -# pylint: disable=no-member -try: - requests.packages.urllib3.disable_warnings() -except AttributeError: - try: - import urllib3 - try: - urllib3.disable_warnings() - except AttributeError: - pass - except ImportError: - pass - -# pylint: disable= - - -class VMRayRESTAPIError(Exception): - """Exception class that is used when API returns an error""" - - def __init__(self, *args, **kwargs): - self.status_code = kwargs.pop("status_code", None) - Exception.__init__(self, *args, **kwargs) - - -def handle_rest_api_result(result): - """Handle result of API request (check for errors)""" - - if (result.status_code < 200) or (result.status_code > 299): - try: - json_result = result.json() - except ValueError: - raise VMRayRESTAPIError("API returned error %u: %s" % (result.status_code, result.text), status_code=result.status_code) - - raise VMRayRESTAPIError(json_result.get("error_msg", "Unknown error"), status_code=result.status_code) - - -class VMRayRESTAPI(object): - """VMRay REST API class""" - - def __init__(self, server, api_key, verify_cert=True): - # split server URL into components - url_desc = urllib.parse.urlsplit(server) - - # assume HTTPS if no scheme is specified - if url_desc.scheme == "": - server = "https://" + server - - # save variables - self.server = server - self.api_key = api_key - self.verify_cert = verify_cert - - def call(self, http_method, api_path, params=None, raw_data=False): - """Call VMRay REST API""" - - # get function of requests package - requests_func = getattr(requests, http_method.lower()) - - # parse parameters - req_params = {} - file_params = {} - - if params is not None: - for key, value in params.items(): - if isinstance(value, (datetime.date, - datetime.datetime, - float, - int)): - req_params[key] = str(value) - elif isinstance(value, str): - req_params[key] = str(value) - elif isinstance(value, dict): - filename = value["filename"] - sample = value["data"] - file_params[key] = (filename, sample, "application/octet-stream") - elif hasattr(value, "read"): - filename = os.path.split(value.name)[1] - # For the following block refer to DEV-1820 - try: - filename.decode("ASCII") - except (UnicodeDecodeError, UnicodeEncodeError): - b64_key = key + "name_b64enc" - byte_value = filename.encode("utf-8") - b64_value = base64.b64encode(byte_value) - - filename = "@param=%s" % b64_key - req_params[b64_key] = b64_value - file_params[key] = (filename, value, "application/octet-stream") - else: - raise VMRayRESTAPIError("Parameter \"%s\" has unknown type \"%s\"" % (key, type(value))) - - # construct request - if file_params: - files = file_params - else: - files = None - - # we need to adjust some stuff for POST requests - if http_method.lower() == "post": - req_data = req_params - req_params = None - else: - req_data = None - - # do request - result = requests_func(self.server + api_path, data=req_data, params=req_params, headers={"Authorization": "api_key " + self.api_key}, files=files, verify=self.verify_cert, stream=raw_data) - handle_rest_api_result(result) - - if raw_data: - return result.raw - - # parse result - try: - json_result = result.json() - except ValueError: - raise ValueError("API returned invalid JSON: %s" % (result.text)) - - # if there are no cached elements then return the data - if "continuation_id" not in json_result: - return json_result.get("data", None) - - data = json_result["data"] - - # get cached results - while "continuation_id" in json_result: - # send request to server - result = requests.get("%s/rest/continuation/%u" % (self.server, json_result["continuation_id"]), headers={"Authorization": "api_key " + self.api_key}, verify=self.verify_cert) - handle_rest_api_result(result) - - # parse result - try: - json_result = result.json() - except ValueError: - raise ValueError("API returned invalid JSON: %s" % (result.text)) - - data.extend(json_result["data"]) - - return data diff --git a/misp_modules/modules/expansion/apivoid.py b/misp_modules/modules/expansion/apivoid.py index 5d6395e..a71b5e6 100755 --- a/misp_modules/modules/expansion/apivoid.py +++ b/misp_modules/modules/expansion/apivoid.py @@ -1,5 +1,6 @@ import json import requests +from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} @@ -74,7 +75,11 @@ def handler(q=False): request = json.loads(q) if not request.get('config', {}).get('apikey'): return {'error': 'An API key for APIVoid is required.'} - attribute = request.get('attribute') + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + attribute = request['attribute'] + if attribute['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} apikey = request['config']['apikey'] apivoid_parser = APIVoidParser(attribute) apivoid_parser.parse_domain(apikey) diff --git a/misp_modules/modules/expansion/assemblyline_query.py b/misp_modules/modules/expansion/assemblyline_query.py index 226e4dd..90bdd3c 100644 --- a/misp_modules/modules/expansion/assemblyline_query.py +++ b/misp_modules/modules/expansion/assemblyline_query.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import json +from . import check_input_attribute, standard_error_message from assemblyline_client import Client, ClientError from collections import defaultdict from pymisp import MISPAttribute, MISPEvent, MISPObject @@ -10,7 +11,7 @@ mispattributes = {'input': ['link'], 'format': 'misp_standard'} moduleinfo = {'version': '1', 'author': 'Christian Studer', 'description': 'Query AssemblyLine with a report URL to get the parsed data.', 'module-type': ['expansion']} -moduleconfig = ["apiurl", "user_id", "apikey", "password"] +moduleconfig = ["apiurl", "user_id", "apikey", "password", "verifyssl"] class AssemblyLineParser(): @@ -124,7 +125,7 @@ def parse_config(apiurl, user_id, config): error = {"error": "Please provide your AssemblyLine API key or Password."} if config.get('apikey'): try: - return Client(apiurl, apikey=(user_id, config['apikey'])) + return Client(apiurl, apikey=(user_id, config['apikey']), verify=config['verifyssl']) except ClientError as e: error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' if config.get('password'): @@ -139,6 +140,10 @@ def handler(q=False): if q is False: return False request = json.loads(q) + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + if request['attribute']['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} if not request.get('config'): return {"error": "Missing configuration."} if not request['config'].get('apiurl'): diff --git a/misp_modules/modules/expansion/assemblyline_submit.py b/misp_modules/modules/expansion/assemblyline_submit.py index 206f5c0..9e019ff 100644 --- a/misp_modules/modules/expansion/assemblyline_submit.py +++ b/misp_modules/modules/expansion/assemblyline_submit.py @@ -7,7 +7,7 @@ from urllib.parse import urljoin moduleinfo = {"version": 1, "author": "Christian Studer", "module-type": ["expansion"], "description": "Submit files or URLs to AssemblyLine"} -moduleconfig = ["apiurl", "user_id", "apikey", "password"] +moduleconfig = ["apiurl", "user_id", "apikey", "password", "verifyssl"] mispattributes = {"input": ["attachment", "malware-sample", "url"], "output": ["link"]} @@ -16,12 +16,12 @@ def parse_config(apiurl, user_id, config): error = {"error": "Please provide your AssemblyLine API key or Password."} if config.get('apikey'): try: - return Client(apiurl, apikey=(user_id, config['apikey'])) + return Client(apiurl, apikey=(user_id, config['apikey']), verify=config['verifyssl']) except ClientError as e: error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' if config.get('password'): try: - return Client(apiurl, auth=(user_id, config['password'])) + return Client(apiurl, auth=(user_id, config['password']), verify=config['verifyssl']) except ClientError as e: error['error'] = f'Error while initiating a connection with AssemblyLine: {e.__str__()}' return error diff --git a/misp_modules/modules/expansion/bgpranking.py b/misp_modules/modules/expansion/bgpranking.py index b01088d..c021d62 100755 --- a/misp_modules/modules/expansion/bgpranking.py +++ b/misp_modules/modules/expansion/bgpranking.py @@ -1,13 +1,15 @@ # -*- coding: utf-8 -*- import json -from datetime import date, timedelta +from . import check_input_attribute, standard_error_message +from datetime import date, datetime, timedelta from pybgpranking import BGPRanking +from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} -mispattributes = {'input': ['AS'], 'output': ['freetext']} +mispattributes = {'input': ['AS'], 'format': 'misp_standard'} moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', - 'description': 'Query an ASN Description history service (https://github.com/CIRCL/ASN-Description-History.git)', + 'description': 'Query BGP Ranking to get the ranking of an Autonomous System number.', 'module-type': ['expansion', 'hover']} @@ -15,19 +17,65 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('AS'): - toquery = request['AS'] - else: - misperrors['error'] = "Unsupported attributes type" - return misperrors + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + toquery = request['attribute'] + if toquery['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} bgpranking = BGPRanking() - values = bgpranking.query(toquery, date=(date.today() - timedelta(1)).isoformat()) + value_toquery = int(toquery['value'][2:]) if toquery['value'].startswith('AS') else int(toquery['value']) + values = bgpranking.query(value_toquery, date=(date.today() - timedelta(1)).isoformat()) - if not values: - misperrors['error'] = 'Unable to find the ASN in BGP Ranking' + if not values['response'] or not values['response']['asn_description']: + misperrors['error'] = 'There is no result about this ASN in BGP Ranking' return misperrors - return {'results': [{'types': mispattributes['output'], 'values': values}]} + + event = MISPEvent() + attribute = MISPAttribute() + attribute.from_dict(**toquery) + event.add_attribute(**attribute) + + asn_object = MISPObject('asn') + asn_object.add_attribute(**{ + 'type': 'AS', + 'object_relation': 'asn', + 'value': values['meta']['asn'] + }) + description, country = values['response']['asn_description'].split(', ') + for relation, value in zip(('description', 'country'), (description, country)): + asn_object.add_attribute(**{ + 'type': 'text', + 'object_relation': relation, + 'value': value + }) + + mapping = { + 'address_family': {'type': 'text', 'object_relation': 'address-family'}, + 'date': {'type': 'datetime', 'object_relation': 'date'}, + 'position': {'type': 'float', 'object_relation': 'position'}, + 'rank': {'type': 'float', 'object_relation': 'ranking'} + } + bgp_object = MISPObject('bgp-ranking') + for feature in ('rank', 'position'): + bgp_attribute = {'value': values['response']['ranking'][feature]} + bgp_attribute.update(mapping[feature]) + bgp_object.add_attribute(**bgp_attribute) + date_attribute = {'value': datetime.strptime(values['meta']['date'], '%Y-%m-%d')} + date_attribute.update(mapping['date']) + bgp_object.add_attribute(**date_attribute) + address_attribute = {'value': values['meta']['address_family']} + address_attribute.update(mapping['address_family']) + bgp_object.add_attribute(**address_attribute) + + asn_object.add_reference(attribute.uuid, 'describes') + asn_object.add_reference(bgp_object.uuid, 'ranked-with') + event.add_object(asn_object) + event.add_object(bgp_object) + + event = json.loads(event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} def introspection(): diff --git a/misp_modules/modules/expansion/censys_enrich.py b/misp_modules/modules/expansion/censys_enrich.py index 0fc61ae..d5823ff 100644 --- a/misp_modules/modules/expansion/censys_enrich.py +++ b/misp_modules/modules/expansion/censys_enrich.py @@ -3,6 +3,7 @@ import json import base64 import codecs from dateutil.parser import isoparse +from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject try: import censys.base @@ -36,11 +37,11 @@ def handler(q=False): api_id = request['config']['api_id'] api_secret = request['config']['api_secret'] - if not request.get('attribute'): - return {'error': 'Unsupported input.'} + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} attribute = request['attribute'] if not any(input_type == attribute['type'] for input_type in mispattributes['input']): - return {'error': 'Unsupported attributes type'} + return {'error': 'Unsupported attribute type.'} attribute = MISPAttribute() attribute.from_dict(**request['attribute']) diff --git a/misp_modules/modules/expansion/circl_passivedns.py b/misp_modules/modules/expansion/circl_passivedns.py index d278a85..5f98314 100755 --- a/misp_modules/modules/expansion/circl_passivedns.py +++ b/misp_modules/modules/expansion/circl_passivedns.py @@ -1,5 +1,6 @@ import json import pypdns +from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'} @@ -58,11 +59,11 @@ def handler(q=False): if not request['config'].get('username') or not request['config'].get('password'): return {'error': 'CIRCL Passive DNS authentication is incomplete, please provide your username and password.'} authentication = (request['config']['username'], request['config']['password']) - if not request.get('attribute'): - return {'error': 'Unsupported input.'} + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} attribute = request['attribute'] if not any(input_type == attribute['type'] for input_type in mispattributes['input']): - return {'error': 'Unsupported attributes type'} + return {'error': 'Unsupported attribute type.'} pdns_parser = PassiveDNSParser(attribute, authentication) pdns_parser.parse() return pdns_parser.get_results() diff --git a/misp_modules/modules/expansion/circl_passivessl.py b/misp_modules/modules/expansion/circl_passivessl.py index 102bed8..65783d7 100755 --- a/misp_modules/modules/expansion/circl_passivessl.py +++ b/misp_modules/modules/expansion/circl_passivessl.py @@ -1,5 +1,6 @@ import json import pypssl +from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject mispattributes = {'input': ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port'], 'format': 'misp_standard'} @@ -83,11 +84,11 @@ def handler(q=False): if not request['config'].get('username') or not request['config'].get('password'): return {'error': 'CIRCL Passive SSL authentication is incomplete, please provide your username and password.'} authentication = (request['config']['username'], request['config']['password']) - if not request.get('attribute'): - return {'error': 'Unsupported input.'} + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} attribute = request['attribute'] if not any(input_type == attribute['type'] for input_type in mispattributes['input']): - return {'error': 'Unsupported attributes type'} + return {'error': 'Unsupported attribute type.'} pssl_parser = PassiveSSLParser(attribute, authentication) pssl_parser.parse() return pssl_parser.get_results() diff --git a/misp_modules/modules/expansion/clamav.py b/misp_modules/modules/expansion/clamav.py new file mode 100644 index 0000000..1582409 --- /dev/null +++ b/misp_modules/modules/expansion/clamav.py @@ -0,0 +1,128 @@ +import base64 +import io +import json +import logging +import sys +import zipfile +import clamd +from . import check_input_attribute, standard_error_message +from typing import Optional +from pymisp import MISPEvent, MISPObject + +log = logging.getLogger("clamav") +log.setLevel(logging.DEBUG) +sh = logging.StreamHandler(sys.stdout) +sh.setLevel(logging.DEBUG) +fmt = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +sh.setFormatter(fmt) +log.addHandler(sh) + +moduleinfo = { + "version": "0.1", + "author": "Jakub Onderka", + "description": "Submit file to ClamAV", + "module-type": ["expansion"] +} +moduleconfig = ["connection"] +mispattributes = { + "input": ["attachment", "malware-sample"], + "format": "misp_standard" +} + + +def create_response(original_attribute: dict, software: str, signature: Optional[str] = None) -> dict: + misp_event = MISPEvent() + if signature: + misp_event.add_attribute(**original_attribute) + + av_signature_object = MISPObject("av-signature") + av_signature_object.add_attribute("signature", signature) + av_signature_object.add_attribute("software", software) + av_signature_object.add_reference(original_attribute["uuid"], "belongs-to") + misp_event.add_object(av_signature_object) + + event = json.loads(misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {"results": results} + + +def connect_to_clamav(connection_string: str) -> clamd.ClamdNetworkSocket: + if connection_string.startswith("unix://"): + return clamd.ClamdUnixSocket(connection_string.replace("unix://", "")) + elif ":" in connection_string: + host, port = connection_string.split(":") + return clamd.ClamdNetworkSocket(host, int(port)) + else: + raise Exception("ClamAV connection string is invalid. It must be unix socket path with 'unix://' prefix or IP:PORT.") + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + + connection_string: str = request["config"].get("connection") + if not connection_string: + return {"error": "No ClamAV connection string provided"} + + attribute = request.get("attribute") + if not attribute: + return {"error": "No attribute provided"} + + if not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + + if attribute["type"] not in mispattributes["input"]: + return {"error": "Invalid attribute type provided, expected 'malware-sample' or 'attachment'"} + + attribute_data = attribute.get("data") + if not attribute_data: + return {"error": "No attribute data provided"} + + try: + clamav = connect_to_clamav(connection_string) + software_version = clamav.version() + except Exception: + logging.exception("Could not connect to ClamAV") + return {"error": "Could not connect to ClamAV"} + + try: + data = base64.b64decode(attribute_data, validate=True) + except Exception: + logging.exception("Provided data is not valid base64 encoded string") + return {"error": "Provided data is not valid base64 encoded string"} + + if attribute["type"] == "malware-sample": + try: + with zipfile.ZipFile(io.BytesIO(data)) as zipf: + data = zipf.read(zipf.namelist()[0], pwd=b"infected") + except Exception: + logging.exception("Could not extract malware sample from ZIP file") + return {"error": "Could not extract malware sample from ZIP file"} + + try: + status, reason = clamav.instream(io.BytesIO(data))["stream"] + except Exception: + logging.exception("Could not send attribute data to ClamAV. Maybe file is too big?") + return {"error": "Could not send attribute data to ClamAV. Maybe file is too big?"} + + if status == "ERROR": + return {"error": "ClamAV returned error message: {}".format(reason)} + elif status == "OK": + return {"results": {}} + elif status == "FOUND": + return create_response(attribute, software_version, reason) + else: + return {"error": "ClamAV returned invalid status {}: {}".format(status, reason)} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo["config"] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/cpe.py b/misp_modules/modules/expansion/cpe.py new file mode 100644 index 0000000..600ff37 --- /dev/null +++ b/misp_modules/modules/expansion/cpe.py @@ -0,0 +1,133 @@ +import json +import requests +from . import check_input_attribute, standard_error_message +from pymisp import MISPEvent, MISPObject + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['cpe'], 'format': 'misp_standard'} +moduleinfo = { + 'version': '2', + 'author': 'Christian Studer', + 'description': 'An expansion module to enrich a CPE attribute with its related vulnerabilities.', + 'module-type': ['expansion', 'hover'] +} +moduleconfig = ["custom_API_URL", "limit"] +cveapi_url = 'https://cvepremium.circl.lu/api/query' +DEFAULT_LIMIT = 10 + + +class VulnerabilitiesParser(): + def __init__(self, attribute): + self.attribute = attribute + self.misp_event = MISPEvent() + self.misp_event.add_attribute(**attribute) + self.vulnerability_mapping = { + 'id': { + 'type': 'vulnerability', + 'object_relation': 'id' + }, + 'summary': { + 'type': 'text', + 'object_relation': 'summary' + }, + 'vulnerable_configuration': { + 'type': 'cpe', + 'object_relation': 'vulnerable-configuration' + }, + 'vulnerable_configuration_cpe_2_2': { + 'type': 'cpe', + 'object_relation': 'vulnerable-configuration' + }, + 'Modified': { + 'type': 'datetime', + 'object_relation': 'modified' + }, + 'Published': { + 'type': 'datetime', + 'object_relation': 'published' + }, + 'references': { + 'type': 'link', + 'object_relation': 'references' + }, + 'cvss': { + 'type': 'float', + 'object_relation': 'cvss-score' + } + } + + def parse_vulnerabilities(self, vulnerabilities): + for vulnerability in vulnerabilities: + vulnerability_object = MISPObject('vulnerability') + for feature in ('id', 'summary', 'Modified', 'Published', 'cvss'): + if vulnerability.get(feature): + attribute = {'value': vulnerability[feature]} + attribute.update(self.vulnerability_mapping[feature]) + vulnerability_object.add_attribute(**attribute) + if vulnerability.get('Published'): + vulnerability_object.add_attribute(**{ + 'type': 'text', + 'object_relation': 'state', + 'value': 'Published' + }) + for feature in ('references', 'vulnerable_configuration', 'vulnerable_configuration_cpe_2_2'): + if vulnerability.get(feature): + for value in vulnerability[feature]: + if isinstance(value, dict): + value = value['title'] + attribute = {'value': value} + attribute.update(self.vulnerability_mapping[feature]) + vulnerability_object.add_attribute(**attribute) + vulnerability_object.add_reference(self.attribute['uuid'], 'related-to') + self.misp_event.add_object(vulnerability_object) + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + +def check_url(url): + return url if url.endswith('/') else f"{url}/" + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + attribute = request['attribute'] + if attribute.get('type') != 'cpe': + return {'error': 'Wrong input attribute type.'} + config = request['config'] + url = check_url(config['custom_API_URL']) if config.get('custom_API_URL') else cveapi_url + limit = int(config['limit']) if config.get('limit') else DEFAULT_LIMIT + params = { + "retrieve": "cves", + "dict_filter": { + "vulnerable_configuration": attribute['value'] + }, + "limit": limit, + "sort": "cvss", + "sort_dir": "DESC" + } + response = requests.post(url, json=params) + if response.status_code == 200: + vulnerabilities = response.json()['data'] + if not vulnerabilities: + return {'error': 'No related vulnerability for this CPE.'} + else: + return {'error': 'API not accessible.'} + parser = VulnerabilitiesParser(attribute) + parser.parse_vulnerabilities(vulnerabilities) + return parser.get_result() + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/cve_advanced.py b/misp_modules/modules/expansion/cve_advanced.py index 86cba8c..a5e807e 100644 --- a/misp_modules/modules/expansion/cve_advanced.py +++ b/misp_modules/modules/expansion/cve_advanced.py @@ -1,7 +1,8 @@ -from collections import defaultdict -from pymisp import MISPEvent, MISPObject import json import requests +from . import check_input_attribute, standard_error_message +from collections import defaultdict +from pymisp import MISPEvent, MISPObject misperrors = {'error': 'Error'} mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'} @@ -22,9 +23,9 @@ class VulnerabilityParser(): self.references = defaultdict(list) self.capec_features = ('id', 'name', 'summary', 'prerequisites', 'solutions') self.vulnerability_mapping = { - 'id': ('text', 'id'), 'summary': ('text', 'summary'), - 'vulnerable_configuration': ('text', 'vulnerable_configuration'), - 'vulnerable_configuration_cpe_2_2': ('text', 'vulnerable_configuration'), + 'id': ('vulnerability', 'id'), 'summary': ('text', 'summary'), + 'vulnerable_configuration': ('cpe', 'vulnerable_configuration'), + 'vulnerable_configuration_cpe_2_2': ('cpe', 'vulnerable_configuration'), 'Modified': ('datetime', 'modified'), 'Published': ('datetime', 'published'), 'references': ('link', 'references'), 'cvss': ('float', 'cvss-score')} self.weakness_mapping = {'name': 'name', 'description_summary': 'description', @@ -55,7 +56,7 @@ class VulnerabilityParser(): value = value['title'] vulnerability_object.add_attribute(relation, **{'type': attribute_type, 'value': value}) vulnerability_object.add_reference(self.attribute['uuid'], 'related-to') - self.misp_event.add_object(**vulnerability_object) + self.misp_event.add_object(vulnerability_object) if 'cwe' in self.vulnerability and self.vulnerability['cwe'] not in ('Unknown', 'NVD-CWE-noinfo'): self.__parse_weakness(vulnerability_object.uuid) if 'capec' in self.vulnerability: @@ -70,33 +71,39 @@ class VulnerabilityParser(): break def __parse_capec(self, vulnerability_uuid): - attribute_type = 'text' for capec in self.vulnerability['capec']: capec_object = MISPObject('attack-pattern') for feature in self.capec_features: - capec_object.add_attribute(feature, **dict(type=attribute_type, value=capec[feature])) + capec_object.add_attribute(feature, **{'type': 'text', 'value': capec[feature]}) for related_weakness in capec['related_weakness']: - attribute = dict(type='weakness', value="CWE-{}".format(related_weakness)) + attribute = {'type': 'weakness', 'value': f"CWE-{related_weakness}"} capec_object.add_attribute('related-weakness', **attribute) - self.misp_event.add_object(**capec_object) - self.references[vulnerability_uuid].append(dict(referenced_uuid=capec_object.uuid, - relationship_type='targeted-by')) + self.misp_event.add_object(capec_object) + self.references[vulnerability_uuid].append( + { + 'referenced_uuid': capec_object.uuid, + 'relationship_type': 'targeted-by' + } + ) def __parse_weakness(self, vulnerability_uuid): - attribute_type = 'text' - cwe_string, cwe_id = self.vulnerability['cwe'].split('-') + cwe_string, cwe_id = self.vulnerability['cwe'].split('-')[:2] cwes = requests.get(self.api_url.replace('/cve/', '/cwe')) if cwes.status_code == 200: for cwe in cwes.json(): if cwe['id'] == cwe_id: weakness_object = MISPObject('weakness') - weakness_object.add_attribute('id', **dict(type=attribute_type, value='-'.join([cwe_string, cwe_id]))) + weakness_object.add_attribute('id', {'type': 'weakness', 'value': f'{cwe_string}-{cwe_id}'}) for feature, relation in self.weakness_mapping.items(): if cwe.get(feature): - weakness_object.add_attribute(relation, **dict(type=attribute_type, value=cwe[feature])) - self.misp_event.add_object(**weakness_object) - self.references[vulnerability_uuid].append(dict(referenced_uuid=weakness_object.uuid, - relationship_type='weakened-by')) + weakness_object.add_attribute(relation, **{'type': 'text', 'value': cwe[feature]}) + self.misp_event.add_object(weakness_object) + self.references[vulnerability_uuid].append( + { + 'referenced_uuid': weakness_object.uuid, + 'relationship_type': 'weakened-by' + } + ) break @@ -108,7 +115,9 @@ def handler(q=False): if q is False: return False request = json.loads(q) - attribute = request.get('attribute') + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + attribute = request['attribute'] if attribute.get('type') != 'vulnerability': misperrors['error'] = 'Vulnerability id missing.' return misperrors diff --git a/misp_modules/modules/expansion/cytomic_orion.py b/misp_modules/modules/expansion/cytomic_orion.py index 9723ed6..c13b254 100755 --- a/misp_modules/modules/expansion/cytomic_orion.py +++ b/misp_modules/modules/expansion/cytomic_orion.py @@ -7,6 +7,7 @@ An expansion module to enrich attributes in MISP and share indicators of comprom ''' +from . import check_input_attribute, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject import json import requests @@ -146,9 +147,11 @@ def handler(q=False): if not request.get('attribute'): return {'error': 'Unsupported input.'} + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} attribute = request['attribute'] if not any(input_type == attribute['type'] for input_type in mispattributes['input']): - return {'error': 'Unsupported attributes type'} + return {'error': 'Unsupported attribute type.'} if not request.get('config'): return {'error': 'Missing configuration'} diff --git a/misp_modules/modules/expansion/farsight_passivedns.py b/misp_modules/modules/expansion/farsight_passivedns.py index 5d32ea8..de18735 100755 --- a/misp_modules/modules/expansion/farsight_passivedns.py +++ b/misp_modules/modules/expansion/farsight_passivedns.py @@ -1,15 +1,129 @@ +import dnsdb2 import json -from ._dnsdb_query.dnsdb_query import DnsdbClient, QueryError - +from . import check_input_attribute, standard_error_message +from datetime import datetime +from pymisp import MISPEvent, MISPObject misperrors = {'error': 'Error'} -mispattributes = {'input': ['hostname', 'domain', 'ip-src', 'ip-dst'], 'output': ['freetext']} -moduleinfo = {'version': '0.1', 'author': 'Christophe Vandeplas', 'description': 'Module to access Farsight DNSDB Passive DNS', 'module-type': ['expansion', 'hover']} -moduleconfig = ['apikey'] +standard_query_input = [ + 'hostname', + 'domain', + 'ip-src', + 'ip-dst' +] +flex_query_input = [ + 'btc', + 'dkim', + 'email', + 'email-src', + 'email-dst', + 'domain|ip', + 'hex', + 'mac-address', + 'mac-eui-64', + 'other', + 'pattern-filename', + 'target-email', + 'text', + 'uri', + 'url', + 'whois-registrant-email', +] +mispattributes = { + 'input': standard_query_input + flex_query_input, + 'format': 'misp_standard' +} +moduleinfo = { + 'version': '0.5', + 'author': 'Christophe Vandeplas', + 'description': 'Module to access Farsight DNSDB Passive DNS', + 'module-type': ['expansion', 'hover'] +} +moduleconfig = ['apikey', 'server', 'limit', 'flex_queries'] -server = 'https://api.dnsdb.info' +DEFAULT_DNSDB_SERVER = 'https://api.dnsdb.info' +DEFAULT_LIMIT = 10 -# TODO return a MISP object with the different attributes +TYPE_TO_FEATURE = { + "btc": "Bitcoin address", + "dkim": "domainkeys identified mail", + "domain": "domain name", + "domain|ip": "domain name / IP address", + "hex": "value in hexadecimal format", + "hostname": "hostname", + "mac-address": "MAC address", + "mac-eui-64": "MAC EUI-64 address", + "pattern-filename": "pattern in the name of a file", + "target-email": "attack target email", + "uri": "Uniform Resource Identifier", + "url": "Uniform Resource Locator", + "whois-registrant-email": "email of a domain's registrant" +} +TYPE_TO_FEATURE.update( + dict.fromkeys( + ("ip-src", "ip-dst"), + "IP address" + ) +) +TYPE_TO_FEATURE.update( + dict.fromkeys( + ("email", "email-src", "email-dst"), + "email address" + ) +) +TYPE_TO_FEATURE.update( + dict.fromkeys( + ("other", "text"), + "text" + ) +) + + +class FarsightDnsdbParser(): + def __init__(self, attribute): + self.attribute = attribute + self.misp_event = MISPEvent() + self.misp_event.add_attribute(**attribute) + self.passivedns_mapping = { + 'bailiwick': {'type': 'text', 'object_relation': 'bailiwick'}, + 'count': {'type': 'counter', 'object_relation': 'count'}, + 'raw_rdata': {'type': 'text', 'object_relation': 'raw_rdata'}, + 'rdata': {'type': 'text', 'object_relation': 'rdata'}, + 'rrname': {'type': 'text', 'object_relation': 'rrname'}, + 'rrtype': {'type': 'text', 'object_relation': 'rrtype'}, + 'time_first': {'type': 'datetime', 'object_relation': 'time_first'}, + 'time_last': {'type': 'datetime', 'object_relation': 'time_last'}, + 'zone_time_first': {'type': 'datetime', 'object_relation': 'zone_time_first'}, + 'zone_time_last': {'type': 'datetime', 'object_relation': 'zone_time_last'} + } + self.comment = 'Result from a %s lookup on DNSDB about the %s: %s' + + def parse_passivedns_results(self, query_response): + for query_type, results in query_response.items(): + comment = self.comment % (query_type, TYPE_TO_FEATURE[self.attribute['type']], self.attribute['value']) + for result in results: + passivedns_object = MISPObject('passive-dns') + if result.get('rdata') and isinstance(result['rdata'], list): + for rdata in result.pop('rdata'): + passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', rdata)) + for feature, value in result.items(): + passivedns_object.add_attribute(**self._parse_attribute(comment, feature, value)) + if result.get('time_first'): + passivedns_object.first_seen = result['time_first'] + if result.get('time_last'): + passivedns_object.last_seen = result['time_last'] + passivedns_object.add_reference(self.attribute['uuid'], 'related-to') + self.misp_event.add_object(passivedns_object) + + def get_results(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object')} + return {'results': results} + + def _parse_attribute(self, comment, feature, value): + attribute = {'value': value, 'comment': comment} + attribute.update(self.passivedns_mapping[feature]) + return attribute def handler(q=False): @@ -19,56 +133,95 @@ def handler(q=False): if not request.get('config') or not request['config'].get('apikey'): misperrors['error'] = 'Farsight DNSDB apikey is missing' return misperrors - client = DnsdbClient(server, request['config']['apikey']) - if request.get('hostname'): - res = lookup_name(client, request['hostname']) - elif request.get('domain'): - res = lookup_name(client, request['domain']) - elif request.get('ip-src'): - res = lookup_ip(client, request['ip-src']) - elif request.get('ip-dst'): - res = lookup_ip(client, request['ip-dst']) - else: - misperrors['error'] = "Unsupported attributes type" - return misperrors - - out = '' - for v in set(res): # uniquify entries - out = out + "{} ".format(v) - r = {'results': [{'types': mispattributes['output'], 'values': out}]} - return r - - -def lookup_name(client, name): + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + attribute = request['attribute'] + if attribute['type'] not in mispattributes['input']: + return {'error': 'Unsupported attributes type'} + config = request['config'] + if not config.get('server'): + config['server'] = DEFAULT_DNSDB_SERVER + client_args = {feature: config[feature] for feature in ('apikey', 'server')} + client = dnsdb2.Client(**client_args) + to_query, args = parse_input(attribute, config) try: - res = client.query_rrset(name) # RRSET = entries in the left-hand side of the domain name related labels - for item in res: - if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: - for i in item.get('rdata'): - yield(i.rstrip('.')) - if item.get('rrtype') in ['SOA']: - for i in item.get('rdata'): - # grab email field and replace first dot by @ to convert to an email address - yield(i.split(' ')[1].rstrip('.').replace('.', '@', 1)) - except QueryError: - pass - - try: - res = client.query_rdata_name(name) # RDATA = entries on the right-hand side of the domain name related labels - for item in res: - if item.get('rrtype') in ['A', 'AAAA', 'CNAME']: - yield(item.get('rrname').rstrip('.')) - except QueryError: - pass + response = to_query(client, *args) + except dnsdb2.DnsdbException as e: + return {'error': e.__str__()} + if not response: + return {'error': f"Empty results on Farsight DNSDB for the {TYPE_TO_FEATURE[attribute['type']]}: {attribute['value']}."} + parser = FarsightDnsdbParser(attribute) + parser.parse_passivedns_results(response) + return parser.get_results() -def lookup_ip(client, ip): - try: - res = client.query_rdata_ip(ip) - for item in res: - yield(item['rrname'].rstrip('.')) - except QueryError: - pass +def parse_input(attribute, config): + lookup_args = { + 'limit': config['limit'] if config.get('limit') else DEFAULT_LIMIT, + 'offset': 0, + 'ignore_limited': True, + 'humantime': True + } + if attribute.get('first_seen'): + lookup_args['time_first_after'] = parse_timestamp(attribute['first_seen']) + attribute_type = attribute['type'] + if attribute_type in flex_query_input: + return flex_queries, (lookup_args, attribute['value']) + flex = add_flex_queries(config.get('flex_queries')) + to_query = lookup_ip if 'ip-' in attribute_type else lookup_name + return to_query, (lookup_args, attribute['value'], flex) + + +def parse_timestamp(str_date): + datetime_date = datetime.strptime(str_date, '%Y-%m-%dT%H:%M:%S.%f%z') + return str(int(datetime_date.timestamp())) + + +def add_flex_queries(flex): + if not flex: + return False + if flex in ('True', 'true', True, '1', 1): + return True + return False + + +def flex_queries(client, lookup_args, name): + response = {} + name = name.replace('@', '.') + for feature in ('rdata', 'rrnames'): + to_call = getattr(client, f'flex_{feature}_regex') + results = list(to_call(name, **lookup_args)) + for result in list(to_call(name.replace('.', '\\.'), **lookup_args)): + if result not in results: + results.append(result) + if results: + response[f'flex_{feature}'] = results + return response + + +def lookup_name(client, lookup_args, name, flex): + response = {} + # RRSET = entries in the left-hand side of the domain name related labels + rrset_response = list(client.lookup_rrset(name, **lookup_args)) + if rrset_response: + response['rrset'] = rrset_response + # RDATA = entries on the right-hand side of the domain name related labels + rdata_response = list(client.lookup_rdata_name(name, **lookup_args)) + if rdata_response: + response['rdata'] = rdata_response + if flex: + response.update(flex_queries(client, lookup_args, name)) + return response + + +def lookup_ip(client, lookup_args, ip, flex): + response = {} + res = list(client.lookup_rdata_ip(ip, **lookup_args)) + if res: + response['rdata'] = res + if flex: + response.update(flex_queries(client, lookup_args, ip)) + return response def introspection(): diff --git a/misp_modules/modules/expansion/google_search.py b/misp_modules/modules/expansion/google_search.py index b7b4e7a..68224ab 100644 --- a/misp_modules/modules/expansion/google_search.py +++ b/misp_modules/modules/expansion/google_search.py @@ -1,6 +1,8 @@ import json +import random +import time try: - from google import google + from googleapi import google except ImportError: print("GoogleAPI not installed. Command : pip install git+https://github.com/abenassi/Google-Search-API") @@ -10,6 +12,10 @@ moduleinfo = {'author': 'Oun & Gindt', 'module-type': ['hover'], 'description': 'An expansion hover module to expand google search information about an URL'} +def sleep(retry): + time.sleep(random.uniform(0, min(40, 0.01 * 2 ** retry))) + + def handler(q=False): if q is False: return False @@ -18,10 +24,16 @@ def handler(q=False): return {'error': "Unsupported attributes type"} num_page = 1 res = "" - search_results = google.search(request['url'], num_page) - for i in range(3): + # The googleapi module sets a random useragent. The output depends on the useragent. + # It's better to retry 3 times. + for retry in range(3): + search_results = google.search(request['url'], num_page) + if len(search_results) > 0: + break + sleep(retry) + for i, search_result in enumerate(search_results): res += "("+str(i+1)+")" + '\t' - res += json.dumps(search_results[i].description, ensure_ascii=False) + res += json.dumps(search_result.description, ensure_ascii=False) res += '\n\n' return {'results': [{'types': mispattributes['output'], 'values':res}]} diff --git a/misp_modules/modules/expansion/greynoise.py b/misp_modules/modules/expansion/greynoise.py index dd54158..19b4653 100644 --- a/misp_modules/modules/expansion/greynoise.py +++ b/misp_modules/modules/expansion/greynoise.py @@ -1,37 +1,112 @@ import requests import json -misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-dst', 'ip-src'], 'output': ['text']} -moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab ', 'description': 'Module to access GreyNoise.io API.', 'module-type': ['hover']} -moduleconfig = ['user-agent'] # TODO take this into account in the code - -greynoise_api_url = 'http://api.greynoise.io:8888/v1/query/ip' -default_user_agent = 'MISP-Module' +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-dst", "ip-src"], "output": ["text"]} +moduleinfo = { + "version": "1.0", + "author": "Brad Chiappetta ", + "description": "Module to access GreyNoise.io API.", + "module-type": ["hover"], +} +moduleconfig = ["api_key", "api_type"] +codes_mapping = { + "0x00": "The IP has never been observed scanning the Internet", + "0x01": "The IP has been observed by the GreyNoise sensor network", + "0x02": "The IP has been observed scanning the GreyNoise sensor network, " + "but has not completed a full connection, meaning this can be spoofed", + "0x03": "The IP is adjacent to another host that has been directly observed by " + "the GreyNoise sensor network", + "0x04": "Reserved", + "0x05": "This IP is commonly spoofed in Internet-scan activity", + "0x06": "This IP has been observed as noise, but this host belongs to a cloud " + "provider where IPs can be cycled frequently", + "0x07": "This IP is invalid", + "0x08": "This IP was classified as noise, but has not been observed engaging in " + "Internet-wide scans or attacks in over 60 days", +} -def handler(q=False): +def handler(q=False): # noqa: C901 if q is False: return False request = json.loads(q) - for input_type in mispattributes['input']: + if not request.get("config") or not request["config"].get("api_key"): + return {"error": "Missing Greynoise API key."} + if request["config"]["api_type"] and request["config"]["api_type"] == "enterprise": + greynoise_api_url = "https://api.greynoise.io/v2/noise/quick/" + else: + greynoise_api_url = "https://api.greynoise.io/v3/community/" + + headers = { + "Accept": "application/json", + "key": request["config"]["api_key"], + "User-Agent": "greynoise-misp-module-{}".format(moduleinfo["version"]), + } + for input_type in mispattributes["input"]: if input_type in request: ip = request[input_type] break else: - misperrors['error'] = "Unsupported attributes type" + misperrors["error"] = "Unsupported attributes type." return misperrors - data = {'ip': ip} - r = requests.post(greynoise_api_url, data=data, headers={'user-agent': default_user_agent}) # Real request - if r.status_code == 200: # OK (record found) - response = r.text - if response: - return {'results': [{'types': mispattributes['output'], 'values': response}]} - elif r.status_code == 404: # Not found (not an error) - return {'results': [{'types': mispattributes['output'], 'values': 'No data'}]} - else: # Real error - misperrors['error'] = 'GreyNoise API not accessible (HTTP ' + str(r.status_code) + ')' - return misperrors['error'] + response = requests.get(f"{greynoise_api_url}{ip}", headers=headers) # Real request + if response.status_code == 200: + if request["config"]["api_type"] == "enterprise": + return { + "results": [ + { + "types": ["text"], + "values": codes_mapping[response.json()["code"]], + } + ] + } + elif response.json()["noise"]: + return { + "results": [ + { + "types": ["text"], + "values": "IP Address ({}) has been observed by GreyNoise " + "scanning the internet in the last 90 days. GreyNoise has " + "classified it as {} and it was last seen on {}. For more " + "information visit {}".format( + response.json()["ip"], + response.json()["classification"], + response.json()["last_seen"], + response.json()["link"], + ), + } + ] + } + elif response.json()["riot"]: + return { + "results": [ + { + "types": ["text"], + "values": "IP Address ({}) is part of GreyNoise Project RIOT " + "and likely belongs to a benign service from {}. For more " + "information visit {}".format( + response.json()["ip"], + response.json()["name"], + response.json()["link"], + ), + } + ] + } + # There is an error + errors = { + 400: "Bad request.", + 404: "IP not observed scanning the internet or contained in RIOT data set.", + 401: "Unauthorized. Please check your API key.", + 429: "Too many requests. You've hit the rate-limit.", + } + try: + misperrors["error"] = errors[response.status_code] + except KeyError: + misperrors[ + "error" + ] = f"GreyNoise API not accessible (HTTP {response.status_code})" + return misperrors def introspection(): @@ -39,5 +114,5 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig + moduleinfo["config"] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/hibp.py b/misp_modules/modules/expansion/hibp.py index 8db3fa7..b2d1c16 100644 --- a/misp_modules/modules/expansion/hibp.py +++ b/misp_modules/modules/expansion/hibp.py @@ -1,13 +1,14 @@ +# -*- coding: utf-8 -*- import requests import json misperrors = {'error': 'Error'} -mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']} # All mails as input -moduleinfo = {'version': '0.1', 'author': 'Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API.', 'module-type': ['hover']} -moduleconfig = ['user-agent'] # TODO take this into account in the code +mispattributes = {'input': ['email-dst', 'email-src'], 'output': ['text']} +moduleinfo = {'version': '0.2', 'author': 'Corsin Camichel, Aurélien Schwab', 'description': 'Module to access haveibeenpwned.com API (v3).', 'module-type': ['hover']} +moduleconfig = ['api_key'] -haveibeenpwned_api_url = 'https://api.haveibeenpwned.com/api/v2/breachedaccount/' -default_user_agent = 'MISP-Module' # User agent (must be set, requiered by API)) +haveibeenpwned_api_url = 'https://haveibeenpwned.com/api/v3/breachedaccount/' +API_KEY = "" # details at https://www.troyhunt.com/authentication-and-the-have-i-been-pwned-api/ def handler(q=False): @@ -22,15 +23,21 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - r = requests.get(haveibeenpwned_api_url + email, headers={'user-agent': default_user_agent}) # Real request - if r.status_code == 200: # OK (record found) + if request.get('config') is None or request['config'].get('api_key') is None: + misperrors['error'] = 'Have I Been Pwned authentication is incomplete (no API key)' + return misperrors + else: + API_KEY = request['config'].get('api_key') + + r = requests.get(haveibeenpwned_api_url + email, headers={'hibp-api-key': API_KEY}) + if r.status_code == 200: breaches = json.loads(r.text) if breaches: return {'results': [{'types': mispattributes['output'], 'values': breaches}]} - elif r.status_code == 404: # Not found (not an error) + elif r.status_code == 404: return {'results': [{'types': mispattributes['output'], 'values': 'OK (Not Found)'}]} - else: # Real error - misperrors['error'] = 'haveibeenpwned.com API not accessible (HTTP ' + str(r.status_code) + ')' + else: + misperrors['error'] = f'haveibeenpwned.com API not accessible (HTTP {str(r.status_code)})' return misperrors['error'] diff --git a/misp_modules/modules/expansion/html_to_markdown.py b/misp_modules/modules/expansion/html_to_markdown.py new file mode 100755 index 0000000..228b4bc --- /dev/null +++ b/misp_modules/modules/expansion/html_to_markdown.py @@ -0,0 +1,53 @@ +import json +import requests +from markdownify import markdownify +from bs4 import BeautifulSoup + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['url'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Sami Mokaddem', + 'description': 'Simple HTML fetcher', + 'module-type': ['expansion']} + + +def fetchHTML(url): + r = requests.get(url) + return r.text + + +def stripUselessTags(html): + soup = BeautifulSoup(html, 'html.parser') + toRemove = ['script', 'head', 'header', 'footer', 'meta', 'link'] + for tag in soup.find_all(toRemove): + tag.decompose() + return str(soup) + + +def convertHTML(html): + toStrip = ['a', 'img'] + return markdownify(html, heading_style='ATX', strip=toStrip) + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('url'): + url = request['url'] + else: + return False + html = fetchHTML(url) + html = stripUselessTags(html) + markdown = convertHTML(html) + + r = {'results': [{'types': mispattributes['output'], + 'values':[str(markdown)]}]} + return r + + +def introspection(): + return mispattributes + + +def version(): + return moduleinfo diff --git a/misp_modules/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py index 3c6867c..3a32358 100755 --- a/misp_modules/modules/expansion/ipasn.py +++ b/misp_modules/modules/expansion/ipasn.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import json +from . import check_input_attribute, standard_error_message from pyipasnhistory import IPASNHistory from pymisp import MISPAttribute, MISPEvent, MISPObject @@ -34,11 +35,11 @@ def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('attribute') and request['attribute'].get('type') in mispattributes['input']: - toquery = request['attribute']['value'] - else: - misperrors['error'] = "Unsupported attributes type" - return misperrors + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + if request['attribute']['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} + toquery = request['attribute']['value'] ipasn = IPASNHistory() values = ipasn.query(toquery) diff --git a/misp_modules/modules/expansion/joesandbox_query.py b/misp_modules/modules/expansion/joesandbox_query.py index 1ace259..b9c4987 100644 --- a/misp_modules/modules/expansion/joesandbox_query.py +++ b/misp_modules/modules/expansion/joesandbox_query.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import jbxapi import json +from . import check_input_attribute, checking_error, standard_error_message from joe_parser import JoeParser misperrors = {'error': 'Error'} @@ -27,6 +28,10 @@ def handler(q=False): if not apikey: return {'error': 'No API key provided'} + if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): + return {'error': f'{standard_error_message}, {checking_error} that is the link to the Joe Sandbox report.'} + if request['attribute']['type'] != 'link': + return {'error': 'Unsupported attribute type.'} url = request['attribute']['value'] if "/submissions/" not in url: return {'error': "The URL does not point to a Joe Sandbox analysis."} diff --git a/misp_modules/modules/expansion/lastline_query.py b/misp_modules/modules/expansion/lastline_query.py index 4ce4e47..dbfdf14 100644 --- a/misp_modules/modules/expansion/lastline_query.py +++ b/misp_modules/modules/expansion/lastline_query.py @@ -3,8 +3,8 @@ Module (type "expansion") to query a Lastline report from an analysis link. """ import json - import lastline_api +from . import check_input_attribute, checking_error, standard_error_message misperrors = { @@ -52,6 +52,8 @@ def handler(q=False): try: config = request["config"] auth_data = lastline_api.LastlineAbstractClient.get_login_params_from_dict(config) + if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): + return {'error': f'{standard_error_message}, {checking_error} that is the link to a Lastline analysis.'} analysis_link = request['attribute']['value'] # The API url changes based on the analysis link host name api_url = lastline_api.get_portal_url_from_task_link(analysis_link) diff --git a/misp_modules/modules/expansion/malwarebazaar.py b/misp_modules/modules/expansion/malwarebazaar.py index 4574b75..60739e8 100644 --- a/misp_modules/modules/expansion/malwarebazaar.py +++ b/misp_modules/modules/expansion/malwarebazaar.py @@ -1,5 +1,6 @@ import json import requests +from . import check_input_attribute, checking_error, standard_error_message from pymisp import MISPEvent, MISPObject mispattributes = {'input': ['md5', 'sha1', 'sha256'], @@ -34,7 +35,11 @@ def handler(q=False): if q is False: return False request = json.loads(q) + if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): + return {'error': f'{standard_error_message}, {checking_error} that is the hash to submit to Malware Bazaar.'} attribute = request['attribute'] + if attribute['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} url = 'https://mb-api.abuse.ch/api/v1/' response = requests.post(url, data={'query': 'get_info', 'hash': attribute['value']}).json() query_status = response['query_status'] diff --git a/misp_modules/modules/expansion/ocr_enrich.py b/misp_modules/modules/expansion/ocr_enrich.py index cd6baca..ff0a70c 100644 --- a/misp_modules/modules/expansion/ocr_enrich.py +++ b/misp_modules/modules/expansion/ocr_enrich.py @@ -6,14 +6,21 @@ import pytesseract misperrors = {'error': 'Error'} mispattributes = {'input': ['attachment'], - 'output': ['freetext', 'text']} -moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'output': ['freetext']} +moduleinfo = {'version': '0.2', 'author': 'Sascha Rommelfangen', 'description': 'OCR decoder', 'module-type': ['expansion']} moduleconfig = [] +def filter_decoded(decoded): + for line in decoded.split('\n'): + decoded_line = line.strip('\t\x0b\x0c\r ') + if decoded_line: + yield decoded_line + + def handler(q=False): if q is False: return False @@ -31,9 +38,16 @@ def handler(q=False): image = img_array image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) try: - decoded = pytesseract.image_to_string(image) - return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename}, - {'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]} + decoded = pytesseract.image_to_string(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) + return { + 'results': [ + { + 'types': ['freetext'], + 'values': list(filter_decoded(decoded)), + 'comment': f"OCR from file {filename}" + } + ] + } except Exception as e: print(e) err = "Couldn't analyze file type. Only images are supported right now." diff --git a/misp_modules/modules/expansion/ransomcoindb.py b/misp_modules/modules/expansion/ransomcoindb.py index 2b9b566..0e05855 100644 --- a/misp_modules/modules/expansion/ransomcoindb.py +++ b/misp_modules/modules/expansion/ransomcoindb.py @@ -1,4 +1,5 @@ import json +from . import check_input_attribute, checking_error, standard_error_message from ._ransomcoindb import ransomcoindb from pymisp import MISPObject @@ -28,6 +29,10 @@ def handler(q=False): q = json.loads(q) if "config" not in q or "api-key" not in q["config"]: return {"error": "Ransomcoindb API key is missing"} + if not q.get('attribute') or not check_input_attribute(q['attribute'], requirements=('type', 'value')): + return {'error': f'{standard_error_message}, {checking_error}.'} + if q['attribute']['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} api_key = q["config"]["api-key"] r = {"results": []} diff --git a/misp_modules/modules/expansion/rbl.py b/misp_modules/modules/expansion/rbl.py index 4d7bba5..d3f661e 100644 --- a/misp_modules/modules/expansion/rbl.py +++ b/misp_modules/modules/expansion/rbl.py @@ -3,78 +3,75 @@ import sys try: import dns.resolver - resolver = dns.resolver.Resolver() - resolver.timeout = 0.2 - resolver.lifetime = 0.2 except ImportError: print("dnspython3 is missing, use 'pip install dnspython3' to install it.") sys.exit(0) misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['text']} -moduleinfo = {'version': '0.1', 'author': 'Christian Studer', +moduleinfo = {'version': '0.2', 'author': 'Christian Studer', 'description': 'Check an IPv4 address against known RBLs.', 'module-type': ['expansion', 'hover']} -moduleconfig = [] +moduleconfig = ['timeout'] -rbls = { - 'spam.spamrats.com': 'http://www.spamrats.com', - 'spamguard.leadmon.net': 'http://www.leadmon.net/SpamGuard/', - 'rbl-plus.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'web.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'ix.dnsbl.manitu.net': 'http://www.dnsbl.manitu.net', - 'virus.rbl.jp': 'http://www.rbl.jp', - 'dul.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'bogons.cymru.com': 'http://www.team-cymru.org/Services/Bogons/', - 'psbl.surriel.com': 'http://psbl.surriel.com', - 'misc.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'httpbl.abuse.ch': 'http://dnsbl.abuse.ch', - 'combined.njabl.org': 'http://combined.njabl.org', - 'smtp.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'korea.services.net': 'http://korea.services.net', - 'drone.abuse.ch': 'http://dnsbl.abuse.ch', - 'rbl.efnetrbl.org': 'http://rbl.efnetrbl.org', - 'cbl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', - 'b.barracudacentral.org': 'http://www.barracudacentral.org/rbl/removal-request', - 'bl.spamcannibal.org': 'http://www.spamcannibal.org', - 'xbl.spamhaus.org': 'http://www.spamhaus.org/xbl/', - 'zen.spamhaus.org': 'http://www.spamhaus.org/zen/', - 'rbl.suresupport.com': 'http://suresupport.com/postmaster', - 'db.wpbl.info': 'http://www.wpbl.info', - 'sbl.spamhaus.org': 'http://www.spamhaus.org/sbl/', - 'http.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'csi.cloudmark.com': 'http://www.cloudmark.com/en/products/cloudmark-sender-intelligence/index', - 'rbl.interserver.net': 'http://rbl.interserver.net', - 'ubl.unsubscore.com': 'http://www.lashback.com/blacklist/', - 'dnsbl.sorbs.net': 'http://www.sorbs.net', - 'virbl.bit.nl': 'http://virbl.bit.nl', - 'pbl.spamhaus.org': 'http://www.spamhaus.org/pbl/', - 'socks.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'short.rbl.jp': 'http://www.rbl.jp', - 'dnsbl.dronebl.org': 'http://www.dronebl.org', - 'blackholes.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'truncate.gbudb.net': 'http://www.gbudb.com/truncate/index.jsp', - 'dyna.spamrats.com': 'http://www.spamrats.com', - 'spamrbl.imp.ch': 'http://antispam.imp.ch', - 'spam.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'wormrbl.imp.ch': 'http://antispam.imp.ch', - 'query.senderbase.org': 'http://www.senderbase.org/about', - 'opm.tornevall.org': 'http://dnsbl.tornevall.org', - 'netblock.pedantic.org': 'http://pedantic.org', - 'access.redhawk.org': 'http://www.redhawk.org/index.php?option=com_wrapper&Itemid=33', - 'cdl.anti-spam.org.cn': 'http://www.anti-spam.org.cn/?Locale=en_US', - 'multi.surbl.org': 'http://www.surbl.org', - 'noptr.spamrats.com': 'http://www.spamrats.com', - 'dnsbl.inps.de': 'http://dnsbl.inps.de/index.cgi?lang=en', - 'bl.spamcop.net': 'http://bl.spamcop.net', - 'cbl.abuseat.org': 'http://cbl.abuseat.org', - 'dsn.rfc-ignorant.org': 'http://www.rfc-ignorant.org/policy-dsn.php', - 'zombie.dnsbl.sorbs.net': 'http://www.sorbs.net', - 'dnsbl.njabl.org': 'http://dnsbl.njabl.org', - 'relays.mail-abuse.org': 'http://www.mail-abuse.com/lookup.html', - 'rbl.spamlab.com': 'http://tools.appriver.com/index.aspx?tool=rbl', - 'all.bl.blocklist.de': 'http://www.blocklist.de/en/rbldns.html' -} +rbls = ( + "spam.spamrats.com", + "spamguard.leadmon.net", + "rbl-plus.mail-abuse.org", + "web.dnsbl.sorbs.net", + "ix.dnsbl.manitu.net", + "virus.rbl.jp", + "dul.dnsbl.sorbs.net", + "bogons.cymru.com", + "psbl.surriel.com", + "misc.dnsbl.sorbs.net", + "httpbl.abuse.ch", + "combined.njabl.org", + "smtp.dnsbl.sorbs.net", + "korea.services.net", + "drone.abuse.ch", + "rbl.efnetrbl.org", + "cbl.anti-spam.org.cn", + "b.barracudacentral.org", + "bl.spamcannibal.org", + "xbl.spamhaus.org", + "zen.spamhaus.org", + "rbl.suresupport.com", + "db.wpbl.info", + "sbl.spamhaus.org", + "http.dnsbl.sorbs.net", + "csi.cloudmark.com", + "rbl.interserver.net", + "ubl.unsubscore.com", + "dnsbl.sorbs.net", + "virbl.bit.nl", + "pbl.spamhaus.org", + "socks.dnsbl.sorbs.net", + "short.rbl.jp", + "dnsbl.dronebl.org", + "blackholes.mail-abuse.org", + "truncate.gbudb.net", + "dyna.spamrats.com", + "spamrbl.imp.ch", + "spam.dnsbl.sorbs.net", + "wormrbl.imp.ch", + "query.senderbase.org", + "opm.tornevall.org", + "netblock.pedantic.org", + "access.redhawk.org", + "cdl.anti-spam.org.cn", + "multi.surbl.org", + "noptr.spamrats.com", + "dnsbl.inps.de", + "bl.spamcop.net", + "cbl.abuseat.org", + "dsn.rfc-ignorant.org", + "zombie.dnsbl.sorbs.net", + "dnsbl.njabl.org", + "relays.mail-abuse.org", + "rbl.spamlab.com", + "all.bl.blocklist.de" +) def handler(q=False): @@ -88,18 +85,23 @@ def handler(q=False): else: misperrors['error'] = "Unsupported attributes type" return misperrors - listeds = [] - infos = [] + resolver = dns.resolver.Resolver() + try: + timeout = float(request['config']['timeout']) + except (KeyError, ValueError): + timeout = 0.4 + resolver.timeout = timeout + resolver.lifetime = timeout + infos = {} ipRev = '.'.join(ip.split('.')[::-1]) for rbl in rbls: query = '{}.{}'.format(ipRev, rbl) try: txt = resolver.query(query, 'TXT') - listeds.append(query) - infos.append([str(t) for t in txt]) + infos[query] = [str(t) for t in txt] except Exception: continue - result = "\n".join([f"{listed}: {' - '.join(info)}" for listed, info in zip(listeds, infos)]) + result = "\n".join([f"{rbl}: {' - '.join(info)}" for rbl, info in infos.items()]) if not result: return {'error': 'No data found by querying known RBLs'} return {'results': [{'types': mispattributes.get('output'), 'values': result}]} diff --git a/misp_modules/modules/expansion/recordedfuture.py b/misp_modules/modules/expansion/recordedfuture.py index c42a42b..ccea31b 100644 --- a/misp_modules/modules/expansion/recordedfuture.py +++ b/misp_modules/modules/expansion/recordedfuture.py @@ -1,35 +1,89 @@ import json import logging import requests -from urllib.parse import quote +from requests.exceptions import HTTPError, ProxyError,\ + InvalidURL, ConnectTimeout, ConnectionError +from . import check_input_attribute, checking_error, standard_error_message +import platform +import os +from urllib.parse import quote, urlparse from pymisp import MISPAttribute, MISPEvent, MISPTag, MISPObject -moduleinfo = {'version': '1.0', 'author': 'Recorded Future', - 'description': 'Module to retrieve data from Recorded Future', - 'module-type': ['expansion', 'hover']} +moduleinfo = { + 'version': '1.0.1', + 'author': 'Recorded Future', + 'description': 'Module to retrieve data from Recorded Future', + 'module-type': ['expansion', 'hover'] +} -moduleconfig = ['token'] +moduleconfig = ['token', 'proxy_host', 'proxy_port', 'proxy_username', 'proxy_password'] misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip', 'ip-src', 'ip-dst', 'domain', 'hostname', 'md5', 'sha1', 'sha256', - 'uri', 'url', 'vulnerability', 'weakness'], - 'output': ['ip', 'ip-src', 'ip-dst', 'domain', 'hostname', 'md5', 'sha1', 'sha256', - 'uri', 'url', 'vulnerability', 'weakness', 'email-src', 'text'], - 'format': 'misp_standard'} +ATTRIBUTES = [ + 'ip', + 'ip-src', + 'ip-dst', + 'domain', + 'hostname', + 'md5', + 'sha1', + 'sha256', + 'uri', + 'url', + 'vulnerability', + 'weakness' +] + +mispattributes = { + 'input': ATTRIBUTES, + 'output': ATTRIBUTES + ['email-src', 'text'], + 'format': 'misp_standard' +} LOGGER = logging.getLogger('recorded_future') LOGGER.setLevel(logging.INFO) -def rf_lookup(api_token: str, category: str, ioc: str) -> requests.Response: - """Do a lookup call using Recorded Future's ConnectAPI.""" - auth_header = {"X-RFToken": api_token} - parsed_ioc = quote(ioc, safe='') - url = f'https://api.recordedfuture.com/v2/{category}/{parsed_ioc}?fields=risk%2CrelatedEntities' - response = requests.get(url, headers=auth_header) - response.raise_for_status() - return response +class RequestHandler: + """A class for handling any outbound requests from this module.""" + def __init__(self): + self.session = requests.Session() + self.app_id = f'{os.path.basename(__file__)}/{moduleinfo["version"]} ({platform.platform()}) ' \ + f'misp_enrichment/{moduleinfo["version"]} python-requests/{requests.__version__}' + self.proxies = None + self.rf_token = None + + def get(self, url: str, headers: dict = None) -> requests.Response: + """General get method with proxy error handling.""" + try: + timeout = 7 if self.proxies else None + response = self.session.get(url, headers=headers, proxies=self.proxies, timeout=timeout) + response.raise_for_status() + return response + except (ConnectTimeout, ProxyError, InvalidURL) as error: + msg = 'Error connecting with proxy, please check the Recorded Future app proxy settings.' + LOGGER.error(f'{msg} Error: {error}') + misperrors['error'] = msg + raise + + def rf_lookup(self, category: str, ioc: str) -> requests.Response: + """Do a lookup call using Recorded Future's ConnectAPI.""" + parsed_ioc = quote(ioc, safe='') + url = f'https://api.recordedfuture.com/v2/{category}/{parsed_ioc}?fields=risk%2CrelatedEntities' + headers = {'X-RFToken': self.rf_token, + 'User-Agent': self.app_id} + try: + response = self.get(url, headers) + except HTTPError as error: + msg = f'Error when requesting data from Recorded Future. {error.response}: {error.response.reason}' + LOGGER.error(msg) + misperrors['error'] = msg + raise + return response + + +GLOBAL_REQUEST_HANDLER = RequestHandler() class GalaxyFinder: @@ -37,46 +91,45 @@ class GalaxyFinder: def __init__(self): self.session = requests.Session() self.sources = { - 'RelatedThreatActor': ['https://raw.githubusercontent.com/MISP/misp-galaxy/' - 'main/clusters/threat-actor.json'], - 'RelatedMalware': ['https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/banker.json', - 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/botnet.json', - 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/exploit-kit.json', - 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/rat.json', - 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/ransomware.json', - 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/malpedia.json'] + 'RelatedThreatActor': [ + 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/threat-actor.json' + ], + 'RelatedMalware': [ + 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/banker.json', + 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/botnet.json', + 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/exploit-kit.json', + 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/rat.json', + 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/ransomware.json', + 'https://raw.githubusercontent.com/MISP/misp-galaxy/main/clusters/malpedia.json' + ] } self.galaxy_clusters = {} - def pull_galaxy_cluster(self, related_type: str): + def pull_galaxy_cluster(self, related_type: str) -> None: """Fetches galaxy clusters for the related_type from the remote json files specified as self.sources.""" # Only fetch clusters if not fetched previously if not self.galaxy_clusters.get(related_type): for source in self.sources.get(related_type): - response = self.session.get(source) - if response.ok: + try: + response = GLOBAL_REQUEST_HANDLER.get(source) name = source.split('/')[-1].split('.')[0] self.galaxy_clusters[related_type] = {name: response.json()} - else: - LOGGER.info(f'pull_galaxy_cluster failed for source: {source},' - f' got response: {response}, {response.reason}.') + except ConnectionError as error: + LOGGER.warning(f'pull_galaxy_cluster failed for source: {source}, with error: {error}.') def find_galaxy_match(self, indicator: str, related_type: str) -> str: """Searches the clusters of the related_type for a match with the indicator. :returns the first matching galaxy string or an empty string if no galaxy match is found. """ self.pull_galaxy_cluster(related_type) - try: - for cluster_name, cluster in self.galaxy_clusters[related_type].items(): - for value in cluster['values']: - try: - if indicator in value['meta']['synonyms'] or indicator in value['value']: - value = value['value'] - return f'misp-galaxy:{cluster_name}="{value}"' - except KeyError: - pass - except KeyError: - pass + for cluster_name, cluster in self.galaxy_clusters.get(related_type, {}).items(): + for value in cluster['values']: + try: + if indicator in value['meta']['synonyms'] or indicator in value['value']: + value = value['value'] + return f'misp-galaxy:{cluster_name}="{value}"' + except KeyError: + pass return '' @@ -112,57 +165,70 @@ class RFEnricher: """Class for enriching an attribute with data from Recorded Future. The enrichment data is returned as a custom MISP object. """ - def __init__(self, api_token: str, attribute_props: dict): - self.api_token = api_token + def __init__(self, attribute_props: dict): self.event = MISPEvent() self.enrichment_object = MISPObject('Recorded Future Enrichment') - self.enrichment_object.from_dict(**{'meta-category': 'misc', - 'description': 'An object containing the enriched attribute and related ' - 'entities from Recorded Future.', - 'distribution': 0}) + description = ( + 'An object containing the enriched attribute and ' + 'related entities from Recorded Future.' + ) + self.enrichment_object.from_dict(**{ + 'meta-category': 'misc', + 'description': description, + 'distribution': 0 + }) # Create a copy of enriched attribute to add tags to temp_attr = MISPAttribute() temp_attr.from_dict(**attribute_props) self.enriched_attribute = MISPAttribute() - self.enriched_attribute.from_dict(**{'value': temp_attr.value, 'type': temp_attr.type, 'distribution': 0}) + self.enriched_attribute.from_dict(**{ + 'value': temp_attr.value, + 'type': temp_attr.type, + 'distribution': 0 + }) self.related_attributes = [] self.color_picker = RFColors() self.galaxy_finder = GalaxyFinder() # Mapping from MISP-type to RF-type - self.type_to_rf_category = {'ip': 'ip', 'ip-src': 'ip', 'ip-dst': 'ip', - 'domain': 'domain', 'hostname': 'domain', - 'md5': 'hash', 'sha1': 'hash', 'sha256': 'hash', - 'uri': 'url', 'url': 'url', - 'vulnerability': 'vulnerability', 'weakness': 'vulnerability'} + self.type_to_rf_category = { + 'ip': 'ip', + 'ip-src': 'ip', + 'ip-dst': 'ip', + 'domain': 'domain', + 'hostname': 'domain', + 'md5': 'hash', + 'sha1': 'hash', + 'sha256': 'hash', + 'uri': 'url', + 'url': 'url', + 'vulnerability': 'vulnerability', + 'weakness': 'vulnerability' + } # Related entities from RF portrayed as related attributes in MISP - self.related_attribute_types = ['RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash', - 'RelatedEmailAddress', 'RelatedCyberVulnerability'] + self.related_attribute_types = [ + 'RelatedIpAddress', 'RelatedInternetDomainName', 'RelatedHash', + 'RelatedEmailAddress', 'RelatedCyberVulnerability' + ] # Related entities from RF portrayed as tags in MISP self.galaxy_tag_types = ['RelatedMalware', 'RelatedThreatActor'] - def enrich(self): + def enrich(self) -> None: """Run the enrichment.""" category = self.type_to_rf_category.get(self.enriched_attribute.type) - - try: - response = rf_lookup(self.api_token, category, self.enriched_attribute.value) - json_response = json.loads(response.content) - except requests.HTTPError as error: - misperrors['error'] = f'Error when requesting data from Recorded Future. ' \ - f'{error.response} : {error.response.reason}' - raise error + json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(category, self.enriched_attribute.value) + response = json.loads(json_response.content) try: # Add risk score and risk rules as tags to the enriched attribute - risk_score = json_response['data']['risk']['score'] + risk_score = response['data']['risk']['score'] hex_color = self.color_picker.riskscore_color(risk_score) tag_name = f'recorded-future:risk-score="{risk_score}"' self.add_tag(tag_name, hex_color) - for evidence in json_response['data']['risk']['evidenceDetails']: + for evidence in response['data']['risk']['evidenceDetails']: risk_rule = evidence['rule'] criticality = evidence['criticality'] hex_color = self.color_picker.riskrule_color(criticality) @@ -170,7 +236,7 @@ class RFEnricher: self.add_tag(tag_name, hex_color) # Retrieve related entities - for related_entity in json_response['data']['relatedEntities']: + for related_entity in response['data']['relatedEntities']: related_type = related_entity['type'] if related_type in self.related_attribute_types: # Related entities returned as additional attributes @@ -190,9 +256,9 @@ class RFEnricher: galaxy_tags.append(galaxy) for galaxy in galaxy_tags: self.add_tag(galaxy) - except KeyError as error: + except KeyError: misperrors['error'] = 'Unexpected format in Recorded Future api response.' - raise error + raise def add_related_attribute(self, indicator: str, related_type: str) -> None: """Helper method for adding an indicator to the related attribute list.""" @@ -246,23 +312,73 @@ class RFEnricher: return {'results': result} +def get_proxy_settings(config: dict) -> dict: + """Returns proxy settings in the requests format. + If no proxy settings are set, return None.""" + proxies = None + host = config.get('proxy_host') + port = config.get('proxy_port') + username = config.get('proxy_username') + password = config.get('proxy_password') + + if host: + if not port: + misperrors['error'] = 'The recordedfuture_proxy_host config is set, ' \ + 'please also set the recordedfuture_proxy_port.' + raise KeyError + parsed = urlparse(host) + if 'http' in parsed.scheme: + scheme = 'http' + else: + scheme = parsed.scheme + netloc = parsed.netloc + host = f'{netloc}:{port}' + + if username: + if not password: + misperrors['error'] = 'The recordedfuture_proxy_username config is set, ' \ + 'please also set the recordedfuture_proxy_password.' + raise KeyError + auth = f'{username}:{password}' + host = auth + '@' + host + + proxies = { + 'http': f'{scheme}://{host}', + 'https': f'{scheme}://{host}' + } + + LOGGER.info(f'Proxy settings: {proxies}') + return proxies + + def handler(q=False): """Handle enrichment.""" if q is False: return False request = json.loads(q) - if request.get('config') and request['config'].get('token'): - token = request['config'].get('token') + config = request.get('config') + if config and config.get('token'): + GLOBAL_REQUEST_HANDLER.rf_token = config.get('token') else: misperrors['error'] = 'Missing Recorded Future token.' return misperrors + if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): + return {'error': f'{standard_error_message}, {checking_error}.'} + if request['attribute']['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} + + try: + GLOBAL_REQUEST_HANDLER.proxies = get_proxy_settings(config) + except KeyError: + return misperrors input_attribute = request.get('attribute') - rf_enricher = RFEnricher(token, input_attribute) + rf_enricher = RFEnricher(input_attribute) + try: rf_enricher.enrich() - except (requests.HTTPError, KeyError): + except (HTTPError, ConnectTimeout, ProxyError, InvalidURL, KeyError): return misperrors return rf_enricher.get_results() diff --git a/misp_modules/modules/expansion/shodan.py b/misp_modules/modules/expansion/shodan.py index 5a4b792..f295deb 100755 --- a/misp_modules/modules/expansion/shodan.py +++ b/misp_modules/modules/expansion/shodan.py @@ -5,38 +5,225 @@ try: import shodan except ImportError: print("shodan module not installed.") +from . import check_input_attribute, standard_error_message +from datetime import datetime +from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} -mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} -moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', +mispattributes = {'input': ['ip-src', 'ip-dst'], + 'format': 'misp_standard'} +moduleinfo = {'version': '0.2', 'author': 'Raphaël Vinot', 'description': 'Query on Shodan', 'module-type': ['expansion']} moduleconfig = ['apikey'] +class ShodanParser(): + def __init__(self, attribute): + self.misp_event = MISPEvent() + self.attribute = MISPAttribute() + self.attribute.from_dict(**attribute) + self.misp_event.add_attribute(**self.attribute) + self.ip_address_mapping = { + 'asn': {'type': 'AS', 'object_relation': 'asn'}, + 'city': {'type': 'text', 'object_relation': 'city'}, + 'country_code': {'type': 'text', 'object_relation': 'country-code'}, + 'country_name': {'type': 'text', 'object_relation': 'country'}, + 'isp': {'type': 'text', 'object_relation': 'ISP'}, + 'latitude': {'type': 'float', 'object_relation': 'latitude'}, + 'longitude': {'type': 'float', 'object_relation': 'longitude'}, + 'org': {'type': 'text', 'object_relation': 'organization'}, + 'postal_code': {'type': 'text', 'object_relation': 'zipcode'}, + 'region_code': {'type': 'text', 'object_relation': 'region-code'} + } + self.ip_port_mapping = { + 'domains': {'type': 'domain', 'object_relation': 'domain'}, + 'hostnames': {'type': 'hostname', 'object_relation': 'hostname'} + } + self.vulnerability_mapping = { + 'cvss': {'type': 'float', 'object_relation': 'cvss-score'}, + 'summary': {'type': 'text', 'object_relation': 'summary'} + } + self.x509_mapping = { + 'bits': {'type': 'text', 'object_relation': 'pubkey-info-size'}, + 'expires': {'type': 'datetime', 'object_relation': 'validity-not-after'}, + 'issued': {'type': 'datetime', 'object_relation': 'validity-not-before'}, + 'issuer': {'type': 'text', 'object_relation': 'issuer'}, + 'serial': {'type': 'text', 'object_relation': 'serial-number'}, + 'sig_alg': {'type': 'text', 'object_relation': 'signature_algorithm'}, + 'subject': {'type': 'text', 'object_relation': 'subject'}, + 'type': {'type': 'text', 'object_relation': 'pubkey-info-algorithm'}, + 'version': {'type': 'text', 'object_relation': 'version'} + } + + def query_shodan(self, apikey): + # Query Shodan and get the results in a json blob + api = shodan.Shodan(apikey) + query_results = api.host(self.attribute.value) + + # Parse the information about the IP address used as input + ip_address_attributes = [] + for feature, mapping in self.ip_address_mapping.items(): + if query_results.get(feature): + attribute = {'value': query_results[feature]} + attribute.update(mapping) + ip_address_attributes.append(attribute) + if ip_address_attributes: + ip_address_object = MISPObject('ip-api-address') + for attribute in ip_address_attributes: + ip_address_object.add_attribute(**attribute) + ip_address_object.add_attribute(**self._get_source_attribute()) + ip_address_object.add_reference(self.attribute.uuid, 'describes') + self.misp_event.add_object(ip_address_object) + + # Parse the hostnames / domains and ports associated with the IP address + if query_results.get('ports'): + ip_port_object = MISPObject('ip-port') + ip_port_object.add_attribute(**self._get_source_attribute()) + feature = self.attribute.type.split('-')[1] + for port in query_results['ports']: + attribute = { + 'type': 'port', + 'object_relation': f'{feature}-port', + 'value': port + } + ip_port_object.add_attribute(**attribute) + for feature, mapping in self.ip_port_mapping.items(): + for value in query_results.get(feature, []): + attribute = {'value': value} + attribute.update(mapping) + ip_port_object.add_attribute(**attribute) + ip_port_object.add_reference(self.attribute.uuid, 'extends') + self.misp_event.add_object(ip_port_object) + else: + if any(query_results.get(feature) for feature in ('domains', 'hostnames')): + domain_ip_object = MISPObject('domain-ip') + domain_ip_object.add_attribute(**self._get_source_attribute()) + for feature in ('domains', 'hostnames'): + for value in query_results[feature]: + attribute = { + 'type': 'domain', + 'object_relation': 'domain', + 'value': value + } + domain_ip_object.add_attribute(**attribute) + domain_ip_object.add_reference(self.attribute.uuid, 'extends') + self.misp_event.add_object(domain_ip_object) + + # Parse data within the "data" field + if query_results.get('vulns'): + vulnerabilities = {} + for data in query_results['data']: + # Parse vulnerabilities + if data.get('vulns'): + for cve, vulnerability in data['vulns'].items(): + if cve not in vulnerabilities: + vulnerabilities[cve] = vulnerability + # Also parse the certificates + if data.get('ssl'): + self._parse_cert(data['ssl']) + for cve, vulnerability in vulnerabilities.items(): + vulnerability_object = MISPObject('vulnerability') + vulnerability_object.add_attribute(**{ + 'type': 'vulnerability', + 'object_relation': 'id', + 'value': cve + }) + for feature, mapping in self.vulnerability_mapping.items(): + if vulnerability.get(feature): + attribute = {'value': vulnerability[feature]} + attribute.update(mapping) + vulnerability_object.add_attribute(**attribute) + if vulnerability.get('references'): + for reference in vulnerability['references']: + vulnerability_object.add_attribute(**{ + 'type': 'link', + 'object_relation': 'references', + 'value': reference + }) + vulnerability_object.add_reference(self.attribute.uuid, 'vulnerability-of') + self.misp_event.add_object(vulnerability_object) + for cve_id in query_results['vulns']: + if cve_id not in vulnerabilities: + attribute = { + 'type': 'vulnerability', + 'value': cve_id + } + self.misp_event.add_attribute(**attribute) + else: + # We have no vulnerability data, we only check if we have + # certificates within the "data" field + for data in query_results['data']: + if data.get('ssl'): + self._parse_cert(data['ssl']['cert']) + + def get_result(self): + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + + # When we want to add the IP address information in objects such as the + # domain-ip or ip-port objects referencing the input IP address attribute + def _get_source_attribute(self): + return { + 'type': self.attribute.type, + 'object_relation': self.attribute.type, + 'value': self.attribute.value + } + + def _parse_cert(self, certificate): + x509_object = MISPObject('x509') + for feature in ('serial', 'sig_alg', 'version'): + if certificate.get(feature): + attribute = {'value': certificate[feature]} + attribute.update(self.x509_mapping[feature]) + x509_object.add_attribute(**attribute) + # Parse issuer and subject value + for feature in ('issuer', 'subject'): + if certificate.get(feature): + attribute_value = (f'{identifier}={value}' for identifier, value in certificate[feature].items()) + attribute = {'value': f'/{"/".join(attribute_value)}'} + attribute.update(self.x509_mapping[feature]) + x509_object.add_attribute(**attribute) + # Parse datetime attributes + for feature in ('expires', 'issued'): + if certificate.get(feature): + attribute = {'value': datetime.strptime(certificate[feature], '%Y%m%d%H%M%SZ')} + attribute.update(self.x509_mapping[feature]) + x509_object.add_attribute(**attribute) + # Parse fingerprints + if certificate.get('fingerprint'): + for hash_type, hash_value in certificate['fingerprint'].items(): + x509_object.add_attribute(**{ + 'type': f'x509-fingerprint-{hash_type}', + 'object_relation': f'x509-fingerprint-{hash_type}', + 'value': hash_value + }) + # Parse public key related info + if certificate.get('pubkey'): + for feature, value in certificate['pubkey'].items(): + attribute = {'value': value} + attribute.update(self.x509_mapping[feature]) + x509_object.add_attribute(**attribute) + x509_object.add_reference(self.attribute.uuid, 'identifies') + self.misp_event.add_object(x509_object) + + def handler(q=False): if q is False: return False request = json.loads(q) - if request.get('ip-src'): - toquery = request['ip-src'] - elif request.get('ip-dst'): - toquery = request['ip-dst'] - else: - misperrors['error'] = "Unsupported attributes type" - return misperrors - - if not request.get('config') or not request['config'].get('apikey'): - misperrors['error'] = 'Shodan authentication is missing' - return misperrors - api = shodan.Shodan(request['config'].get('apikey')) - - return handle_expansion(api, toquery) - - -def handle_expansion(api, domain): - return {'results': [{'types': mispattributes['output'], 'values': json.dumps(api.host(domain))}]} + if not request.get('config', {}).get('apikey'): + return {'error': 'Shodan authentication is missing'} + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + attribute = request['attribute'] + if attribute['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} + shodan_parser = ShodanParser(attribute) + shodan_parser.query_shodan(request['config']['apikey']) + return shodan_parser.get_result() def introspection(): diff --git a/misp_modules/modules/expansion/socialscan.py b/misp_modules/modules/expansion/socialscan.py new file mode 100644 index 0000000..54f58f6 --- /dev/null +++ b/misp_modules/modules/expansion/socialscan.py @@ -0,0 +1,101 @@ +import json +from socialscan.platforms import Platforms +from socialscan.util import sync_execute_queries + +moduleinfo = { + 'version': '1', + 'author': 'Christian Studer', + 'description': 'Module to query several online platforms to look for existing accounts.', + 'module-type': ['hover'] +} +mispattributes = { + 'input': [ + 'github-username', + 'target-user', + 'email', + 'email-src', + 'email-dst', + 'target-email', + 'whois-registrant-email' + ], + 'output': ['text'] +} +moduleconfig = [] + +_PLATFORMS = [ + Platforms.INSTAGRAM, + Platforms.TWITTER, + Platforms.GITHUB, + Platforms.TUMBLR, + Platforms.LASTFM +] +_EMAIL_PLATFORMS = [ + Platforms.PINTEREST, + Platforms.SPOTIFY, + Platforms.FIREFOX +] +_EMAIL_PLATFORMS.extend(_PLATFORMS) +_USERNAME_PLATFORMS = [ + Platforms.SNAPCHAT, + Platforms.GITLAB, + Platforms.REDDIT, + Platforms.YAHOO +] +_USERNAME_PLATFORMS.extend(_PLATFORMS) + + +def parse_results(query_results, feature): + results = [] + for result in query_results: + if not result.success: + results.append(f'Unable to retrieve the {feature} on {result.platform}.') + continue + if not result.valid: + results.append(f'Invalid response from {result.platform}, or invalid {feature}.') + continue + statement = 'No account' if result.available else 'There is an account' + results.append(f'{statement} linked to the {feature} on {result.platform}.') + to_return = [ + { + 'types': mispattributes['output'], + 'values': result + } for result in results + ] + return {'results': to_return} + + +def parse_email(email): + results = sync_execute_queries([email], platforms=_EMAIL_PLATFORMS) + return parse_results(results, 'email address') + + +def parse_username(username, platforms=_USERNAME_PLATFORMS): + results = sync_execute_queries([username], platforms=platforms) + return parse_results(results, 'username') + + +def parse_github_username(username): + return parse_username(username, platforms=[Platforms.GITHUB]) + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + if request.get('github-username'): + return parse_github_username(request['github-username']) + if request.get('target-user'): + return parse_username(request['target-user']) + for attribute_type in mispattributes['input'][2:]: + if request.get(attribute_type): + return parse_email(request[attribute_type]) + return {'error': 'Unsupported attributes type'} + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/sophoslabs_intelix.py b/misp_modules/modules/expansion/sophoslabs_intelix.py index 017683a..4d7c413 100644 --- a/misp_modules/modules/expansion/sophoslabs_intelix.py +++ b/misp_modules/modules/expansion/sophoslabs_intelix.py @@ -1,7 +1,8 @@ -from pymisp import MISPEvent, MISPObject import json import requests import base64 +from . import check_input_attribute, checking_error, standard_error_message +from pymisp import MISPEvent, MISPObject from urllib.parse import quote moduleinfo = {'version': '1.0', @@ -105,13 +106,25 @@ def handler(q=False): misperrors['error'] = "Missing client_id or client_secret value for SOPHOSLabs Intelix. \ It's free to sign up here https://aws.amazon.com/marketplace/pp/B07SLZPMCS." return misperrors + to_check = (('type', 'value'), ('type', 'value1')) + if not j.get('attribute') or not any(check_input_attribute(j['attribute'], requirements=check) for check in to_check): + return {'error': f'{standard_error_message}, {checking_error}.'} + attribute = j['attribute'] + if attribute['type'] not in misp_types_in: + return {'error': 'Unsupported attribute type.'} client = SophosLabsApi(j['config']['client_id'], j['config']['client_secret']) - if j['attribute']['type'] == "sha256": - client.hash_lookup(j['attribute']['value1']) - if j['attribute']['type'] in ['ip-dst', 'ip-src', 'ip']: - client.ip_lookup(j["attribute"]["value1"]) - if j['attribute']['type'] in ['uri', 'url', 'domain', 'hostname']: - client.url_lookup(j["attribute"]["value1"]) + mapping = { + 'sha256': 'hash_lookup', + 'ip-dst': 'ip_lookup', + 'ip-src': 'ip_lookup', + 'ip': 'ip_lookup', + 'uri': 'url_lookup', + 'url': 'url_lookup', + 'domain': 'url_lookup', + 'hostname': 'url_lookup' + } + attribute_value = attribute['value'] if 'value' in attribute else attribute['value1'] + getattr(client, mapping[attribute['type']])(attribute_value) return client.get_result() diff --git a/misp_modules/modules/expansion/threatfox.py b/misp_modules/modules/expansion/threatfox.py new file mode 100644 index 0000000..4a89918 --- /dev/null +++ b/misp_modules/modules/expansion/threatfox.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +import requests +import json + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['md5', 'sha1', 'sha256', 'domain', 'url', 'email-src', 'ip-dst|port', 'ip-src|port'], 'output': ['text']} +moduleinfo = {'version': '0.1', 'author': 'Corsin Camichel', 'description': 'Module to search for an IOC on ThreatFox by abuse.ch.', 'module-type': ['hover', 'expansion']} +moduleconfig = [] + +API_URL = "https://threatfox-api.abuse.ch/api/v1/" + + +# copied from +# https://github.com/marjatech/threatfox2misp/blob/main/threatfox2misp.py +def confidence_level_to_tag(level: int) -> str: + confidence_tagging = { + 0: 'misp:confidence-level="unconfident"', + 10: 'misp:confidence-level="rarely-confident"', + 37: 'misp:confidence-level="fairly-confident"', + 63: 'misp:confidence-level="usually-confident"', + 90: 'misp:confidence-level="completely-confident"', + } + + confidence_tag = "" + for tag_minvalue, tag in confidence_tagging.items(): + if level >= tag_minvalue: + confidence_tag = tag + return confidence_tag + + +def handler(q=False): + if q is False: + return False + + request = json.loads(q) + ret_val = "" + + for input_type in mispattributes['input']: + if input_type in request: + to_query = request[input_type] + break + else: + misperrors['error'] = "Unsupported attributes type:" + return misperrors + + data = {"query": "search_ioc", "search_term": f"{to_query}"} + response = requests.post(API_URL, data=json.dumps(data)) + if response.status_code == 200: + result = json.loads(response.text) + if(result["query_status"] == "ok"): + confidence_tag = confidence_level_to_tag(result["data"][0]["confidence_level"]) + ret_val = {'results': [{'types': mispattributes['output'], 'values': [result["data"][0]["threat_type_desc"]], 'tags': [result["data"][0]["malware"], result["data"][0]["malware_printable"], confidence_tag]}]} + + return ret_val + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/expansion/trustar_enrich.py b/misp_modules/modules/expansion/trustar_enrich.py index efe7c53..b7ee2a4 100644 --- a/misp_modules/modules/expansion/trustar_enrich.py +++ b/misp_modules/modules/expansion/trustar_enrich.py @@ -1,7 +1,11 @@ import json import pymisp +from base64 import b64encode +from collections import OrderedDict +from . import check_input_attribute, checking_error, standard_error_message from pymisp import MISPAttribute, MISPEvent, MISPObject -from trustar import TruStar +from trustar import TruStar, Indicator +from urllib.parse import quote misperrors = {'error': "Error"} mispattributes = { @@ -33,9 +37,13 @@ class TruSTARParser: 'SHA256': "sha256" } + # Relevant fields from each TruSTAR endpoint + SUMMARY_FIELDS = ["severityLevel", "source", "score", "attributes"] + METADATA_FIELDS = ["sightings", "firstSeen", "lastSeen", "tags"] + REPORT_BASE_URL = "https://station.trustar.co/constellation/reports/{}" - CLIENT_METATAG = "MISP-{}".format(pymisp.__version__) + CLIENT_METATAG = f"MISP-{pymisp.__version__}" def __init__(self, attribute, config): config['enclave_ids'] = config.get('enclave_ids', "").strip().split(',') @@ -51,45 +59,111 @@ class TruSTARParser: """ Returns the MISP Event enriched with TruSTAR indicator summary data. """ - event = json.loads(self.misp_event.to_json()) - results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} - return {'results': results} + try: + event = json.loads(self.misp_event.to_json()) + results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} + return {'results': results} + except Exception as e: + misperrors['error'] += f" -- Encountered issue serializing enrichment data -- {e}" + return misperrors - def generate_trustar_links(self, entity_value): + def generate_trustar_link(self, entity_type, entity_value): """ - Generates links to TruSTAR reports if they exist. + Generates link to TruSTAR report of entity. + :param entity_type: Type of entity. :param entity_value: Value of entity. + :return: Link to indicator report in TruSTAR platform. """ - report_links = list() - trustar_reports = self.ts_client.search_reports(entity_value) - for report in trustar_reports: - report_links.append(self.REPORT_BASE_URL.format(report.id)) + report_id = b64encode(quote(f"{entity_type}|{entity_value}").encode()).decode() - return report_links + return self.REPORT_BASE_URL.format(report_id) - def parse_indicator_summary(self, summaries): + @staticmethod + def extract_tags(enrichment_report): """ - Converts a response from the TruSTAR /1.3/indicators/summaries endpoint - a MISP trustar_report object and adds the summary data and links as attributes. + Extracts tags from the enrichment report in order to add them + to the TruSTAR MISP Object. Removes tags from report to avoid + redundancy. - :param summaries: A TruSTAR Python SDK Page.generator object for generating - indicator summaries pages. + :param: Enrichment data. + :return: List of tags. + """ + if enrichment_report and enrichment_report.get('tags'): + return [tag.get('name') for tag in enrichment_report.pop('tags')] + return None + + def generate_enrichment_report(self, summary, metadata): + """ + Extracts desired fields from summary and metadata reports and + generates an enrichment report. + + :param summary: Indicator summary report. + :param metadata: Indicator metadata report. + :return: Enrichment report. + """ + # Preserve order of fields as they exist in SUMMARY_FIELDS and METADATA_FIELDS + enrichment_report = OrderedDict() + + if summary: + summary_dict = summary.to_dict() + enrichment_report.update( + {field: summary_dict[field] for field in self.SUMMARY_FIELDS if summary_dict.get(field)}) + + if metadata: + metadata_dict = metadata.to_dict() + enrichment_report.update( + {field: metadata_dict[field] for field in self.METADATA_FIELDS if metadata_dict.get(field)}) + + return enrichment_report + + def parse_indicator_summary(self, indicator, summary, metadata): + """ + Pulls enrichment data from the TruSTAR /indicators/summaries and /indicators/metadata endpoints + and creates a MISP trustar_report. + + :param indicator: Value of the attribute + :summary: Indicator summary response object. + :metadata: Indicator response object. """ - for summary in summaries: - trustar_obj = MISPObject('trustar_report') + # Verify that the indicator type is supported by TruSTAR + if summary and summary.indicator_type in self.ENTITY_TYPE_MAPPINGS: indicator_type = summary.indicator_type - indicator_value = summary.value - if indicator_type in self.ENTITY_TYPE_MAPPINGS: + elif metadata and metadata.type in self.ENTITY_TYPE_MAPPINGS: + indicator_type = metadata.type + else: + misperrors['error'] += " -- Attribute not found or not supported" + raise Exception + + try: + # Extract most relevant fields from indicator summary and metadata responses + enrichment_report = self.generate_enrichment_report(summary, metadata) + tags = self.extract_tags(enrichment_report) + + if enrichment_report: + # Create MISP trustar_report object and populate it with enrichment data + trustar_obj = MISPObject('trustar_report') trustar_obj.add_attribute(indicator_type, attribute_type=self.ENTITY_TYPE_MAPPINGS[indicator_type], - value=indicator_value) + value=indicator) trustar_obj.add_attribute("INDICATOR_SUMMARY", attribute_type="text", - value=json.dumps(summary.to_dict(), sort_keys=True, indent=4)) - report_links = self.generate_trustar_links(indicator_value) - for link in report_links: - trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=link) + value=json.dumps(enrichment_report, indent=4)) + + report_link = self.generate_trustar_link(indicator_type, indicator) + trustar_obj.add_attribute("REPORT_LINK", attribute_type="link", value=report_link) + self.misp_event.add_object(**trustar_obj) + elif not tags: + # If enrichment report is empty and there are no tags, nothing to add to attribute + raise Exception("No relevant data found") + + if tags: + for tag in tags: + self.misp_event.add_attribute_tag(tag, indicator) + + except Exception as e: + misperrors['error'] += f" -- Error enriching attribute {indicator} -- {e}" + raise e def handler(q=False): @@ -110,17 +184,35 @@ def handler(q=False): misperrors['error'] = "Your TruSTAR API key and secret are required for indicator enrichment." return misperrors + if not request.get('attribute') or not check_input_attribute(request['attribute'], requirements=('type', 'value')): + return {'error': f'{standard_error_message}, {checking_error}.'} attribute = request['attribute'] + if attribute['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} trustar_parser = TruSTARParser(attribute, config) + metadata = None + summary = None try: - summaries = list( - trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE)) + metadata = trustar_parser.ts_client.get_indicators_metadata([Indicator(value=attribute['value'])])[0] + except IndexError: + misperrors['error'] += f" -- No metadata found for indicator {attribute['value']}" except Exception as e: - misperrors['error'] = "Unable to retrieve TruSTAR summary data: {}".format(e) + misperrors['error'] += f" -- Could not retrieve indicator metadata from TruSTAR {e}" + + try: + summary = list( + trustar_parser.ts_client.get_indicator_summaries([attribute['value']], page_size=MAX_PAGE_SIZE))[0] + except IndexError: + misperrors['error'] += f" -- No summary data found for indicator {attribute['value']}" + except Exception as e: + misperrors['error'] += f" -- Unable to retrieve TruSTAR summary data: {e}" + + try: + trustar_parser.parse_indicator_summary(attribute['value'], summary, metadata) + except Exception: return misperrors - trustar_parser.parse_indicator_summary(summaries) return trustar_parser.get_results() diff --git a/misp_modules/modules/expansion/urlhaus.py b/misp_modules/modules/expansion/urlhaus.py index baaaaf6..ed13b77 100644 --- a/misp_modules/modules/expansion/urlhaus.py +++ b/misp_modules/modules/expansion/urlhaus.py @@ -1,6 +1,8 @@ -from pymisp import MISPAttribute, MISPEvent, MISPObject +# -*- coding: utf-8 -*- import json import requests +from . import check_input_attribute, standard_error_message +from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} mispattributes = {'input': ['domain', 'hostname', 'ip-src', 'ip-dst', 'md5', 'sha256', 'url'], @@ -134,7 +136,11 @@ def handler(q=False): if q is False: return False request = json.loads(q) + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} attribute = request['attribute'] + if attribute['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} urlhaus_parser = _misp_type_mapping[attribute['type']](attribute) return urlhaus_parser.query_api() diff --git a/misp_modules/modules/expansion/virustotal.py b/misp_modules/modules/expansion/virustotal.py index b09de81..f5f29c5 100644 --- a/misp_modules/modules/expansion/virustotal.py +++ b/misp_modules/modules/expansion/virustotal.py @@ -1,6 +1,7 @@ -from pymisp import MISPAttribute, MISPEvent, MISPObject import json import requests +from . import check_input_attribute, standard_error_message +from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], @@ -143,7 +144,7 @@ class VirusTotalParser(object): def parse_resolutions(self, resolutions, subdomains=None, uuids=None): domain_ip_object = MISPObject('domain-ip') - if self.attribute.type == 'domain': + if self.attribute.type in ('domain', 'hostname'): domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value) attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address') else: @@ -174,7 +175,7 @@ class VirusTotalParser(object): vt_object = MISPObject('virustotal-report') vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) - vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) + vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True) self.misp_event.add_object(**vt_object) return vt_object.uuid @@ -195,6 +196,11 @@ def handler(q=False): if not request.get('config') or not request['config'].get('apikey'): misperrors['error'] = "A VirusTotal api key is required for this module." return misperrors + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} + if request['attribute']['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} + event_limit = request['config'].get('event_limit') if not isinstance(event_limit, int): event_limit = 5 diff --git a/misp_modules/modules/expansion/virustotal_public.py b/misp_modules/modules/expansion/virustotal_public.py index e7c2e96..989e48d 100644 --- a/misp_modules/modules/expansion/virustotal_public.py +++ b/misp_modules/modules/expansion/virustotal_public.py @@ -1,6 +1,7 @@ -from pymisp import MISPAttribute, MISPEvent, MISPObject import json import requests +from . import check_input_attribute, standard_error_message +from pymisp import MISPAttribute, MISPEvent, MISPObject misperrors = {'error': 'Error'} mispattributes = {'input': ['hostname', 'domain', "ip-src", "ip-dst", "md5", "sha1", "sha256", "url"], @@ -36,7 +37,7 @@ class VirusTotalParser(): def parse_resolutions(self, resolutions, subdomains=None, uuids=None): domain_ip_object = MISPObject('domain-ip') - if self.attribute.type == 'domain': + if self.attribute.type in ('domain', 'hostname'): domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value) attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address') else: @@ -174,7 +175,11 @@ def handler(q=False): if not request.get('config') or not request['config'].get('apikey'): misperrors['error'] = "A VirusTotal api key is required for this module." return misperrors + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'} attribute = request['attribute'] + if attribute['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} query_type, to_call = misp_type_mapping[attribute['type']] parser = to_call(request['config']['apikey'], attribute) query_result = parser.get_query_result(query_type) diff --git a/misp_modules/modules/expansion/vmray_submit.py b/misp_modules/modules/expansion/vmray_submit.py index 1c0d553..fa0a073 100644 --- a/misp_modules/modules/expansion/vmray_submit.py +++ b/misp_modules/modules/expansion/vmray_submit.py @@ -19,7 +19,7 @@ from distutils.util import strtobool import io import zipfile -from ._vmray.vmray_rest_api import VMRayRESTAPI +from _vmray.rest_api import VMRayRESTAPI misperrors = {'error': 'Error'} mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']} diff --git a/misp_modules/modules/expansion/xforceexchange.py b/misp_modules/modules/expansion/xforceexchange.py index 7999ce2..936917f 100644 --- a/misp_modules/modules/expansion/xforceexchange.py +++ b/misp_modules/modules/expansion/xforceexchange.py @@ -1,6 +1,7 @@ import requests import json import sys +from . import check_input_attribute, standard_error_message from collections import defaultdict from pymisp import MISPAttribute, MISPEvent, MISPObject from requests.auth import HTTPBasicAuth @@ -160,6 +161,10 @@ def handler(q=False): return misperrors key = request["config"]["apikey"] password = request['config']['apipassword'] + if not request.get('attribute') or not check_input_attribute(request['attribute']): + return {'error': f'{standard_error_message} which should contain at least a type, a value and an uuid.'} + if request['attribute']['type'] not in mispattributes['input']: + return {'error': 'Unsupported attribute type.'} parser = XforceExchange(request['attribute'], key, password) parser.parse() return parser.get_result() diff --git a/misp_modules/modules/export_mod/__init__.py b/misp_modules/modules/export_mod/__init__.py index 1b0e1d0..5b69d02 100644 --- a/misp_modules/modules/export_mod/__init__.py +++ b/misp_modules/modules/export_mod/__init__.py @@ -1,2 +1,2 @@ __all__ = ['cef_export', 'mass_eql_export', 'liteexport', 'goamlexport', 'threat_connect_export', 'pdfexport', - 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph'] + 'threatStream_misp_export', 'osqueryexport', 'nexthinkexport', 'vt_graph', 'defender_endpoint_export'] diff --git a/misp_modules/modules/export_mod/defender_endpoint_export.py b/misp_modules/modules/export_mod/defender_endpoint_export.py new file mode 100755 index 0000000..1c36efb --- /dev/null +++ b/misp_modules/modules/export_mod/defender_endpoint_export.py @@ -0,0 +1,109 @@ +""" +Export module for coverting MISP events into Defender for Endpoint KQL queries. +Config['Period'] : allows to define period over witch to look for IOC from now +""" + +import base64 +import json + +misperrors = {"error": "Error"} + +types_to_use = ['sha1', 'md5', 'domain', 'ip', 'url'] + +userConfig = { + +} + +moduleconfig = ["Period"] +inputSource = ['event'] + +outputFileExtension = 'kql' +responseType = 'application/txt' + +moduleinfo = {'version': '1.0', 'author': 'Julien Bachmann, Hacknowledge', + 'description': 'Defender for Endpoint KQL hunting query export module', + 'module-type': ['export']} + + +def handle_sha1(value, period): + query = f"""find in (DeviceAlertEvents, DeviceFileEvents, DeviceImageLoadEvents, DeviceProcessEvents) + where SHA1 == '{value}' or InitiatingProcessSHA1 == '{value}'""" + return query.replace('\n', ' ') + + +def handle_md5(value, period): + query = f"""find in (DeviceAlertEvents, DeviceFileEvents, DeviceImageLoadEvents, DeviceProcessEvents) + where MD5 == '{value}' or InitiatingProcessMD5 == '{value}'""" + return query.replace('\n', ' ') + + +def handle_domain(value, period): + query = f"""find in (DeviceAlertEvents, DeviceNetworkEvents) + where RemoteUrl contains '{value}'""" + return query.replace('\n', ' ') + + +def handle_ip(value, period): + query = f"""find in (DeviceAlertEvents, DeviceNetworkEvents) + where RemoteIP == '{value}'""" + return query.replace('\n', ' ') + + +def handle_url(value, period): + query = f"""find in (DeviceAlertEvents, DeviceNetworkEvents) + where RemoteUrl startswith '{value}'""" + return query.replace('\n', ' ') + + +handlers = { + 'sha1': handle_sha1, + 'md5': handle_md5, + 'domain': handle_domain, + 'ip': handle_ip, + 'url': handle_url +} + + +def handler(q=False): + if q is False: + return False + request = json.loads(q) + config = request.get("config", {"Period": ""}) + output = '' + + for event in request["data"]: + for attribute in event["Attribute"]: + if attribute['type'] in types_to_use: + output = output + handlers[attribute['type']](attribute['value'], config['Period']) + '\n' + r = {"response": [], "data": str(base64.b64encode(bytes(output, 'utf-8')), 'utf-8')} + return r + + +def introspection(): + modulesetup = {} + try: + responseType + modulesetup['responseType'] = responseType + except NameError: + pass + try: + userConfig + modulesetup['userConfig'] = userConfig + except NameError: + pass + try: + outputFileExtension + modulesetup['outputFileExtension'] = outputFileExtension + except NameError: + pass + try: + inputSource + modulesetup['inputSource'] = inputSource + except NameError: + pass + return modulesetup + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo diff --git a/misp_modules/modules/import_mod/__init__.py b/misp_modules/modules/import_mod/__init__.py index fbad911..694a434 100644 --- a/misp_modules/modules/import_mod/__init__.py +++ b/misp_modules/modules/import_mod/__init__.py @@ -1,4 +1,3 @@ -from . import _vmray # noqa import os import sys sys.path.append('{}/lib'.format('/'.join((os.path.realpath(__file__)).split('/')[:-3]))) diff --git a/misp_modules/modules/import_mod/_vmray/__init__.py b/misp_modules/modules/import_mod/_vmray/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/misp_modules/modules/import_mod/email_import.py b/misp_modules/modules/import_mod/email_import.py index 114f8c9..3ebf3a2 100644 --- a/misp_modules/modules/import_mod/email_import.py +++ b/misp_modules/modules/import_mod/email_import.py @@ -42,7 +42,7 @@ def handler(q=False): # request data is always base 64 byte encoded data = base64.b64decode(request["data"]) - email_object = EMailObject(pseudofile=BytesIO(data), attach_original_mail=True, standalone=False) + email_object = EMailObject(pseudofile=BytesIO(data), attach_original_email=True, standalone=False) # Check if we were given a configuration config = request.get("config", {}) @@ -110,21 +110,20 @@ def handler(q=False): email_object.add_reference(f_object.uuid, 'includes', 'Email attachment') mail_body = email_object.email.get_body(preferencelist=('html', 'plain')) - if extract_urls: - if mail_body: - charset = mail_body.get_content_charset() - if mail_body.get_content_type() == 'text/html': - url_parser = HTMLURLParser() - url_parser.feed(mail_body.get_payload(decode=True).decode(charset, errors='ignore')) - urls = url_parser.urls - else: - urls = re.findall(r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+', mail_body.get_payload(decode=True).decode(charset, errors='ignore')) - for url in urls: - if not url: - continue - url_object = URLObject(url, standalone=False) - file_objects.append(url_object) - email_object.add_reference(url_object.uuid, 'includes', 'URL in email body') + if extract_urls and mail_body: + charset = mail_body.get_content_charset('utf-8') + if mail_body.get_content_type() == 'text/html': + url_parser = HTMLURLParser() + url_parser.feed(mail_body.get_payload(decode=True).decode(charset, errors='ignore')) + urls = url_parser.urls + else: + urls = re.findall(r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+', mail_body.get_payload(decode=True).decode(charset, errors='ignore')) + for url in urls: + if not url: + continue + url_object = URLObject(url, standalone=False) + file_objects.append(url_object) + email_object.add_reference(url_object.uuid, 'includes', 'URL in email body') objects = [email_object.to_json()] if file_objects: diff --git a/misp_modules/modules/import_mod/vmray_import.py b/misp_modules/modules/import_mod/vmray_import.py index 824c970..8385634 100644 --- a/misp_modules/modules/import_mod/vmray_import.py +++ b/misp_modules/modules/import_mod/vmray_import.py @@ -6,8 +6,6 @@ Import VMRay results. This version supports import from different analyze jobs, starting from one sample (the supplied sample_id). -Requires "vmray_rest_api" - The expansion module vmray_submit and import module vmray_import are a two step process to import data from VMRay. You can automate this by setting the PyMISP example script 'vmray_automation' @@ -17,378 +15,72 @@ as a cron job import json -from ._vmray.vmray_rest_api import VMRayRESTAPI +from _vmray.parser import VMRayParser, VMRayParseError + misperrors = {'error': 'Error'} -inputSource = [] -moduleinfo = {'version': '0.2', 'author': 'Koen Van Impe', - 'description': 'Import VMRay results', + +moduleinfo = {'version': '0.4', 'author': 'Jens Thom (VMRay), Koen van Impe', + 'description': 'Import VMRay analysis results from a server', 'module-type': ['import']} -userConfig = {'include_analysisid': {'type': 'Boolean', - 'message': 'Include link to VMRay analysis' - }, - 'include_analysisdetails': {'type': 'Boolean', - 'message': 'Include (textual) analysis details' - }, - 'include_vtidetails': {'type': 'Boolean', - 'message': 'Include VMRay Threat Identifier (VTI) rules' - }, - 'include_imphash_ssdeep': {'type': 'Boolean', - 'message': 'Include imphash and ssdeep' - }, - 'include_extracted_files': {'type': 'Boolean', - 'message': 'Include extracted files section' - }, - 'sample_id': {'type': 'Integer', - 'errorMessage': 'Expected a sample ID', - 'message': 'The VMRay sample_id' - } - } +mispattributes = { + 'inputSource': [], + 'output': ['MISP objects'], + 'format': 'misp_standard', +} -moduleconfig = ['apikey', 'url', 'wait_period'] +userConfig = { + "Sample ID": { + "type": "Integer", + "errorMessage": "The VMRay sample ID to download the reports", + }, + "VTI": { + "type": "Boolean", + "message": "Include VMRay Threat Identifiers", + "checked": "True" + }, + "IOCs": { + "type": "Boolean", + "message": "Include IOCs", + "checked": "True" + }, + "Artifacts": { + "type": "Boolean", + "message": "Include other Artifacts", + }, + "Analysis Details": { + "type": "Boolean", + "message": "Include Analysis Details", + "checked": "True" + } +} + +moduleconfig = ["apikey", "url", "disable_tags", "disable_misp_objects", "ignore_analysis_finished"] def handler(q=False): - global include_analysisid, include_imphash_ssdeep, include_extracted_files, include_analysisdetails, include_vtidetails, include_static_to_ids - if q is False: return False request = json.loads(q) - include_analysisid = bool(int(request["config"].get("include_analysisid"))) - include_imphash_ssdeep = bool(int(request["config"].get("include_imphash_ssdeep"))) - include_extracted_files = bool(int(request["config"].get("include_extracted_files"))) - include_analysisdetails = bool(int(request["config"].get("include_extracted_files"))) - include_vtidetails = bool(int(request["config"].get("include_vtidetails"))) - include_static_to_ids = True - - # print("include_analysisid: %s include_imphash_ssdeep: %s include_extracted_files: %s include_analysisdetails: %s include_vtidetails: %s" % ( include_analysisid, include_imphash_ssdeep, include_extracted_files, include_analysisdetails, include_vtidetails)) - - sample_id = int(request["config"].get("sample_id")) - - if (request["config"].get("apikey") is None) or (request["config"].get("url") is None): - misperrors["error"] = "Missing API key or server URL (hint: try cloud.vmray.com)" + parser = VMRayParser() + try: + parser.from_api(request["config"]) + parser.parse() + except VMRayParseError as exc: + misperrors["error"] = str(exc) return misperrors - if sample_id > 0: - try: - api = VMRayRESTAPI(request["config"].get("url"), request["config"].get("apikey"), False) - vmray_results = {'results': []} - - # Get all information on the sample, returns a set of finished analyze jobs - data = vmrayGetInfoAnalysis(api, sample_id) - if data["data"]: - for analysis in data["data"]: - analysis_id = int(analysis["analysis_id"]) - if analysis_id > 0: - # Get the details for an analyze job - analysis_data = vmrayDownloadAnalysis(api, analysis_id) - - if analysis_data: - if include_analysisdetails and "analysis_details" in analysis_data: - analysis_details = vmrayAnalysisDetails(analysis_data["analysis_details"], analysis_id) - if analysis_details and len(analysis_details["results"]) > 0: - vmray_results = {'results': vmray_results["results"] + analysis_details["results"]} - - if "classifications" in analysis_data: - classifications = vmrayClassifications(analysis_data["classifications"], analysis_id) - if classifications and len(classifications["results"]) > 0: - vmray_results = {'results': vmray_results["results"] + classifications["results"]} - - if include_extracted_files and "extracted_files" in analysis_data: - extracted_files = vmrayExtractedfiles(analysis_data["extracted_files"]) - if extracted_files and len(extracted_files["results"]) > 0: - vmray_results = {'results': vmray_results["results"] + extracted_files["results"]} - - if include_vtidetails and "vti" in analysis_data: - vti = vmrayVti(analysis_data["vti"]) - if vti and len(vti["results"]) > 0: - vmray_results = {'results': vmray_results["results"] + vti["results"]} - - if "artifacts" in analysis_data: - artifacts = vmrayArtifacts(analysis_data["artifacts"]) - if artifacts and len(artifacts["results"]) > 0: - vmray_results = {'results': vmray_results["results"] + artifacts["results"]} - - if include_analysisid: - a_id = {'results': []} - url1 = request["config"].get("url") + "/user/analysis/view?from_sample_id=%u" % sample_id - url2 = "&id=%u" % analysis_id - url3 = "&sub=%2Freport%2Foverview.html" - a_id["results"].append({"values": url1 + url2 + url3, "types": "link"}) - vmray_results = {'results': vmray_results["results"] + a_id["results"]} - - # Clean up (remove doubles) - if len(vmray_results["results"]) > 0: - vmray_results = vmrayCleanup(vmray_results) - return vmray_results - else: - misperrors['error'] = "No vti_results returned or jobs not finished" - return misperrors - else: - if "result" in data: - if data["result"] == "ok": - return vmray_results - - # Fallback - misperrors['error'] = "Unable to fetch sample id %u" % (sample_id) - return misperrors - except Exception as e: # noqa - misperrors['error'] = "Unable to access VMRay API : %s" % (e) - return misperrors - else: - misperrors['error'] = "Not a valid sample id" - return misperrors + event = parser.to_json() + return event def introspection(): - modulesetup = {} - try: - userConfig - modulesetup['userConfig'] = userConfig - except NameError: - pass - try: - inputSource - modulesetup['inputSource'] = inputSource - except NameError: - pass - return modulesetup + mispattributes["userConfig"] = userConfig + return mispattributes def version(): moduleinfo['config'] = moduleconfig return moduleinfo - - -def vmrayGetInfoAnalysis(api, sample_id): - ''' Get information from a sample, returns a set of analyzed reports''' - - if sample_id: - data = api.call("GET", "/rest/analysis/sample/%u" % (sample_id), raw_data=True) - return json.loads(data.read().decode()) - else: - return False - - -def vmrayDownloadAnalysis(api, analysis_id): - ''' Get the details from an analysis''' - if analysis_id: - try: - data = api.call("GET", "/rest/analysis/%u/archive/logs/summary.json" % (analysis_id), raw_data=True) - return json.loads(data.read().decode()) - except Exception as e: # noqa - misperrors['error'] = "Unable to download summary.json for analysis %s" % (analysis_id) - return misperrors - else: - return False - - -def vmrayVti(vti): - '''VMRay Threat Identifier (VTI) rules that matched for this analysis''' - - if vti: - r = {'results': []} - for rule in vti: - if rule == "vti_rule_matches": - vti_rule = vti["vti_rule_matches"] - for el in vti_rule: - if "operation_desc" in el: - comment = "" - types = ["text"] - values = el["operation_desc"] - r['results'].append({'types': types, 'values': values, 'comment': comment}) - - return r - - else: - return False - - -def vmrayExtractedfiles(extracted_files): - ''' Information about files which were extracted during the analysis, such as files that were created, modified, or embedded by the malware''' - - if extracted_files: - r = {'results': []} - - for file in extracted_files: - if "file_type" and "norm_filename" in file: - comment = "%s - %s" % (file["file_type"], file["norm_filename"]) - else: - comment = "" - - if "norm_filename" in file: - attr_filename_c = file["norm_filename"].rsplit("\\", 1) - if len(attr_filename_c) > 1: - attr_filename = attr_filename_c[len(attr_filename_c) - 1] - else: - attr_filename = "vmray_sample" - else: - attr_filename = "vmray_sample" - - if "md5_hash" in file and file["md5_hash"] is not None: - r['results'].append({'types': ["filename|md5"], 'values': '{}|{}'.format(attr_filename, file["md5_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if include_imphash_ssdeep and "imp_hash" in file and file["imp_hash"] is not None: - r['results'].append({'types': ["filename|imphash"], 'values': '{}|{}'.format(attr_filename, file["imp_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if "sha1_hash" in file and file["sha1_hash"] is not None: - r['results'].append({'types': ["filename|sha1"], 'values': '{}|{}'.format(attr_filename, file["sha1_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if "sha256_hash" in file and file["sha256_hash"] is not None: - r['results'].append({'types': ["filename|sha256"], 'values': '{}|{}'.format(attr_filename, file["sha256_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if include_imphash_ssdeep and "ssdeep_hash" in file and file["ssdeep_hash"] is not None: - r['results'].append({'types': ["filename|ssdeep"], 'values': '{}|{}'.format(attr_filename, file["ssdeep_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - - return r - - else: - return False - - -def vmrayClassifications(classification, analysis_id): - ''' List the classifications, tag them on a "text" attribute ''' - - if classification: - r = {'results': []} - types = ["text"] - comment = "" - values = "Classification : %s " % (", ".join(str(x) for x in classification)) - r['results'].append({'types': types, 'values': values, 'comment': comment}) - - return r - - else: - return False - - -def vmrayAnalysisDetails(details, analysis_id): - ''' General information about the analysis information ''' - - if details: - r = {'results': []} - types = ["text"] - comment = "" - if "execution_successful" in details: - values = "Analysis %s : execution_successful : %s " % (analysis_id, str(details["execution_successful"])) - r['results'].append({'types': types, 'values': values, 'comment': comment}) - if "termination_reason" in details: - values = "Analysis %s : termination_reason : %s " % (analysis_id, str(details["termination_reason"])) - r['results'].append({'types': types, 'values': values, 'comment': comment}) - if "result_str" in details: - values = "Analysis %s : result : %s " % (analysis_id, details["result_str"]) - r['results'].append({'types': types, 'values': values, 'comment': comment}) - - return r - - else: - return False - - -def vmrayArtifacts(patterns): - ''' IOCs that were seen during the analysis ''' - - if patterns: - r = {'results': []} - y = {'results': []} - - for pattern in patterns: - if pattern == "domains": - for el in patterns[pattern]: - values = el["domain"] - types = ["domain", "hostname"] - if "sources" in el: - sources = el["sources"] - comment = "Found in: " + ", ".join(str(x) for x in sources) - else: - comment = "" - r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids}) - if pattern == "files": - for el in patterns[pattern]: - filename_values = el["filename"] - attr_filename_c = filename_values.rsplit("\\", 1) - if len(attr_filename_c) > 1: - attr_filename = attr_filename_c[len(attr_filename_c) - 1] - else: - attr_filename = "" - filename_types = ["filename"] - filename_operations = el["operations"] - comment = "File operations: " + ", ".join(str(x) for x in filename_operations) - r['results'].append({'types': filename_types, 'values': filename_values, 'comment': comment}) - - # Run through all hashes - if "hashes" in el: - for hash in el["hashes"]: - if "md5_hash" in hash and hash["md5_hash"] is not None: - r['results'].append({'types': ["filename|md5"], 'values': '{}|{}'.format(attr_filename, hash["md5_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if include_imphash_ssdeep and "imp_hash" in hash and hash["imp_hash"] is not None: - r['results'].append({'types': ["filename|imphash"], 'values': '{}|{}'.format(attr_filename, hash["imp_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if "sha1_hash" in hash and hash["sha1_hash"] is not None: - r['results'].append({'types': ["filename|sha1"], 'values': '{}|{}'.format(attr_filename, hash["sha1_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if "sha256_hash" in hash and hash["sha256_hash"] is not None: - r['results'].append({'types': ["filename|sha256"], 'values': '{}|{}'.format(attr_filename, hash["sha256_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if include_imphash_ssdeep and "ssdeep_hash" in hash and hash["ssdeep_hash"] is not None: - r['results'].append({'types': ["filename|ssdeep"], 'values': '{}|{}'.format(attr_filename, hash["ssdeep_hash"]), 'comment': comment, 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': include_static_to_ids}) - if pattern == "ips": - for el in patterns[pattern]: - values = el["ip_address"] - types = ["ip-dst"] - if "sources" in el: - sources = el["sources"] - comment = "Found in: " + ", ".join(str(x) for x in sources) - else: - comment = "" - - r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids}) - if pattern == "mutexes": - for el in patterns[pattern]: - values = el["mutex_name"] - types = ["mutex"] - if "operations" in el: - sources = el["operations"] - comment = "Operations: " + ", ".join(str(x) for x in sources) - else: - comment = "" - - r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids}) - if pattern == "registry": - for el in patterns[pattern]: - values = el["reg_key_name"] - types = ["regkey"] - include_static_to_ids_tmp = include_static_to_ids - if "operations" in el: - sources = el["operations"] - if sources == ["access"]: - include_static_to_ids_tmp = False - comment = "Operations: " + ", ".join(str(x) for x in sources) - else: - comment = "" - - r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids_tmp}) - if pattern == "urls": - for el in patterns[pattern]: - values = el["url"] - types = ["url"] - if "operations" in el: - sources = el["operations"] - comment = "Operations: " + ", ".join(str(x) for x in sources) - else: - comment = "" - - r['results'].append({'types': types, 'values': values, 'comment': comment, 'to_ids': include_static_to_ids}) - - # Remove doubles - for el in r["results"]: - if el not in y["results"]: - y["results"].append(el) - return y - - else: - return False - - -def vmrayCleanup(x): - ''' Remove doubles''' - y = {'results': []} - for el in x["results"]: - if el not in y["results"]: - y["results"].append(el) - return y diff --git a/misp_modules/modules/import_mod/vmray_summary_json_import.py b/misp_modules/modules/import_mod/vmray_summary_json_import.py new file mode 100644 index 0000000..e7f4985 --- /dev/null +++ b/misp_modules/modules/import_mod/vmray_summary_json_import.py @@ -0,0 +1,80 @@ +import json + +from _vmray.parser import VMRayParser, VMRayParseError + + +misperrors = {'error': 'Error'} + +moduleconfig = ["disable_tags"] + +moduleinfo = { + "version": "0.1", + "author": "VMRay", + "description": "Import a VMRay Summary JSON report.", + "module-type": ["import"], +} + +mispattributes = { + "inputSource": ["file"], + "output": ["MISP objects", "MISP attributes"], + "format": "misp_standard", +} + +user_config = { + "Analysis ID": { + "type": "Boolean", + "message": "Include Analysis ID", + "checked": "True" + }, + "VTI": { + "type": "Boolean", + "message": "Include VMRay Threat Identifiers", + "checked": "True" + }, + "IOCs": { + "type": "Boolean", + "message": "Include IOCs", + "checked": "True" + }, + "Artifacts": { + "type": "Boolean", + "message": "Include other Artifacts", + }, + "Analysis Details": { + "type": "Boolean", + "message": "Include Analysis Details", + }, + "Attach Report": { + "type": "Boolean", + "message": "Include the original imported file as attachment", + } +} + + +def handler(q=False): + # In case there's no data + if q is False: + return False + + q = json.loads(q) + + parser = VMRayParser() + try: + parser.from_base64_string(q["config"], q["data"], q["filename"]) + parser.parse() + except VMRayParseError as exc: + misperrors["error"] = str(exc) + return misperrors + + event = parser.to_json() + return event + + +def introspection(): + mispattributes["userConfig"] = user_config + return mispattributes + + +def version(): + moduleinfo["config"] = moduleconfig + return moduleinfo diff --git a/mkdocs.yml b/mkdocs.yml index be23ba7..bafd3d6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -16,7 +16,7 @@ edit_uri: "" use_directory_urls: true # Copyright -copyright: "Copyright © 2019 MISP Project" +copyright: "Copyright © 2019-2021 MISP Project" # Options extra: diff --git a/tests/test_expansions.py b/tests/test_expansions.py index b853c25..84958ac 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -8,6 +8,7 @@ from base64 import b64encode import json import os +LiveCI = True class TestExpansions(unittest.TestCase): @@ -97,9 +98,16 @@ class TestExpansions(unittest.TestCase): self.assertEqual(self.get_errors(response), 'An API key for APIVoid is required.') def test_bgpranking(self): - query = {"module": "bgpranking", "AS": "13335"} + query = { + "module": "bgpranking", + "attribute": { + "type": "AS", + "value": "13335", + "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d" + } + } response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response)['response']['asn_description'], 'CLOUDFLARENET, US') + self.assertEqual(self.get_object(response), 'asn') def test_btc_steroids(self): query = {"module": "btc_steroids", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} @@ -108,7 +116,7 @@ class TestExpansions(unittest.TestCase): self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC / -0.0005355700 BTC)')) except Exception: - self.assertEqual(self.get_values(response), 'Not a valid BTC address, or Balance has changed') + self.assertTrue(self.get_values(response).startswith('Not a valid BTC address')) def test_btc_scam_check(self): query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"} @@ -137,7 +145,7 @@ class TestExpansions(unittest.TestCase): module_name = "circl_passivessl" query = {"module": module_name, "attribute": {"type": "ip-dst", - "value": "149.13.33.14", + "value": "185.194.93.14", "uuid": "ea89a33b-4ab7-4515-9f02-922a0bee333d"}, "config": {}} if module_name in self.configs: @@ -192,7 +200,7 @@ class TestExpansions(unittest.TestCase): def test_dns(self): query = {"module": "dns", "hostname": "www.circl.lu", "config": {"nameserver": "8.8.8.8"}} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), '149.13.33.14') + self.assertEqual(self.get_values(response), '185.194.93.14') def test_docx(self): filename = 'test.docx' @@ -214,26 +222,43 @@ class TestExpansions(unittest.TestCase): try: self.assertIn(result, self.get_values(response)) except Exception: - self.assertTrue(self.get_errors(response).startwith('Something went wrong')) + self.assertTrue(self.get_errors(response).startswith('Something went wrong')) else: query = {"module": module_name, "ip-src": "8.8.8.8"} response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), 'Farsight DNSDB apikey is missing') def test_haveibeenpwned(self): + module_name = 'hibp' query = {"module": "hibp", "email-src": "info@circl.lu"} response = self.misp_modules_post(query) - to_check = self.get_values(response) - if to_check == "haveibeenpwned.com API not accessible (HTTP 401)": - self.skipTest(f"haveibeenpwned blocks travis IPs: {response}") - self.assertEqual(to_check, 'OK (Not Found)', response) + if module_name in self.configs: + to_check = self.get_values(response) + if to_check == "haveibeenpwned.com API not accessible (HTTP 401)": + self.skipTest(f"haveibeenpwned blocks travis IPs: {response}") + self.assertEqual(to_check, 'OK (Not Found)', response) + else: + self.assertEqual(self.get_errors(response), 'Have I Been Pwned authentication is incomplete (no API key)') def test_greynoise(self): - query = {"module": "greynoise", "ip-dst": "1.1.1.1"} - response = self.misp_modules_post(query) - value = self.get_values(response) - if value != 'GreyNoise API not accessible (HTTP 429)': - self.assertTrue(value.startswith('{"ip":"1.1.1.1","status":"ok"')) + module_name = 'greynoise' + query = {"module": module_name, "ip-dst": "1.1.1.1"} + if module_name in self.configs: + query['config'] = self.configs[module_name] + response = self.misp_modules_post(query) + try: + self.assertEqual(self.get_values(response), 'This IP is commonly spoofed in Internet-scan activity') + except Exception: + self.assertIn( + self.get_errors(reponse), + ( + "Unauthorized. Please check your API key.", + "Too many requests. You've hit the rate-limit." + ) + ) + else: + response = self.misp_modules_post(query) + self.assertEqual(self.get_errors(response), 'Missing Greynoise API key.') def test_ipasn(self): query = {"module": "ipasn", @@ -265,7 +290,7 @@ class TestExpansions(unittest.TestCase): encoded = b64encode(f.read()).decode() query = {"module": "ocr_enrich", "attachment": filename, "data": encoded} response = self.misp_modules_post(query) - self.assertEqual(self.get_values(response), 'Threat Sharing') + self.assertEqual(self.get_values(response).strip('\n'), 'Threat Sharing') def test_ods(self): filename = 'test.ods' @@ -328,12 +353,12 @@ class TestExpansions(unittest.TestCase): def test_passivetotal(self): module_name = "passivetotal" - query = {"module": module_name, "ip-src": "149.13.33.14", "config": {}} + query = {"module": module_name, "ip-src": "185.194.93.14", "config": {}} if module_name in self.configs: query["config"] = self.configs[module_name] response = self.misp_modules_post(query) try: - self.assertEqual(self.get_values(response), 'circl.lu') + self.assertIn('www.circl.lu', response.json()['results'][0]['values']) except Exception: self.assertIn(self.get_errors(response), ('We hit an error, time to bail!', 'API quota exceeded.')) else: @@ -374,10 +399,12 @@ class TestExpansions(unittest.TestCase): self.assertEqual(self.get_errors(response), "Ransomcoindb API key is missing") def test_rbl(self): + if LiveCI: + return True query = {"module": "rbl", "ip-src": "8.8.8.8"} response = self.misp_modules_post(query) try: - self.assertTrue(self.get_values(response).startswith('8.8.8.8.query.senderbase.org: "0-0=1|1=GOOGLE')) + self.assertTrue(self.get_values(response).startswith('8.8.8.8.bl.spamcannibal.org')) except Exception: self.assertEqual(self.get_errors(response), "No data found by querying known RBLs") @@ -406,11 +433,18 @@ class TestExpansions(unittest.TestCase): def test_shodan(self): module_name = "shodan" - query = {"module": module_name, "ip-src": "149.13.33.14"} + query = { + "module": module_name, + "attribute": { + "uuid": "a21aae0c-7426-4762-9b79-854314d69059", + "type": "ip-src", + "value": "149.13.33.14" + } + } if module_name in self.configs: query['config'] = self.configs[module_name] response = self.misp_modules_post(query) - self.assertIn("circl.lu", self.get_values(response)) + self.assertEqual(self.get_object(response), 'ip-api-address') else: response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), 'Shodan authentication is missing') @@ -436,17 +470,20 @@ class TestExpansions(unittest.TestCase): query = {"module": "stix2_pattern_syntax_validator", "stix2-pattern": "[ipv4-addr:value = '8.8.8.8']"} response = self.misp_modules_post(query) self.assertEqual(self.get_values(response), 'Syntax valid') - def test_threatcrowd(self): + if LiveCI: + return True query_types = ('domain', 'ip-src', 'md5', 'whois-registrant-email') - query_values = ('circl.lu', '149.13.33.4', '616eff3e9a7575ae73821b4668d2801c', 'hostmaster@eurodns.com') - results = ('149.13.33.14', 'cve.circl.lu', 'devilreturns.com', 'navabi.lu') + query_values = ('circl.lu', '149.13.33.14', '616eff3e9a7575ae73821b4668d2801c', 'hostmaster@eurodns.com') + results = ('149.13.33.4', 'cve.circl.lu', 'devilreturns.com', 'navabi.lu') for query_type, query_value, result in zip(query_types, query_values, results): query = {"module": "threatcrowd", query_type: query_value} response = self.misp_modules_post(query) self.assertTrue(self.get_values(response), result) def test_threatminer(self): + if LiveCI: + return True query_types = ('domain', 'ip-src', 'md5') query_values = ('circl.lu', '149.13.33.4', 'b538dbc6160ef54f755a540e06dc27cd980fc4a12005e90b3627febb44a1a90f') results = ('149.13.33.14', 'f6ecb9d5c21defb1f622364a30cb8274f817a1a2', 'http://www.circl.lu/') @@ -489,16 +526,33 @@ class TestExpansions(unittest.TestCase): def test_virustotal_public(self): module_name = "virustotal_public" - query_types = ('domain', 'ip-src', 'sha256', 'url') - query_values = ('circl.lu', '149.13.33.14', - 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - 'http://194.169.88.56:49151/.i') + attributes = ( + { + "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3", + "type": "domain", + "value": "circl.lu" + }, + { + "uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979", + "type": "ip-src", + "value": "149.13.33.14" + }, + { + "uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f", + "type": "sha256", + "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3" + }, + { + "uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51", + "type": "url", + "value": "http://194.169.88.56:49151/.i" + } + ) results = ('whois', 'asn', 'file', 'virustotal-report') if module_name in self.configs: - for query_type, query_value, result in zip(query_types, query_values, results): + for attribute, result in zip(attributes, results): query = {"module": module_name, - "attribute": {"type": query_type, - "value": query_value}, + "attribute": attribute, "config": self.configs[module_name]} response = self.misp_modules_post(query) try: @@ -506,24 +560,42 @@ class TestExpansions(unittest.TestCase): except Exception: self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") else: - query = {"module": module_name, - "attribute": {"type": query_types[0], - "value": query_values[0]}} + query = { + "module": module_name, + "attribute": attributes[0] + } response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") def test_virustotal(self): module_name = "virustotal" - query_types = ('domain', 'ip-src', 'sha256', 'url') - query_values = ('circl.lu', '149.13.33.14', - 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - 'http://194.169.88.56:49151/.i') + attributes = ( + { + "uuid": "ffea0594-355a-42fe-9b98-fad28fd248b3", + "type": "domain", + "value": "circl.lu" + }, + { + "uuid": "1f3f0f2d-5143-4b05-a0f1-8ac82f51a979", + "type": "ip-src", + "value": "149.13.33.14" + }, + { + "uuid": "b4be6652-f4ff-4515-ae63-3f016df37e8f", + "type": "sha256", + "value": "a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3" + }, + { + "uuid": "6cead544-b683-48cb-b19b-a2561ffa1f51", + "type": "url", + "value": "http://194.169.88.56:49151/.i" + } + ) results = ('domain-ip', 'asn', 'virustotal-report', 'virustotal-report') if module_name in self.configs: - for query_type, query_value, result in zip(query_types, query_values, results): + for attribute, result in zip(attributes, results): query = {"module": module_name, - "attribute": {"type": query_type, - "value": query_value}, + "attribute": attribute, "config": self.configs[module_name]} response = self.misp_modules_post(query) try: @@ -531,9 +603,10 @@ class TestExpansions(unittest.TestCase): except Exception: self.assertEqual(self.get_errors(response), "VirusTotal request rate limit exceeded.") else: - query = {"module": module_name, - "attribute": {"type": query_types[0], - "value": query_values[0]}} + query = { + "module": module_name, + "attribute": attributes[0] + } response = self.misp_modules_post(query) self.assertEqual(self.get_errors(response), "A VirusTotal api key is required for this module.") @@ -582,6 +655,8 @@ class TestExpansions(unittest.TestCase): self.assertEqual(self.get_errors(response), "An API authentication is required (key and password).") def test_xlsx(self): + if LiveCI: + return True filename = 'test.xlsx' with open(f'{self.dirname}/test_files/{filename}', 'rb') as f: encoded = b64encode(f.read()).decode()