From 55f05e05249689ce83799ebe408bab54a61babf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Jan 2019 13:31:52 +0100 Subject: [PATCH 1/4] chg: Use pipenv, update bgpranking/ipasn modules --- .travis.yml | 19 +- Pipfile | 45 ++ Pipfile.lock | 735 ++++++++++++++++++ README.md | 2 +- REQUIREMENTS | 94 ++- doc/documentation.md | 26 +- doc/expansion/asn_history.json | 8 - doc/expansion/bgpranking.json | 8 + doc/expansion/ipasn.json | 10 +- doc/generate_documentation.py | 6 +- misp_modules/helpers/cache.py | 10 +- .../{asn_history.py => bgpranking.py} | 20 +- misp_modules/modules/expansion/ipasn.py | 19 +- setup.py | 15 +- 14 files changed, 896 insertions(+), 121 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock delete mode 100644 doc/expansion/asn_history.json create mode 100644 doc/expansion/bgpranking.json rename misp_modules/modules/expansion/{asn_history.py => bgpranking.py} (50%) diff --git a/.travis.yml b/.travis.yml index 2b59712..b574d4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,25 +11,24 @@ python: - "3.7-dev" install: - - pip install -U nose codecov pytest flake8 - - pip install -U -r REQUIREMENTS - - pip install . + - pip install pipenv + - pipenv install --dev script: - - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 & + - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 & - pid=$! - sleep 5 - - nosetests --with-coverage --cover-package=misp_modules + - pipenv run nosetests --with-coverage --cover-package=misp_modules - kill -s INT $pid - pushd ~/ - - coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & + - pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 & - pid=$! - popd - sleep 5 - - nosetests --with-coverage --cover-package=misp_modules + - pipenv run nosetests --with-coverage --cover-package=misp_modules - kill -s INT $pid - - flake8 --ignore=E501,W503 misp_modules + - pipenv run flake8 --ignore=E501,W503 misp_modules after_success: - - coverage combine .coverage* - - codecov + - pipenv run coverage combine .coverage* + - pipenv run codecov diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..c086e62 --- /dev/null +++ b/Pipfile @@ -0,0 +1,45 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +nose = "*" +codecov = "*" +pytest = "*" +flake8 = "*" + +[packages] +dnspython = "*" +requests = "*" +urlarchiver = "*" +passivetotal = "*" +pypdns = "*" +pypssl = "*" +pyeupi = "*" +uwhois = {editable = true,git = "https://github.com/Rafiot/uwhoisd.git",ref = "testing",subdirectory = "client"} +pymisp = {editable = true,git = "https://github.com/MISP/PyMISP.git"} +pyonyphe = {editable = true,git = "https://github.com/sebdraven/pyonyphe"} +pydnstrails = {editable = true,git = "https://github.com/sebdraven/pydnstrails"} +pytesseract = "*" +pygeoip = "*" +beautifulsoup4 = "*" +oauth2 = "*" +yara-python = ">=3.8.0" +sigmatools = "*" +stix2-patterns = "*" +maclookup = "*" +vulners = "*" +blockchain = "*" +pyintel471 = {editable = true,git = "https://github.com/MISP/PyIntel471.git"} +shodan = "*" +Pillow = "*" +Wand = "*" +SPARQLWrapper = "*" +domaintools_api = "*" +misp-modules = {editable = true,path = "."} +pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} +pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..02a61d1 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,735 @@ +{ + "_meta": { + "hash": { + "sha256": "f501a84bdd41ca21a2af020278ce030985cccd5f2f5683cd075797be4523587d" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "aiohttp": { + "hashes": [ + "sha256:0419705a36b43c0ac6f15469f9c2a08cad5c939d78bd12a5c23ea167c8253b2b", + "sha256:1812fc4bc6ac1bde007daa05d2d0f61199324e0cc893b11523e646595047ca08", + "sha256:2214b5c0153f45256d5d52d1e0cafe53f9905ed035a142191727a5fb620c03dd", + "sha256:275909137f0c92c61ba6bb1af856a522d5546f1de8ea01e4e726321c697754ac", + "sha256:3983611922b561868428ea1e7269e757803713f55b53502423decc509fef1650", + "sha256:51afec6ffa50a9da4cdef188971a802beb1ca8e8edb40fa429e5e529db3475fa", + "sha256:589f2ec8a101a0f340453ee6945bdfea8e1cd84c8d88e5be08716c34c0799d95", + "sha256:789820ddc65e1f5e71516adaca2e9022498fa5a837c79ba9c692a9f8f916c330", + "sha256:7a968a0bdaaf9abacc260911775611c9a602214a23aeb846f2eb2eeaa350c4dc", + "sha256:7aeefbed253f59ea39e70c5848de42ed85cb941165357fc7e87ab5d8f1f9592b", + "sha256:7b2eb55c66512405103485bd7d285a839d53e7fdc261ab20e5bcc51d7aaff5de", + "sha256:87bc95d3d333bb689c8d755b4a9d7095a2356108002149523dfc8e607d5d32a4", + "sha256:9d80e40db208e29168d3723d1440ecbb06054d349c5ece6a2c5a611490830dd7", + "sha256:a1b442195c2a77d33e4dbee67c9877ccbdd3a1f686f91eb479a9577ed8cc326b", + "sha256:ab3d769413b322d6092f169f316f7b21cd261a7589f7e31db779d5731b0480d8", + "sha256:b066d3dec5d0f5aee6e34e5765095dc3d6d78ef9839640141a2b20816a0642bd", + "sha256:b24e7845ae8de3e388ef4bcfcf7f96b05f52c8e633b33cf8003a6b1d726fc7c2", + "sha256:c59a953c3f8524a7c86eaeaef5bf702555be12f5668f6384149fe4bb75c52698", + "sha256:cf2cc6c2c10d242790412bea7ccf73726a9a44b4c4b073d2699ef3b48971fd95", + "sha256:e0c9c8d4150ae904f308ff27b35446990d2b1dfc944702a21925937e937394c6", + "sha256:f1839db4c2b08a9c8f9788112644f8a8557e8e0ecc77b07091afabb941dc55d0", + "sha256:f3df52362be39908f9c028a65490fae0475e4898b43a03d8aa29d1e765b45e07" + ], + "version": "==3.4.4" + }, + "antlr4-python3-runtime": { + "hashes": [ + "sha256:168cdcec8fb9152e84a87ca6fd261b3d54c8f6358f42ab3b813b14a7193bb50b" + ], + "markers": "python_version >= '3'", + "version": "==4.7.2" + }, + "async-timeout": { + "hashes": [ + "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", + "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" + ], + "version": "==3.0.1" + }, + "attrs": { + "hashes": [ + "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", + "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + ], + "version": "==18.2.0" + }, + "beautifulsoup4": { + "hashes": [ + "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", + "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", + "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" + ], + "index": "pypi", + "version": "==4.7.1" + }, + "blockchain": { + "hashes": [ + "sha256:dbaa3eebb6f81b4245005739da802c571b09f98d97eb66520afd95d9ccafebe2" + ], + "index": "pypi", + "version": "==1.4.4" + }, + "certifi": { + "hashes": [ + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + ], + "version": "==2018.11.29" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "click-plugins": { + "hashes": [ + "sha256:b1ee1ccc9421c73007fe290680d97984eb6eaf5f4512b7620c6aa46031d6cb6b", + "sha256:dfed74b5063546a137de99baaaf742b4de4337ad2b3e1df5ec7c8a256adc0847" + ], + "version": "==1.0.4" + }, + "colorama": { + "hashes": [ + "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", + "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" + ], + "version": "==0.4.1" + }, + "dnspython": { + "hashes": [ + "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01", + "sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d" + ], + "index": "pypi", + "version": "==1.16.0" + }, + "domaintools-api": { + "hashes": [ + "sha256:f567f407b8997e947df5badf7c2bea64fdfd33c54ade24eab36ef575fb71ccb7" + ], + "index": "pypi", + "version": "==0.3.3" + }, + "enum-compat": { + "hashes": [ + "sha256:939ceff18186a5762ae4db9fa7bfe017edbd03b66526b798dd8245394c8a4192" + ], + "version": "==0.0.2" + }, + "ez-setup": { + "hashes": [ + "sha256:303c5b17d552d1e3fb0505d80549f8579f557e13d8dc90e5ecef3c07d7f58642" + ], + "version": "==0.9" + }, + "future": { + "hashes": [ + "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" + ], + "version": "==0.17.1" + }, + "httplib2": { + "hashes": [ + "sha256:f61fb838a94ce3b349aa32c92fd8430f7e3511afdb18bf9640d647e30c90a6d6" + ], + "version": "==0.12.0" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "idna-ssl": { + "hashes": [ + "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c" + ], + "markers": "python_version < '3.7'", + "version": "==1.1.0" + }, + "isodate": { + "hashes": [ + "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", + "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" + ], + "version": "==0.6.0" + }, + "jsonschema": { + "hashes": [ + "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", + "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02" + ], + "version": "==2.6.0" + }, + "maclookup": { + "hashes": [ + "sha256:33bf8eaebe3b1e4ab4ae9277dd93c78024e0ebf6b3c42f76c37695bc26ce287a", + "sha256:795e792cd3e03c9bdad77e52904d43ff71d3ac03b360443f99d4bae08a6bffef" + ], + "index": "pypi", + "version": "==1.0.3" + }, + "misp-modules": { + "editable": true, + "path": "." + }, + "multidict": { + "hashes": [ + "sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f", + "sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3", + "sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef", + "sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b", + "sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73", + "sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc", + "sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3", + "sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd", + "sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351", + "sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941", + "sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d", + "sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1", + "sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b", + "sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a", + "sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3", + "sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7", + "sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0", + "sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0", + "sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014", + "sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5", + "sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036", + "sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d", + "sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a", + "sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce", + "sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1", + "sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a", + "sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9", + "sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7", + "sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b" + ], + "version": "==4.5.2" + }, + "oauth2": { + "hashes": [ + "sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6", + "sha256:c006a85e7c60107c7cc6da1b184b5c719f6dd7202098196dfa6e55df669b59bf" + ], + "index": "pypi", + "version": "==1.9.0.post1" + }, + "passivetotal": { + "hashes": [ + "sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d" + ], + "index": "pypi", + "version": "==1.0.30" + }, + "pillow": { + "hashes": [ + "sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e", + "sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7", + "sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a", + "sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3", + "sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1", + "sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1", + "sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7", + "sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1", + "sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3", + "sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055", + "sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf", + "sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f", + "sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f", + "sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239", + "sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe", + "sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c", + "sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697", + "sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494", + "sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356", + "sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6", + "sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000", + "sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f", + "sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c", + "sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca", + "sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8", + "sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3", + "sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad", + "sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9", + "sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc", + "sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e" + ], + "index": "pypi", + "version": "==5.4.1" + }, + "psutil": { + "hashes": [ + "sha256:1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736", + "sha256:1c71b9716790e202a00ab0931a6d1e25db1aa1198bcacaea2f5329f75d257fff", + "sha256:3b7a4daf4223dae171a67a89314ac5ca0738e94064a78d99cfd751c55d05f315", + "sha256:3e19be3441134445347af3767fa7770137d472a484070840eee6653b94ac5576", + "sha256:6e265c8f3da00b015d24b842bfeb111f856b13d24f2c57036582568dc650d6c3", + "sha256:809c9cef0402e3e48b5a1dddc390a8a6ff58b15362ea5714494073fa46c3d293", + "sha256:b4d1b735bf5b120813f4c89db8ac22d89162c558cbd7fdd298866125fe906219", + "sha256:bbffac64cfd01c6bcf90eb1bedc6c80501c4dae8aef4ad6d6dd49f8f05f6fc5a", + "sha256:bfcea4f189177b2d2ce4a34b03c4ac32c5b4c22e21f5b093d9d315e6e253cd81" + ], + "version": "==5.4.8" + }, + "pybgpranking": { + "editable": true, + "git": "https://github.com/D4-project/BGP-Ranking.git/", + "ref": "7e698f87366e6f99b4d0d11852737db28e3ddc62", + "subdirectory": "client" + }, + "pydnstrails": { + "editable": true, + "git": "https://github.com/sebdraven/pydnstrails", + "ref": "48c1f740025c51289f43a24863d1845ff12fd21a" + }, + "pyeupi": { + "hashes": [ + "sha256:35b0e6b430f23ecd303f7cc7a8fe5147cf2509a5b2254eaf9695392c0af02901" + ], + "index": "pypi", + "version": "==1.0" + }, + "pygeoip": { + "hashes": [ + "sha256:1938b9dac7b00d77f94d040b9465ea52c938f3fcdcd318b5537994f3c16aef96", + "sha256:f22c4e00ddf1213e0fae36dc60b46ee7c25a6339941ec1a975539014c1f9a96d" + ], + "index": "pypi", + "version": "==0.3.2" + }, + "pyintel471": { + "editable": true, + "git": "https://github.com/MISP/PyIntel471.git", + "ref": "0df8d51f1c1425de66714b3a5a45edb69b8cc2fc" + }, + "pyipasnhistory": { + "editable": true, + "git": "https://github.com/D4-project/IPASN-History.git/", + "ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59", + "subdirectory": "client" + }, + "pymisp": { + "editable": true, + "git": "https://github.com/MISP/PyMISP.git", + "ref": "d4934cdf5f537c9f42ae37be7878de1848961de0" + }, + "pyonyphe": { + "editable": true, + "git": "https://github.com/sebdraven/pyonyphe", + "ref": "66329baeee7cab844f2203c047c2551828eaf14d" + }, + "pyparsing": { + "hashes": [ + "sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a", + "sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3" + ], + "version": "==2.3.1" + }, + "pypdns": { + "hashes": [ + "sha256:0356360156dd26d2cf27a415a10ff2bd1ff1d2eb3b2dd51b35553d60b87fd328" + ], + "index": "pypi", + "version": "==1.3" + }, + "pypssl": { + "hashes": [ + "sha256:4dbe772aefdf4ab18934d83cde79e2fc5d5ba9d2b4153dc419a63faab3432643" + ], + "index": "pypi", + "version": "==2.1" + }, + "pytesseract": { + "hashes": [ + "sha256:11c20321595b6e2e904b594633edf1a717212b13bac7512986a2d807b8849770" + ], + "index": "pypi", + "version": "==0.2.6" + }, + "python-dateutil": { + "hashes": [ + "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", + "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + ], + "version": "==2.7.5" + }, + "pyyaml": { + "hashes": [ + "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", + "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", + "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", + "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", + "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", + "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", + "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", + "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", + "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", + "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", + "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" + ], + "version": "==3.13" + }, + "rdflib": { + "hashes": [ + "sha256:58d5994610105a457cff7fdfe3d683d87786c5028a45ae032982498a7e913d6f", + "sha256:da1df14552555c5c7715d8ce71c08f404c988c58a1ecd38552d0da4fc261280d" + ], + "version": "==4.2.2" + }, + "redis": { + "hashes": [ + "sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf", + "sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a" + ], + "version": "==3.0.1" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "index": "pypi", + "version": "==2.21.0" + }, + "requests-cache": { + "hashes": [ + "sha256:e9270030becc739b0a7f7f834234c73a878b2d794122bf76f40055a22419eb67", + "sha256:fe561ca119879bbcfb51f03a35e35b425e18f338248e59fd5cf2166c77f457a2" + ], + "version": "==0.4.13" + }, + "shodan": { + "hashes": [ + "sha256:c40abb6ff2fd66bdee9f773746fb961eefdfaa8e720a07cb12fb70def136268d" + ], + "index": "pypi", + "version": "==1.10.4" + }, + "sigmatools": { + "hashes": [ + "sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b" + ], + "index": "pypi", + "version": "==0.7.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "soupsieve": { + "hashes": [ + "sha256:10687fc53eeb3518e01a0ac84d3d711da623d3298a3039459d3f649927c4a270", + "sha256:b23a0d7da0247200fe83c67c34de9d7599ad404106367313d8e65e04174d0b4b" + ], + "version": "==1.7.2" + }, + "sparqlwrapper": { + "hashes": [ + "sha256:2a95fdede2833be660b81092934c4a0054ff85f2693098556762a2759ea486f1", + "sha256:7f4c8d38ea1bfcffbc358c9a05de35a3fd7152cc3e8ea57963ee7a0a242f7a5e", + "sha256:acf6d60f0a3684cb673653b07871acb0c350a974b891f20f8ac94926ff9eb2ff" + ], + "index": "pypi", + "version": "==1.8.2" + }, + "stix2-patterns": { + "hashes": [ + "sha256:137cbe28d29af774d526a49d60b3a40af7c19fe1e5f252e741bb25f253d5616f" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "tornado": { + "hashes": [ + "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", + "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", + "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", + "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", + "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", + "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", + "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" + ], + "version": "==5.1.1" + }, + "url-normalize": { + "hashes": [ + "sha256:3468d64cb22a9092a2c086e46c781f741dc9a1689b24e9b48ab5e8244ffa6c02", + "sha256:51e0f14050c79e732d175c33d12167f5e642cc23e0cb23275236af843faf884f" + ], + "version": "==1.4.1" + }, + "urlarchiver": { + "hashes": [ + "sha256:652e0890dab58bf62a759656671dcfb9a40eb4a77aac8a8d93154f00360238b5" + ], + "index": "pypi", + "version": "==0.2" + }, + "urllib3": { + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + }, + "uwhois": { + "editable": true, + "git": "https://github.com/Rafiot/uwhoisd.git", + "ref": "f6f035e52213c8abc20f2084d28cfffb399457cb", + "subdirectory": "client" + }, + "vulners": { + "hashes": [ + "sha256:8b468db8f8b0bad39ae51ebd4247f6ead90b6f53699e03b91ff9d63da70554d7", + "sha256:ad72378c842096cad9ebf83aa53d330117ece5d208ed7c419a21c70a8d5e2236", + "sha256:ffc92a099eeddea840fd199665992c0eb6d7ad69ac3a6730a286d00600bc5f2c" + ], + "index": "pypi", + "version": "==1.3.6" + }, + "wand": { + "hashes": [ + "sha256:3e59e4bda9ef9d643d90e881cc950c8eee1508ec2cde1c150a1cbd5a12c1c007", + "sha256:52763dbf65d00cf98d7bc910b49329eea15896249c5555d47e169f2b6efbe166" + ], + "index": "pypi", + "version": "==0.5.0" + }, + "xlsxwriter": { + "hashes": [ + "sha256:7cc07619760641b67112dbe0df938399d4d915d9b9924bb58eb5c17384d29cc6", + "sha256:ae22658a0fc5b9e875fa97c213d1ffd617d86dc49bf08be99ebdac814db7bf36" + ], + "version": "==1.1.2" + }, + "yara-python": { + "hashes": [ + "sha256:03e5c5e333c8572e7994b0b11964d515d61a393f23c5e272f8d0e4229f368c58", + "sha256:0423e08bd618752a028ac0405ff8e0103f3a8fd607dde7618a64a4c010c3757b", + "sha256:0a0dd632dcdb347d1a9a8b1f6a83b3a77d5e63f691357ea4021fb1cf1d7ff0a4", + "sha256:728b99627a8072a877eaaa4dafb4eff39d1b14ff4fd70d39f18899ce81e29625", + "sha256:7cb0d5724eccfa52e1bcd352a56cb4dc422aa51f5f6d0945d4f830783927513b", + "sha256:8c76531e89806c0309586dd4863a972d12f1d5d63261c6d4b9331a99859fd1d8", + "sha256:9472676583e212bc4e17c2236634e02273d53c872b350f0571b48e06183de233", + "sha256:9735b680a7d95c1d3f255c351bb067edc62cdb3c0999f7064278cb2c85245405", + "sha256:997f104590167220a9af5564c042ec4d6534261e7b8a5b49655d8dffecc6b8a2", + "sha256:a48e071d02a3699363e628ac899b5b7237803bcb4b512c92ebcb4fb9b1488497", + "sha256:b67c0d75a6519ca357b4b85ede9768c96a81fff20fbc169bd805ff009ddee561" + ], + "index": "pypi", + "version": "==3.8.1" + }, + "yarl": { + "hashes": [ + "sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9", + "sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f", + "sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb", + "sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320", + "sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842", + "sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0", + "sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829", + "sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310", + "sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4", + "sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8", + "sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1" + ], + "version": "==1.3.0" + } + }, + "develop": { + "atomicwrites": { + "hashes": [ + "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", + "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + ], + "version": "==1.2.1" + }, + "attrs": { + "hashes": [ + "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", + "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + ], + "version": "==18.2.0" + }, + "certifi": { + "hashes": [ + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + ], + "version": "==2018.11.29" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "codecov": { + "hashes": [ + "sha256:8ed8b7c6791010d359baed66f84f061bba5bd41174bf324c31311e8737602788", + "sha256:ae00d68e18d8a20e9c3288ba3875ae03db3a8e892115bf9b83ef20507732bed4" + ], + "index": "pypi", + "version": "==2.0.15" + }, + "coverage": { + "hashes": [ + "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", + "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", + "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", + "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", + "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", + "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", + "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", + "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", + "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", + "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", + "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", + "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", + "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", + "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", + "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", + "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", + "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", + "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", + "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", + "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", + "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", + "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", + "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", + "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", + "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", + "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", + "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", + "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", + "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", + "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", + "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + ], + "version": "==4.5.2" + }, + "flake8": { + "hashes": [ + "sha256:6a35f5b8761f45c5513e3405f110a86bea57982c3b75b766ce7b65217abe1670", + "sha256:c01f8a3963b3571a8e6bd7a4063359aff90749e160778e03817cd9b71c9e07d2" + ], + "index": "pypi", + "version": "==3.6.0" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "more-itertools": { + "hashes": [ + "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", + "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", + "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9" + ], + "version": "==5.0.0" + }, + "nose": { + "hashes": [ + "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", + "sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a", + "sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98" + ], + "index": "pypi", + "version": "==1.3.7" + }, + "pluggy": { + "hashes": [ + "sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616", + "sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a" + ], + "version": "==0.8.1" + }, + "py": { + "hashes": [ + "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", + "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + ], + "version": "==1.7.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", + "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" + ], + "version": "==2.4.0" + }, + "pyflakes": { + "hashes": [ + "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", + "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" + ], + "version": "==2.0.0" + }, + "pytest": { + "hashes": [ + "sha256:41568ea7ecb4a68d7f63837cf65b92ce8d0105e43196ff2b26622995bb3dc4b2", + "sha256:c3c573a29d7c9547fb90217ece8a8843aa0c1328a797e200290dc3d0b4b823be" + ], + "index": "pypi", + "version": "==4.1.1" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "index": "pypi", + "version": "==2.21.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "urllib3": { + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + } + } +} diff --git a/README.md b/README.md index 7a93d88..31efe73 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ### Expansion modules -* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history. +* [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking. * [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP. * [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information. * [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen. diff --git a/REQUIREMENTS b/REQUIREMENTS index 29387f1..709620a 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -1,31 +1,63 @@ -tornado -dnspython -requests -urlarchiver -passivetotal -PyPDNS -pypssl -redis -pyeupi -ipasn-redis -asnhistory -git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client -git+https://github.com/MISP/PyMISP.git#egg=pymisp -git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe -git+https://github.com/sebdraven/pydnstrails#egg=pydnstrails -pillow -pytesseract -wand -SPARQLWrapper -domaintools_api -pygeoip -bs4 -oauth2 -yara-python==3.8.0 -sigmatools -stix2-patterns -maclookup -vulners -psutil -blockchain -git+https://github.com/MISP/PyIntel471.git +-i https://pypi.org/simple +-e . +-e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client +-e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client +-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 +-e git+https://github.com/MISP/PyMISP.git@d4934cdf5f537c9f42ae37be7878de1848961de0#egg=pymisp +-e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client +-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails +-e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe +aiohttp==3.4.4 +antlr4-python3-runtime==4.7.2 ; python_version >= '3' +async-timeout==3.0.1 +attrs==18.2.0 +beautifulsoup4==4.7.1 +blockchain==1.4.4 +certifi==2018.11.29 +chardet==3.0.4 +click-plugins==1.0.4 +click==7.0 +colorama==0.4.1 +dnspython==1.16.0 +domaintools-api==0.3.3 +enum-compat==0.0.2 +ez-setup==0.9 +future==0.17.1 +httplib2==0.12.0 +idna-ssl==1.1.0 ; python_version < '3.7' +idna==2.8 +isodate==0.6.0 +jsonschema==2.6.0 +maclookup==1.0.3 +multidict==4.5.2 +oauth2==1.9.0.post1 +passivetotal==1.0.30 +pillow==5.4.1 +psutil==5.4.8 +pyeupi==1.0 +pygeoip==0.3.2 +pyparsing==2.3.1 +pypdns==1.3 +pypssl==2.1 +pytesseract==0.2.6 +python-dateutil==2.7.5 +pyyaml==3.13 +rdflib==4.2.2 +redis==3.0.1 +requests-cache==0.4.13 +requests==2.21.0 +shodan==1.10.4 +sigmatools==0.7.1 +six==1.12.0 +soupsieve==1.7.2 +sparqlwrapper==1.8.2 +stix2-patterns==1.1.0 +tornado==5.1.1 +url-normalize==1.4.1 +urlarchiver==0.2 +urllib3==1.24.1 +vulners==1.3.6 +wand==0.5.0 +xlsxwriter==1.1.2 +yara-python==3.8.1 +yarl==1.3.0 diff --git a/doc/documentation.md b/doc/documentation.md index 49fb6b5..df5ce6e 100644 --- a/doc/documentation.md +++ b/doc/documentation.md @@ -2,25 +2,21 @@ ## Expansion Modules -#### [asn_history](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/asn_history.py) +#### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py) -Query an ASN description history service (https://github.com/CIRCL/ASN-Description-History.git). +Query BGP Ranking (https://bgpranking-ng.circl.lu/). - **features**: ->The module takes an AS number attribute as input and displays its description and history. +>The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking. > ->For a proper working, a communication with a redis database is needed, thus 3 parameters are needed: ->- host, the address of the redis server ->- port, the port used by redis ->- db, the index of the database used > - **input**: >Autonomous system number. - **output**: ->Text containing a description of the ASN and its history. +>Text containing a description of the ASN, its history, and the position in BGP Ranking. - **references**: ->https://github.com/CIRCL/ASN-Description-History.git +>https://github.com/D4-project/BGP-Ranking/ - **requirements**: ->asnhistory python library +>pybgpranking python library ----- @@ -331,17 +327,17 @@ Module to access intelmqs eventdb. #### [ipasn](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py) -Module to query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git). +Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History). - **features**: ->This module takes an IP address attribute as input and queries the CIRCL IP ASN service to get additional information about the input. +>This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input. - **input**: >An IP address MISP attribute. - **output**: ->Text describing additional information about the input after a query on the IP-ASN-history database. +>Text describing additional information about the input after a query on the IPASN-history database. - **references**: ->https://www.circl.lu/services/ip-asn-history/ +>https://github.com/D4-project/IPASN-History - **requirements**: ->ipasn_redis: Python library to access IP-ASN-history instance via redis, An IP-ASN-history instance information (host, port and database index) +>pyipasnhistory: Python library to access IPASN-history instance ----- diff --git a/doc/expansion/asn_history.json b/doc/expansion/asn_history.json deleted file mode 100644 index b3eea26..0000000 --- a/doc/expansion/asn_history.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "description": "Query an ASN description history service (https://github.com/CIRCL/ASN-Description-History.git).", - "requirements": ["asnhistory python library"], - "features": "The module takes an AS number attribute as input and displays its description and history.\n\nFor a proper working, a communication with a redis database is needed, thus 3 parameters are needed:\n- host, the address of the redis server\n- port, the port used by redis\n- db, the index of the database used\n", - "references": ["https://github.com/CIRCL/ASN-Description-History.git"], - "input": "Autonomous system number.", - "output": "Text containing a description of the ASN and its history." -} diff --git a/doc/expansion/bgpranking.json b/doc/expansion/bgpranking.json new file mode 100644 index 0000000..a98b780 --- /dev/null +++ b/doc/expansion/bgpranking.json @@ -0,0 +1,8 @@ +{ + "description": "Query BGP Ranking (https://bgpranking-ng.circl.lu/).", + "requirements": ["pybgpranking python library"], + "features": "The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking.\n\n", + "references": ["https://github.com/D4-project/BGP-Ranking/"], + "input": "Autonomous system number.", + "output": "Text containing a description of the ASN, its history, and the position in BGP Ranking." +} diff --git a/doc/expansion/ipasn.json b/doc/expansion/ipasn.json index aa9d0b1..68b10d1 100644 --- a/doc/expansion/ipasn.json +++ b/doc/expansion/ipasn.json @@ -1,8 +1,8 @@ { - "description": "Module to query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git).", - "requirements": ["ipasn_redis: Python library to access IP-ASN-history instance via redis", "An IP-ASN-history instance information (host, port and database index)"], + "description": "Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).", + "requirements": ["pyipasnhistory: Python library to access IPASN-history instance"], "input": "An IP address MISP attribute.", - "output": "Text describing additional information about the input after a query on the IP-ASN-history database.", - "references": ["https://www.circl.lu/services/ip-asn-history/"], - "features": "This module takes an IP address attribute as input and queries the CIRCL IP ASN service to get additional information about the input." + "output": "Text describing additional information about the input after a query on the IPASN-history database.", + "references": ["https://github.com/D4-project/IPASN-History"], + "features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input." } diff --git a/doc/generate_documentation.py b/doc/generate_documentation.py index 6be61de..980ddf6 100644 --- a/doc/generate_documentation.py +++ b/doc/generate_documentation.py @@ -4,9 +4,10 @@ import json module_types = ['expansion', 'export_mod', 'import_mod'] titles = ['Expansion Modules', 'Export Modules', 'Import Modules'] -markdown= ["# MISP modules documentation\n"] +markdown = ["# MISP modules documentation\n"] githublink = 'https://github.com/MISP/misp-modules/tree/master/misp_modules/modules' + def generate_doc(root_path): for _path, title in zip(module_types, titles): markdown.append('\n## {}\n'.format(title)) @@ -18,7 +19,7 @@ def generate_doc(root_path): githubref = '{}/{}.py'.format(githubpath, modulename) markdown.append('\n#### [{}]({})\n'.format(modulename, githubref)) filename = os.path.join(current_path, _file) - with open(filename, 'rt', encoding='utf-8') as f: + with open(filename, 'rt') as f: definition = json.loads(f.read()) if 'logo' in definition: markdown.append('\n\n'.format(definition.pop('logo'))) @@ -32,6 +33,7 @@ def generate_doc(root_path): with open('documentation.md', 'w') as w: w.write(''.join(markdown)) + if __name__ == '__main__': root_path = os.path.dirname(os.path.realpath(__file__)) generate_doc(root_path) diff --git a/misp_modules/helpers/cache.py b/misp_modules/helpers/cache.py index 77506fa..93a1c88 100644 --- a/misp_modules/helpers/cache.py +++ b/misp_modules/helpers/cache.py @@ -30,7 +30,7 @@ db = 5 def selftest(enable=True): if not enable: return False - r = redis.StrictRedis(host=hostname, port=port, db=db) + r = redis.Redis(host=hostname, port=port, db=db) try: r.ping() except Exception: @@ -40,11 +40,11 @@ def selftest(enable=True): def get(modulename=None, query=None, value=None, debug=False): if (modulename is None or query is None): return False - r = redis.StrictRedis(host=hostname, port=port, db=db) + r = redis.Redis(host=hostname, port=port, db=db, decode_responses=True) h = hashlib.sha1() h.update(query.encode('UTF-8')) hv = h.hexdigest() - key = "m:" + modulename + ":" + hv + key = "m:{}:{}".format(modulename, hv) if not r.exists(key): if debug: @@ -58,7 +58,7 @@ def get(modulename=None, query=None, value=None, debug=False): def flush(): - r = redis.StrictRedis(host=hostname, port=port, db=db) + r = redis.StrictRedis(host=hostname, port=port, db=db, decode_responses=True) returncode = r.flushdb() return returncode @@ -70,7 +70,7 @@ if __name__ == "__main__": else: print("Selftest ok") v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True) - if v == b'barfoo': + if v == 'barfoo': print("Cache ok") v = get(modulename="testmodule", query="abcdef") print(v) diff --git a/misp_modules/modules/expansion/asn_history.py b/misp_modules/modules/expansion/bgpranking.py similarity index 50% rename from misp_modules/modules/expansion/asn_history.py rename to misp_modules/modules/expansion/bgpranking.py index 5a2f53d..b01088d 100755 --- a/misp_modules/modules/expansion/asn_history.py +++ b/misp_modules/modules/expansion/bgpranking.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import json -from asnhistory import ASNHistory +from datetime import date, timedelta +from pybgpranking import BGPRanking misperrors = {'error': 'Error'} mispattributes = {'input': ['AS'], 'output': ['freetext']} @@ -9,8 +10,6 @@ moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', 'description': 'Query an ASN Description history service (https://github.com/CIRCL/ASN-Description-History.git)', 'module-type': ['expansion', 'hover']} -moduleconfig = ['host', 'port', 'db'] - def handler(q=False): if q is False: @@ -22,19 +21,11 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('host') - and request['config'].get('port') - and request['config'].get('db')): - misperrors['error'] = 'ASN description history configuration is missing' - return misperrors - - asnhistory = ASNHistory(host=request['config'].get('host'), - port=request['config'].get('port'), db=request['config'].get('db')) - - values = ['{} {}'.format(date.isoformat(), description) for date, description in asnhistory.get_all_descriptions(toquery)] + bgpranking = BGPRanking() + values = bgpranking.query(toquery, date=(date.today() - timedelta(1)).isoformat()) if not values: - misperrors['error'] = 'Unable to find descriptions for this ASN' + misperrors['error'] = 'Unable to find the ASN in BGP Ranking' return misperrors return {'results': [{'types': mispattributes['output'], 'values': values}]} @@ -44,5 +35,4 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/misp_modules/modules/expansion/ipasn.py b/misp_modules/modules/expansion/ipasn.py index f47d780..8489aa0 100755 --- a/misp_modules/modules/expansion/ipasn.py +++ b/misp_modules/modules/expansion/ipasn.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import json -from ipasn_redis import IPASN +from pyipasnhistory import IPASNHistory misperrors = {'error': 'Error'} mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']} @@ -9,8 +9,6 @@ moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot', 'description': 'Query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git)', 'module-type': ['expansion', 'hover']} -moduleconfig = ['host', 'port', 'db'] - def handler(q=False): if q is False: @@ -24,18 +22,8 @@ def handler(q=False): misperrors['error'] = "Unsupported attributes type" return misperrors - if not request.get('config') and not (request['config'].get('host') - and request['config'].get('port') - and request['config'].get('db')): - misperrors['error'] = 'IP ASN history configuration is missing' - return misperrors - - ipasn = IPASN(host=request['config'].get('host'), - port=request['config'].get('port'), db=request['config'].get('db')) - - values = [] - for first_seen, last_seen, asn, block in ipasn.aggregate_history(toquery): - values.append('{} {} {} {}'.format(first_seen.decode(), last_seen.decode(), asn.decode(), block)) + ipasn = IPASNHistory() + values = ipasn.query(toquery) if not values: misperrors['error'] = 'Unable to find the history of this IP' @@ -48,5 +36,4 @@ def introspection(): def version(): - moduleinfo['config'] = moduleconfig return moduleinfo diff --git a/setup.py b/setup.py index f6c3a64..fc78750 100644 --- a/setup.py +++ b/setup.py @@ -23,18 +23,7 @@ setup( ], install_requires=[ 'tornado', - 'dnspython3', - 'requests', - 'urlarchiver', - 'passivetotal', - 'PyPDNS', - 'pypssl', - 'redis', - 'pyeupi', - 'ipasn-redis', - 'asnhistory', - 'pillow', - 'pytesseract', - 'shodan', + 'psutil', + 'redis>=3' ], ) From d5ec09fe4ad4209c387b1b0da82a412ea83f7658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Jan 2019 13:57:45 +0100 Subject: [PATCH 2/4] fix: Change module name --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 18fd78c..559e5aa 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -1,6 +1,6 @@ from . import _vmray # noqa -__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', +__all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', From b791b177c3a3c0f69c6e2c606989aa24c0203c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Jan 2019 14:06:38 +0100 Subject: [PATCH 3/4] fix: Change in the imports --- misp_modules/modules/expansion/sigma_syntax_validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/expansion/sigma_syntax_validator.py b/misp_modules/modules/expansion/sigma_syntax_validator.py index b5fc0cf..658b4d3 100644 --- a/misp_modules/modules/expansion/sigma_syntax_validator.py +++ b/misp_modules/modules/expansion/sigma_syntax_validator.py @@ -1,8 +1,8 @@ import json try: import yaml - from sigma.parser import SigmaParser - from sigma.config import SigmaConfiguration + from sigma.parser.rule import SigmaParser + from sigma.configuration import SigmaConfiguration except ImportError: print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.") From 0189a117a3c2839e73173e69f1e0659df1c4bd02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 21 Jan 2019 14:14:19 +0100 Subject: [PATCH 4/4] fix: Change in the imports in other sigma module --- misp_modules/modules/expansion/sigma_queries.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/sigma_queries.py b/misp_modules/modules/expansion/sigma_queries.py index 37a8c14..7799f2a 100644 --- a/misp_modules/modules/expansion/sigma_queries.py +++ b/misp_modules/modules/expansion/sigma_queries.py @@ -2,9 +2,10 @@ import sys import io import json try: - from sigma.parser import SigmaCollectionParser - from sigma.config import SigmaConfiguration - from sigma.backends import getBackend, BackendOptions + from sigma.parser.collection import SigmaCollectionParser + from sigma.configuration import SigmaConfiguration + from sigma.backends.base import BackendOptions + from sigma.backends.discovery import getBackend except ImportError: print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.")