MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #266 from MISP/pipenv 

chg: Use pipenv, update bgpranking/ipasn modules, fix imports for sigma
pull/285/head
Raphaël Vinot 2019-01-21 14:18:26 +01:00 committed by GitHub
parent e6f4503763 0189a117a3
commit aa0581d3ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 903 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.travis.yml
View File

@ -11,25 +11,24 @@ python:
- "3.7-dev"
install:
- pip install -U nose codecov pytest flake8
- pip install -U -r REQUIREMENTS
- pip install .
- pip install pipenv
- pipenv install --dev
script:
- coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 &
- pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 &
- pid=$!
- sleep 5
- nosetests --with-coverage --cover-package=misp_modules
- pipenv run nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid
- pushd ~/
- coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
- pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
- pid=$!
- popd
- sleep 5
- nosetests --with-coverage --cover-package=misp_modules
- pipenv run nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid
- flake8 --ignore=E501,W503 misp_modules
- pipenv run flake8 --ignore=E501,W503 misp_modules
after_success:
- coverage combine .coverage*
- codecov
- pipenv run coverage combine .coverage*
- pipenv run codecov

45
Pipfile Normal file
View File

@ -0,0 +1,45 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
nose = "*"
codecov = "*"
pytest = "*"
flake8 = "*"
[packages]
dnspython = "*"
requests = "*"
urlarchiver = "*"
passivetotal = "*"
pypdns = "*"
pypssl = "*"
pyeupi = "*"
uwhois = {editable = true,git = "https://github.com/Rafiot/uwhoisd.git",ref = "testing",subdirectory = "client"}
pymisp = {editable = true,git = "https://github.com/MISP/PyMISP.git"}
pyonyphe = {editable = true,git = "https://github.com/sebdraven/pyonyphe"}
pydnstrails = {editable = true,git = "https://github.com/sebdraven/pydnstrails"}
pytesseract = "*"
pygeoip = "*"
beautifulsoup4 = "*"
oauth2 = "*"
yara-python = ">=3.8.0"
sigmatools = "*"
stix2-patterns = "*"
maclookup = "*"
vulners = "*"
blockchain = "*"
pyintel471 = {editable = true,git = "https://github.com/MISP/PyIntel471.git"}
shodan = "*"
Pillow = "*"
Wand = "*"
SPARQLWrapper = "*"
domaintools_api = "*"
misp-modules = {editable = true,path = "."}
pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"}
pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"}
[requires]
python_version = "3.6"

735
Pipfile.lock generated Normal file
View File

@ -0,0 +1,735 @@
{
"_meta": {
"hash": {
"sha256": "f501a84bdd41ca21a2af020278ce030985cccd5f2f5683cd075797be4523587d"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.6"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"aiohttp": {
"hashes": [
"sha256:0419705a36b43c0ac6f15469f9c2a08cad5c939d78bd12a5c23ea167c8253b2b",
"sha256:1812fc4bc6ac1bde007daa05d2d0f61199324e0cc893b11523e646595047ca08",
"sha256:2214b5c0153f45256d5d52d1e0cafe53f9905ed035a142191727a5fb620c03dd",
"sha256:275909137f0c92c61ba6bb1af856a522d5546f1de8ea01e4e726321c697754ac",
"sha256:3983611922b561868428ea1e7269e757803713f55b53502423decc509fef1650",
"sha256:51afec6ffa50a9da4cdef188971a802beb1ca8e8edb40fa429e5e529db3475fa",
"sha256:589f2ec8a101a0f340453ee6945bdfea8e1cd84c8d88e5be08716c34c0799d95",
"sha256:789820ddc65e1f5e71516adaca2e9022498fa5a837c79ba9c692a9f8f916c330",
"sha256:7a968a0bdaaf9abacc260911775611c9a602214a23aeb846f2eb2eeaa350c4dc",
"sha256:7aeefbed253f59ea39e70c5848de42ed85cb941165357fc7e87ab5d8f1f9592b",
"sha256:7b2eb55c66512405103485bd7d285a839d53e7fdc261ab20e5bcc51d7aaff5de",
"sha256:87bc95d3d333bb689c8d755b4a9d7095a2356108002149523dfc8e607d5d32a4",
"sha256:9d80e40db208e29168d3723d1440ecbb06054d349c5ece6a2c5a611490830dd7",
"sha256:a1b442195c2a77d33e4dbee67c9877ccbdd3a1f686f91eb479a9577ed8cc326b",
"sha256:ab3d769413b322d6092f169f316f7b21cd261a7589f7e31db779d5731b0480d8",
"sha256:b066d3dec5d0f5aee6e34e5765095dc3d6d78ef9839640141a2b20816a0642bd",
"sha256:b24e7845ae8de3e388ef4bcfcf7f96b05f52c8e633b33cf8003a6b1d726fc7c2",
"sha256:c59a953c3f8524a7c86eaeaef5bf702555be12f5668f6384149fe4bb75c52698",
"sha256:cf2cc6c2c10d242790412bea7ccf73726a9a44b4c4b073d2699ef3b48971fd95",
"sha256:e0c9c8d4150ae904f308ff27b35446990d2b1dfc944702a21925937e937394c6",
"sha256:f1839db4c2b08a9c8f9788112644f8a8557e8e0ecc77b07091afabb941dc55d0",
"sha256:f3df52362be39908f9c028a65490fae0475e4898b43a03d8aa29d1e765b45e07"
],
"version": "==3.4.4"
},
"antlr4-python3-runtime": {
"hashes": [
"sha256:168cdcec8fb9152e84a87ca6fd261b3d54c8f6358f42ab3b813b14a7193bb50b"
],
"markers": "python_version >= '3'",
"version": "==4.7.2"
},
"async-timeout": {
"hashes": [
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
"sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
],
"version": "==3.0.1"
},
"attrs": {
"hashes": [
"sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
"sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
],
"version": "==18.2.0"
},
"beautifulsoup4": {
"hashes": [
"sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858",
"sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348",
"sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718"
],
"index": "pypi",
"version": "==4.7.1"
},
"blockchain": {
"hashes": [
"sha256:dbaa3eebb6f81b4245005739da802c571b09f98d97eb66520afd95d9ccafebe2"
],
"index": "pypi",
"version": "==1.4.4"
},
"certifi": {
"hashes": [
"sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
"sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
],
"version": "==2018.11.29"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"click": {
"hashes": [
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
"sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
],
"version": "==7.0"
},
"click-plugins": {
"hashes": [
"sha256:b1ee1ccc9421c73007fe290680d97984eb6eaf5f4512b7620c6aa46031d6cb6b",
"sha256:dfed74b5063546a137de99baaaf742b4de4337ad2b3e1df5ec7c8a256adc0847"
],
"version": "==1.0.4"
},
"colorama": {
"hashes": [
"sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d",
"sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"
],
"version": "==0.4.1"
},
"dnspython": {
"hashes": [
"sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
"sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d"
],
"index": "pypi",
"version": "==1.16.0"
},
"domaintools-api": {
"hashes": [
"sha256:f567f407b8997e947df5badf7c2bea64fdfd33c54ade24eab36ef575fb71ccb7"
],
"index": "pypi",
"version": "==0.3.3"
},
"enum-compat": {
"hashes": [
"sha256:939ceff18186a5762ae4db9fa7bfe017edbd03b66526b798dd8245394c8a4192"
],
"version": "==0.0.2"
},
"ez-setup": {
"hashes": [
"sha256:303c5b17d552d1e3fb0505d80549f8579f557e13d8dc90e5ecef3c07d7f58642"
],
"version": "==0.9"
},
"future": {
"hashes": [
"sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8"
],
"version": "==0.17.1"
},
"httplib2": {
"hashes": [
"sha256:f61fb838a94ce3b349aa32c92fd8430f7e3511afdb18bf9640d647e30c90a6d6"
],
"version": "==0.12.0"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
],
"version": "==2.8"
},
"idna-ssl": {
"hashes": [
"sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c"
],
"markers": "python_version < '3.7'",
"version": "==1.1.0"
},
"isodate": {
"hashes": [
"sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8",
"sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81"
],
"version": "==0.6.0"
},
"jsonschema": {
"hashes": [
"sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08",
"sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02"
],
"version": "==2.6.0"
},
"maclookup": {
"hashes": [
"sha256:33bf8eaebe3b1e4ab4ae9277dd93c78024e0ebf6b3c42f76c37695bc26ce287a",
"sha256:795e792cd3e03c9bdad77e52904d43ff71d3ac03b360443f99d4bae08a6bffef"
],
"index": "pypi",
"version": "==1.0.3"
},
"misp-modules": {
"editable": true,
"path": "."
},
"multidict": {
"hashes": [
"sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f",
"sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3",
"sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef",
"sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b",
"sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73",
"sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc",
"sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3",
"sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd",
"sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351",
"sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941",
"sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d",
"sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1",
"sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b",
"sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a",
"sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3",
"sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7",
"sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0",
"sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0",
"sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014",
"sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5",
"sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036",
"sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d",
"sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a",
"sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce",
"sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1",
"sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a",
"sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9",
"sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7",
"sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b"
],
"version": "==4.5.2"
},
"oauth2": {
"hashes": [
"sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6",
"sha256:c006a85e7c60107c7cc6da1b184b5c719f6dd7202098196dfa6e55df669b59bf"
],
"index": "pypi",
"version": "==1.9.0.post1"
},
"passivetotal": {
"hashes": [
"sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d"
],
"index": "pypi",
"version": "==1.0.30"
},
"pillow": {
"hashes": [
"sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e",
"sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7",
"sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a",
"sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3",
"sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1",
"sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1",
"sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7",
"sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1",
"sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3",
"sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055",
"sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf",
"sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f",
"sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f",
"sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239",
"sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe",
"sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c",
"sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697",
"sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494",
"sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356",
"sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6",
"sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000",
"sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f",
"sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c",
"sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca",
"sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8",
"sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3",
"sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad",
"sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9",
"sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc",
"sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e"
],
"index": "pypi",
"version": "==5.4.1"
},
"psutil": {
"hashes": [
"sha256:1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736",
"sha256:1c71b9716790e202a00ab0931a6d1e25db1aa1198bcacaea2f5329f75d257fff",
"sha256:3b7a4daf4223dae171a67a89314ac5ca0738e94064a78d99cfd751c55d05f315",
"sha256:3e19be3441134445347af3767fa7770137d472a484070840eee6653b94ac5576",
"sha256:6e265c8f3da00b015d24b842bfeb111f856b13d24f2c57036582568dc650d6c3",
"sha256:809c9cef0402e3e48b5a1dddc390a8a6ff58b15362ea5714494073fa46c3d293",
"sha256:b4d1b735bf5b120813f4c89db8ac22d89162c558cbd7fdd298866125fe906219",
"sha256:bbffac64cfd01c6bcf90eb1bedc6c80501c4dae8aef4ad6d6dd49f8f05f6fc5a",
"sha256:bfcea4f189177b2d2ce4a34b03c4ac32c5b4c22e21f5b093d9d315e6e253cd81"
],
"version": "==5.4.8"
},
"pybgpranking": {
"editable": true,
"git": "https://github.com/D4-project/BGP-Ranking.git/",
"ref": "7e698f87366e6f99b4d0d11852737db28e3ddc62",
"subdirectory": "client"
},
"pydnstrails": {
"editable": true,
"git": "https://github.com/sebdraven/pydnstrails",
"ref": "48c1f740025c51289f43a24863d1845ff12fd21a"
},
"pyeupi": {
"hashes": [
"sha256:35b0e6b430f23ecd303f7cc7a8fe5147cf2509a5b2254eaf9695392c0af02901"
],
"index": "pypi",
"version": "==1.0"
},
"pygeoip": {
"hashes": [
"sha256:1938b9dac7b00d77f94d040b9465ea52c938f3fcdcd318b5537994f3c16aef96",
"sha256:f22c4e00ddf1213e0fae36dc60b46ee7c25a6339941ec1a975539014c1f9a96d"
],
"index": "pypi",
"version": "==0.3.2"
},
"pyintel471": {
"editable": true,
"git": "https://github.com/MISP/PyIntel471.git",
"ref": "0df8d51f1c1425de66714b3a5a45edb69b8cc2fc"
},
"pyipasnhistory": {
"editable": true,
"git": "https://github.com/D4-project/IPASN-History.git/",
"ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59",
"subdirectory": "client"
},
"pymisp": {
"editable": true,
"git": "https://github.com/MISP/PyMISP.git",
"ref": "d4934cdf5f537c9f42ae37be7878de1848961de0"
},
"pyonyphe": {
"editable": true,
"git": "https://github.com/sebdraven/pyonyphe",
"ref": "66329baeee7cab844f2203c047c2551828eaf14d"
},
"pyparsing": {
"hashes": [
"sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a",
"sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3"
],
"version": "==2.3.1"
},
"pypdns": {
"hashes": [
"sha256:0356360156dd26d2cf27a415a10ff2bd1ff1d2eb3b2dd51b35553d60b87fd328"
],
"index": "pypi",
"version": "==1.3"
},
"pypssl": {
"hashes": [
"sha256:4dbe772aefdf4ab18934d83cde79e2fc5d5ba9d2b4153dc419a63faab3432643"
],
"index": "pypi",
"version": "==2.1"
},
"pytesseract": {
"hashes": [
"sha256:11c20321595b6e2e904b594633edf1a717212b13bac7512986a2d807b8849770"
],
"index": "pypi",
"version": "==0.2.6"
},
"python-dateutil": {
"hashes": [
"sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
"sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
],
"version": "==2.7.5"
},
"pyyaml": {
"hashes": [
"sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b",
"sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf",
"sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a",
"sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3",
"sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1",
"sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1",
"sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613",
"sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04",
"sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f",
"sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537",
"sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531"
],
"version": "==3.13"
},
"rdflib": {
"hashes": [
"sha256:58d5994610105a457cff7fdfe3d683d87786c5028a45ae032982498a7e913d6f",
"sha256:da1df14552555c5c7715d8ce71c08f404c988c58a1ecd38552d0da4fc261280d"
],
"version": "==4.2.2"
},
"redis": {
"hashes": [
"sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf",
"sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a"
],
"version": "==3.0.1"
},
"requests": {
"hashes": [
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
],
"index": "pypi",
"version": "==2.21.0"
},
"requests-cache": {
"hashes": [
"sha256:e9270030becc739b0a7f7f834234c73a878b2d794122bf76f40055a22419eb67",
"sha256:fe561ca119879bbcfb51f03a35e35b425e18f338248e59fd5cf2166c77f457a2"
],
"version": "==0.4.13"
},
"shodan": {
"hashes": [
"sha256:c40abb6ff2fd66bdee9f773746fb961eefdfaa8e720a07cb12fb70def136268d"
],
"index": "pypi",
"version": "==1.10.4"
},
"sigmatools": {
"hashes": [
"sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b"
],
"index": "pypi",
"version": "==0.7.1"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
},
"soupsieve": {
"hashes": [
"sha256:10687fc53eeb3518e01a0ac84d3d711da623d3298a3039459d3f649927c4a270",
"sha256:b23a0d7da0247200fe83c67c34de9d7599ad404106367313d8e65e04174d0b4b"
],
"version": "==1.7.2"
},
"sparqlwrapper": {
"hashes": [
"sha256:2a95fdede2833be660b81092934c4a0054ff85f2693098556762a2759ea486f1",
"sha256:7f4c8d38ea1bfcffbc358c9a05de35a3fd7152cc3e8ea57963ee7a0a242f7a5e",
"sha256:acf6d60f0a3684cb673653b07871acb0c350a974b891f20f8ac94926ff9eb2ff"
],
"index": "pypi",
"version": "==1.8.2"
},
"stix2-patterns": {
"hashes": [
"sha256:137cbe28d29af774d526a49d60b3a40af7c19fe1e5f252e741bb25f253d5616f"
],
"index": "pypi",
"version": "==1.1.0"
},
"tornado": {
"hashes": [
"sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d",
"sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409",
"sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f",
"sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f",
"sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5",
"sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb",
"sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444"
],
"version": "==5.1.1"
},
"url-normalize": {
"hashes": [
"sha256:3468d64cb22a9092a2c086e46c781f741dc9a1689b24e9b48ab5e8244ffa6c02",
"sha256:51e0f14050c79e732d175c33d12167f5e642cc23e0cb23275236af843faf884f"
],
"version": "==1.4.1"
},
"urlarchiver": {
"hashes": [
"sha256:652e0890dab58bf62a759656671dcfb9a40eb4a77aac8a8d93154f00360238b5"
],
"index": "pypi",
"version": "==0.2"
},
"urllib3": {
"hashes": [
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
],
"version": "==1.24.1"
},
"uwhois": {
"editable": true,
"git": "https://github.com/Rafiot/uwhoisd.git",
"ref": "f6f035e52213c8abc20f2084d28cfffb399457cb",
"subdirectory": "client"
},
"vulners": {
"hashes": [
"sha256:8b468db8f8b0bad39ae51ebd4247f6ead90b6f53699e03b91ff9d63da70554d7",
"sha256:ad72378c842096cad9ebf83aa53d330117ece5d208ed7c419a21c70a8d5e2236",
"sha256:ffc92a099eeddea840fd199665992c0eb6d7ad69ac3a6730a286d00600bc5f2c"
],
"index": "pypi",
"version": "==1.3.6"
},
"wand": {
"hashes": [
"sha256:3e59e4bda9ef9d643d90e881cc950c8eee1508ec2cde1c150a1cbd5a12c1c007",
"sha256:52763dbf65d00cf98d7bc910b49329eea15896249c5555d47e169f2b6efbe166"
],
"index": "pypi",
"version": "==0.5.0"
},
"xlsxwriter": {
"hashes": [
"sha256:7cc07619760641b67112dbe0df938399d4d915d9b9924bb58eb5c17384d29cc6",
"sha256:ae22658a0fc5b9e875fa97c213d1ffd617d86dc49bf08be99ebdac814db7bf36"
],
"version": "==1.1.2"
},
"yara-python": {
"hashes": [
"sha256:03e5c5e333c8572e7994b0b11964d515d61a393f23c5e272f8d0e4229f368c58",
"sha256:0423e08bd618752a028ac0405ff8e0103f3a8fd607dde7618a64a4c010c3757b",
"sha256:0a0dd632dcdb347d1a9a8b1f6a83b3a77d5e63f691357ea4021fb1cf1d7ff0a4",
"sha256:728b99627a8072a877eaaa4dafb4eff39d1b14ff4fd70d39f18899ce81e29625",
"sha256:7cb0d5724eccfa52e1bcd352a56cb4dc422aa51f5f6d0945d4f830783927513b",
"sha256:8c76531e89806c0309586dd4863a972d12f1d5d63261c6d4b9331a99859fd1d8",
"sha256:9472676583e212bc4e17c2236634e02273d53c872b350f0571b48e06183de233",
"sha256:9735b680a7d95c1d3f255c351bb067edc62cdb3c0999f7064278cb2c85245405",
"sha256:997f104590167220a9af5564c042ec4d6534261e7b8a5b49655d8dffecc6b8a2",
"sha256:a48e071d02a3699363e628ac899b5b7237803bcb4b512c92ebcb4fb9b1488497",
"sha256:b67c0d75a6519ca357b4b85ede9768c96a81fff20fbc169bd805ff009ddee561"
],
"index": "pypi",
"version": "==3.8.1"
},
"yarl": {
"hashes": [
"sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9",
"sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f",
"sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb",
"sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320",
"sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842",
"sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0",
"sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829",
"sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310",
"sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4",
"sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8",
"sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1"
],
"version": "==1.3.0"
}
},
"develop": {
"atomicwrites": {
"hashes": [
"sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
"sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
],
"version": "==1.2.1"
},
"attrs": {
"hashes": [
"sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
"sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
],
"version": "==18.2.0"
},
"certifi": {
"hashes": [
"sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
"sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
],
"version": "==2018.11.29"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"codecov": {
"hashes": [
"sha256:8ed8b7c6791010d359baed66f84f061bba5bd41174bf324c31311e8737602788",
"sha256:ae00d68e18d8a20e9c3288ba3875ae03db3a8e892115bf9b83ef20507732bed4"
],
"index": "pypi",
"version": "==2.0.15"
},
"coverage": {
"hashes": [
"sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f",
"sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe",
"sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d",
"sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0",
"sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607",
"sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d",
"sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b",
"sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3",
"sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e",
"sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815",
"sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36",
"sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1",
"sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14",
"sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c",
"sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794",
"sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b",
"sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840",
"sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd",
"sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82",
"sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952",
"sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389",
"sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f",
"sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4",
"sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da",
"sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647",
"sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d",
"sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42",
"sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478",
"sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b",
"sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb",
"sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9"
],
"version": "==4.5.2"
},
"flake8": {
"hashes": [
"sha256:6a35f5b8761f45c5513e3405f110a86bea57982c3b75b766ce7b65217abe1670",
"sha256:c01f8a3963b3571a8e6bd7a4063359aff90749e160778e03817cd9b71c9e07d2"
],
"index": "pypi",
"version": "==3.6.0"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
],
"version": "==2.8"
},
"mccabe": {
"hashes": [
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
],
"version": "==0.6.1"
},
"more-itertools": {
"hashes": [
"sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4",
"sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc",
"sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"
],
"version": "==5.0.0"
},
"nose": {
"hashes": [
"sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac",
"sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a",
"sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98"
],
"index": "pypi",
"version": "==1.3.7"
},
"pluggy": {
"hashes": [
"sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616",
"sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a"
],
"version": "==0.8.1"
},
"py": {
"hashes": [
"sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
"sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
],
"version": "==1.7.0"
},
"pycodestyle": {
"hashes": [
"sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
"sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
],
"version": "==2.4.0"
},
"pyflakes": {
"hashes": [
"sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49",
"sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae"
],
"version": "==2.0.0"
},
"pytest": {
"hashes": [
"sha256:41568ea7ecb4a68d7f63837cf65b92ce8d0105e43196ff2b26622995bb3dc4b2",
"sha256:c3c573a29d7c9547fb90217ece8a8843aa0c1328a797e200290dc3d0b4b823be"
],
"index": "pypi",
"version": "==4.1.1"
},
"requests": {
"hashes": [
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
],
"index": "pypi",
"version": "==2.21.0"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
},
"urllib3": {
"hashes": [
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
],
"version": "==1.24.1"
}
}
}

2
README.md
View File

@ -17,7 +17,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
### Expansion modules
* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
* [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking.
* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.

94
REQUIREMENTS
View File

@ -1,31 +1,63 @@
tornado
dnspython
requests
urlarchiver
passivetotal
PyPDNS
pypssl
redis
pyeupi
ipasn-redis
asnhistory
git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
git+https://github.com/MISP/PyMISP.git#egg=pymisp
git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe
git+https://github.com/sebdraven/pydnstrails#egg=pydnstrails
pillow
pytesseract
wand
SPARQLWrapper
domaintools_api
pygeoip
bs4
oauth2
yara-python==3.8.0
sigmatools
stix2-patterns
maclookup
vulners
psutil
blockchain
git+https://github.com/MISP/PyIntel471.git
-i https://pypi.org/simple
-e .
-e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client
-e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client
-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
-e git+https://github.com/MISP/PyMISP.git@d4934cdf5f537c9f42ae37be7878de1848961de0#egg=pymisp
-e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
-e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe
aiohttp==3.4.4
antlr4-python3-runtime==4.7.2 ; python_version >= '3'
async-timeout==3.0.1
attrs==18.2.0
beautifulsoup4==4.7.1
blockchain==1.4.4
certifi==2018.11.29
chardet==3.0.4
click-plugins==1.0.4
click==7.0
colorama==0.4.1
dnspython==1.16.0
domaintools-api==0.3.3
enum-compat==0.0.2
ez-setup==0.9
future==0.17.1
httplib2==0.12.0
idna-ssl==1.1.0 ; python_version < '3.7'
idna==2.8
isodate==0.6.0
jsonschema==2.6.0
maclookup==1.0.3
multidict==4.5.2
oauth2==1.9.0.post1
passivetotal==1.0.30
pillow==5.4.1
psutil==5.4.8
pyeupi==1.0
pygeoip==0.3.2
pyparsing==2.3.1
pypdns==1.3
pypssl==2.1
pytesseract==0.2.6
python-dateutil==2.7.5
pyyaml==3.13
rdflib==4.2.2
redis==3.0.1
requests-cache==0.4.13
requests==2.21.0
shodan==1.10.4
sigmatools==0.7.1
six==1.12.0
soupsieve==1.7.2
sparqlwrapper==1.8.2
stix2-patterns==1.1.0
tornado==5.1.1
url-normalize==1.4.1
urlarchiver==0.2
urllib3==1.24.1
vulners==1.3.6
wand==0.5.0
xlsxwriter==1.1.2
yara-python==3.8.1
yarl==1.3.0

24
doc/documentation.md
View File

@ -2,25 +2,21 @@
## Expansion Modules
#### [asn_history](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/asn_history.py)
#### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py)
Query an ASN description history service (https://github.com/CIRCL/ASN-Description-History.git).
Query BGP Ranking (https://bgpranking-ng.circl.lu/).
- **features**:
>The module takes an AS number attribute as input and displays its description and history.
>The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking.
>
>For a proper working, a communication with a redis database is needed, thus 3 parameters are needed:
>- host, the address of the redis server
>- port, the port used by redis
>- db, the index of the database used
>
- **input**:
>Autonomous system number.
- **output**:
>Text containing a description of the ASN and its history.
>Text containing a description of the ASN, its history, and the position in BGP Ranking.
- **references**:
>https://github.com/CIRCL/ASN-Description-History.git
>https://github.com/D4-project/BGP-Ranking/
- **requirements**:
>asnhistory python library
>pybgpranking python library
-----
@ -331,17 +327,17 @@ Module to access intelmqs eventdb.
#### [ipasn](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py)
Module to query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git).
Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).
- **features**:
>This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input.
- **input**:
>An IP address MISP attribute.
- **output**:
>Text describing additional information about the input after a query on the IP-ASN-history database.
>Text describing additional information about the input after a query on the IPASN-history database.
- **references**:
>https://www.circl.lu/services/ip-asn-history/
>https://github.com/D4-project/IPASN-History
- **requirements**:
>ipasn_redis: Python library to access IP-ASN-history instance via redis, An IP-ASN-history instance information (host, port and database index)
>pyipasnhistory: Python library to access IPASN-history instance
-----

8
doc/expansion/asn_history.json
View File

@ -1,8 +0,0 @@
{
"description": "Query an ASN description history service (https://github.com/CIRCL/ASN-Description-History.git).",
"requirements": ["asnhistory python library"],
"features": "The module takes an AS number attribute as input and displays its description and history.\n\nFor a proper working, a communication with a redis database is needed, thus 3 parameters are needed:\n- host, the address of the redis server\n- port, the port used by redis\n- db, the index of the database used\n",
"references": ["https://github.com/CIRCL/ASN-Description-History.git"],
"input": "Autonomous system number.",
"output": "Text containing a description of the ASN and its history."
}

8
doc/expansion/bgpranking.json Normal file
View File

@ -0,0 +1,8 @@
{
"description": "Query BGP Ranking (https://bgpranking-ng.circl.lu/).",
"requirements": ["pybgpranking python library"],
"features": "The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking.\n\n",
"references": ["https://github.com/D4-project/BGP-Ranking/"],
"input": "Autonomous system number.",
"output": "Text containing a description of the ASN, its history, and the position in BGP Ranking."
}

8
doc/expansion/ipasn.json
View File

@ -1,8 +1,8 @@
{
"description": "Module to query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git).",
"requirements": ["ipasn_redis: Python library to access IP-ASN-history instance via redis", "An IP-ASN-history instance information (host, port and database index)"],
"description": "Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).",
"requirements": ["pyipasnhistory: Python library to access IPASN-history instance"],
"input": "An IP address MISP attribute.",
"output": "Text describing additional information about the input after a query on the IP-ASN-history database.",
"references": ["https://www.circl.lu/services/ip-asn-history/"],
"output": "Text describing additional information about the input after a query on the IPASN-history database.",
"references": ["https://github.com/D4-project/IPASN-History"],
"features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input."
}

4
doc/generate_documentation.py
View File

@ -7,6 +7,7 @@ titles = ['Expansion Modules', 'Export Modules', 'Import Modules']
markdown = ["# MISP modules documentation\n"]
githublink = 'https://github.com/MISP/misp-modules/tree/master/misp_modules/modules'
def generate_doc(root_path):
for _path, title in zip(module_types, titles):
markdown.append('\n## {}\n'.format(title))
@ -18,7 +19,7 @@ def generate_doc(root_path):
githubref = '{}/{}.py'.format(githubpath, modulename)
markdown.append('\n#### [{}]({})\n'.format(modulename, githubref))
filename = os.path.join(current_path, _file)
with open(filename, 'rt', encoding='utf-8') as f:
with open(filename, 'rt') as f:
definition = json.loads(f.read())
if 'logo' in definition:
markdown.append('\n<img src={} height=60>\n'.format(definition.pop('logo')))
@ -32,6 +33,7 @@ def generate_doc(root_path):
with open('documentation.md', 'w') as w:
w.write(''.join(markdown))
if __name__ == '__main__':
root_path = os.path.dirname(os.path.realpath(__file__))
generate_doc(root_path)

10
misp_modules/helpers/cache.py
View File

@ -32,7 +32,7 @@ db = int(os.getenv("REDIS_DATABASE")) if os.getenv("REDIS_DATABASE") else 0
def selftest(enable=True):
if not enable:
return False
r = redis.StrictRedis(host=hostname, port=port, db=db)
r = redis.Redis(host=hostname, port=port, db=db)
try:
r.ping()
except Exception:
@ -42,11 +42,11 @@ def selftest(enable=True):
def get(modulename=None, query=None, value=None, debug=False):
if (modulename is None or query is None):
return False
r = redis.StrictRedis(host=hostname, port=port, db=db)
r = redis.Redis(host=hostname, port=port, db=db, decode_responses=True)
h = hashlib.sha1()
h.update(query.encode('UTF-8'))
hv = h.hexdigest()
key = "m:" + modulename + ":" + hv
key = "m:{}:{}".format(modulename, hv)
if not r.exists(key):
if debug:
@ -60,7 +60,7 @@ def get(modulename=None, query=None, value=None, debug=False):
def flush():
r = redis.StrictRedis(host=hostname, port=port, db=db)
r = redis.StrictRedis(host=hostname, port=port, db=db, decode_responses=True)
returncode = r.flushdb()
return returncode
@ -72,7 +72,7 @@ if __name__ == "__main__":
else:
print("Selftest ok")
v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True)
if v == b'barfoo':
if v == 'barfoo':
print("Cache ok")
v = get(modulename="testmodule", query="abcdef")
print(v)

2
misp_modules/modules/expansion/__init__.py
View File

@ -1,6 +1,6 @@
from . import _vmray # noqa
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl',
__all__ = ['vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',
'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal',
'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep',

20
misp_modules/modules/expansion/asn_history.py → misp_modules/modules/expansion/bgpranking.py
View File

@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
import json
from asnhistory import ASNHistory
from datetime import date, timedelta
from pybgpranking import BGPRanking
misperrors = {'error': 'Error'}
mispattributes = {'input': ['AS'], 'output': ['freetext']}
@ -9,8 +10,6 @@ moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot',
'description': 'Query an ASN Description history service (https://github.com/CIRCL/ASN-Description-History.git)',
'module-type': ['expansion', 'hover']}
moduleconfig = ['host', 'port', 'db']
def handler(q=False):
if q is False:
@ -22,19 +21,11 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host')
and request['config'].get('port')
and request['config'].get('db')):
misperrors['error'] = 'ASN description history configuration is missing'
return misperrors
asnhistory = ASNHistory(host=request['config'].get('host'),
port=request['config'].get('port'), db=request['config'].get('db'))
values = ['{} {}'.format(date.isoformat(), description) for date, description in asnhistory.get_all_descriptions(toquery)]
bgpranking = BGPRanking()
values = bgpranking.query(toquery, date=(date.today() - timedelta(1)).isoformat())
if not values:
misperrors['error'] = 'Unable to find descriptions for this ASN'
misperrors['error'] = 'Unable to find the ASN in BGP Ranking'
return misperrors
return {'results': [{'types': mispattributes['output'], 'values': values}]}
@ -44,5 +35,4 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

19
misp_modules/modules/expansion/ipasn.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
import json
from ipasn_redis import IPASN
from pyipasnhistory import IPASNHistory
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']}
@ -9,8 +9,6 @@ moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot',
'description': 'Query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git)',
'module-type': ['expansion', 'hover']}
moduleconfig = ['host', 'port', 'db']
def handler(q=False):
if q is False:
@ -24,18 +22,8 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host')
and request['config'].get('port')
and request['config'].get('db')):
misperrors['error'] = 'IP ASN history configuration is missing'
return misperrors
ipasn = IPASN(host=request['config'].get('host'),
port=request['config'].get('port'), db=request['config'].get('db'))
values = []
for first_seen, last_seen, asn, block in ipasn.aggregate_history(toquery):
values.append('{} {} {} {}'.format(first_seen.decode(), last_seen.decode(), asn.decode(), block))
ipasn = IPASNHistory()
values = ipasn.query(toquery)
if not values:
misperrors['error'] = 'Unable to find the history of this IP'
@ -48,5 +36,4 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

7
misp_modules/modules/expansion/sigma_queries.py
View File

@ -2,9 +2,10 @@ import sys
import io
import json
try:
from sigma.parser import SigmaCollectionParser
from sigma.config import SigmaConfiguration
from sigma.backends import getBackend, BackendOptions
from sigma.parser.collection import SigmaCollectionParser
from sigma.configuration import SigmaConfiguration
from sigma.backends.base import BackendOptions
from sigma.backends.discovery import getBackend
except ImportError:
print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.")

4
misp_modules/modules/expansion/sigma_syntax_validator.py
View File

@ -1,8 +1,8 @@
import json
try:
import yaml
from sigma.parser import SigmaParser
from sigma.config import SigmaConfiguration
from sigma.parser.rule import SigmaParser
from sigma.configuration import SigmaConfiguration
except ImportError:
print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.")

15
setup.py
View File

@ -23,18 +23,7 @@ setup(
],
install_requires=[
'tornado',
'dnspython3',
'requests',
'urlarchiver',
'passivetotal',
'PyPDNS',
'pypssl',
'redis',
'pyeupi',
'ipasn-redis',
'asnhistory',
'pillow',
'pytesseract',
'shodan',
'psutil',
'redis>=3'
],
)