Browse Source

chg: Use pipenv, update bgpranking/ipasn modules

pipenv
Raphaël Vinot 3 years ago
parent
commit
55f05e0524
  1. 19
      .travis.yml
  2. 45
      Pipfile
  3. 735
      Pipfile.lock
  4. 2
      README.md
  5. 94
      REQUIREMENTS
  6. 26
      doc/documentation.md
  7. 8
      doc/expansion/asn_history.json
  8. 8
      doc/expansion/bgpranking.json
  9. 10
      doc/expansion/ipasn.json
  10. 6
      doc/generate_documentation.py
  11. 10
      misp_modules/helpers/cache.py
  12. 20
      misp_modules/modules/expansion/bgpranking.py
  13. 19
      misp_modules/modules/expansion/ipasn.py
  14. 15
      setup.py

19
.travis.yml

@ -11,25 +11,24 @@ python:
- "3.7-dev"
install:
- pip install -U nose codecov pytest flake8
- pip install -U -r REQUIREMENTS
- pip install .
- pip install pipenv
- pipenv install --dev
script:
- coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 &
- pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -l 127.0.0.1 &
- pid=$!
- sleep 5
- nosetests --with-coverage --cover-package=misp_modules
- pipenv run nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid
- pushd ~/
- coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
- pipenv run coverage run -m --parallel-mode --source=misp_modules misp_modules.__init__ -s -l 127.0.0.1 &
- pid=$!
- popd
- sleep 5
- nosetests --with-coverage --cover-package=misp_modules
- pipenv run nosetests --with-coverage --cover-package=misp_modules
- kill -s INT $pid
- flake8 --ignore=E501,W503 misp_modules
- pipenv run flake8 --ignore=E501,W503 misp_modules
after_success:
- coverage combine .coverage*
- codecov
- pipenv run coverage combine .coverage*
- pipenv run codecov

45
Pipfile

@ -0,0 +1,45 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
nose = "*"
codecov = "*"
pytest = "*"
flake8 = "*"
[packages]
dnspython = "*"
requests = "*"
urlarchiver = "*"
passivetotal = "*"
pypdns = "*"
pypssl = "*"
pyeupi = "*"
uwhois = {editable = true,git = "https://github.com/Rafiot/uwhoisd.git",ref = "testing",subdirectory = "client"}
pymisp = {editable = true,git = "https://github.com/MISP/PyMISP.git"}
pyonyphe = {editable = true,git = "https://github.com/sebdraven/pyonyphe"}
pydnstrails = {editable = true,git = "https://github.com/sebdraven/pydnstrails"}
pytesseract = "*"
pygeoip = "*"
beautifulsoup4 = "*"
oauth2 = "*"
yara-python = ">=3.8.0"
sigmatools = "*"
stix2-patterns = "*"
maclookup = "*"
vulners = "*"
blockchain = "*"
pyintel471 = {editable = true,git = "https://github.com/MISP/PyIntel471.git"}
shodan = "*"
Pillow = "*"
Wand = "*"
SPARQLWrapper = "*"
domaintools_api = "*"
misp-modules = {editable = true,path = "."}
pybgpranking = {editable = true,git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"}
pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"}
[requires]
python_version = "3.6"

735
Pipfile.lock

@ -0,0 +1,735 @@
{
"_meta": {
"hash": {
"sha256": "f501a84bdd41ca21a2af020278ce030985cccd5f2f5683cd075797be4523587d"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.6"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"aiohttp": {
"hashes": [
"sha256:0419705a36b43c0ac6f15469f9c2a08cad5c939d78bd12a5c23ea167c8253b2b",
"sha256:1812fc4bc6ac1bde007daa05d2d0f61199324e0cc893b11523e646595047ca08",
"sha256:2214b5c0153f45256d5d52d1e0cafe53f9905ed035a142191727a5fb620c03dd",
"sha256:275909137f0c92c61ba6bb1af856a522d5546f1de8ea01e4e726321c697754ac",
"sha256:3983611922b561868428ea1e7269e757803713f55b53502423decc509fef1650",
"sha256:51afec6ffa50a9da4cdef188971a802beb1ca8e8edb40fa429e5e529db3475fa",
"sha256:589f2ec8a101a0f340453ee6945bdfea8e1cd84c8d88e5be08716c34c0799d95",
"sha256:789820ddc65e1f5e71516adaca2e9022498fa5a837c79ba9c692a9f8f916c330",
"sha256:7a968a0bdaaf9abacc260911775611c9a602214a23aeb846f2eb2eeaa350c4dc",
"sha256:7aeefbed253f59ea39e70c5848de42ed85cb941165357fc7e87ab5d8f1f9592b",
"sha256:7b2eb55c66512405103485bd7d285a839d53e7fdc261ab20e5bcc51d7aaff5de",
"sha256:87bc95d3d333bb689c8d755b4a9d7095a2356108002149523dfc8e607d5d32a4",
"sha256:9d80e40db208e29168d3723d1440ecbb06054d349c5ece6a2c5a611490830dd7",
"sha256:a1b442195c2a77d33e4dbee67c9877ccbdd3a1f686f91eb479a9577ed8cc326b",
"sha256:ab3d769413b322d6092f169f316f7b21cd261a7589f7e31db779d5731b0480d8",
"sha256:b066d3dec5d0f5aee6e34e5765095dc3d6d78ef9839640141a2b20816a0642bd",
"sha256:b24e7845ae8de3e388ef4bcfcf7f96b05f52c8e633b33cf8003a6b1d726fc7c2",
"sha256:c59a953c3f8524a7c86eaeaef5bf702555be12f5668f6384149fe4bb75c52698",
"sha256:cf2cc6c2c10d242790412bea7ccf73726a9a44b4c4b073d2699ef3b48971fd95",
"sha256:e0c9c8d4150ae904f308ff27b35446990d2b1dfc944702a21925937e937394c6",
"sha256:f1839db4c2b08a9c8f9788112644f8a8557e8e0ecc77b07091afabb941dc55d0",
"sha256:f3df52362be39908f9c028a65490fae0475e4898b43a03d8aa29d1e765b45e07"
],
"version": "==3.4.4"
},
"antlr4-python3-runtime": {
"hashes": [
"sha256:168cdcec8fb9152e84a87ca6fd261b3d54c8f6358f42ab3b813b14a7193bb50b"
],
"markers": "python_version >= '3'",
"version": "==4.7.2"
},
"async-timeout": {
"hashes": [
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
"sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
],
"version": "==3.0.1"
},
"attrs": {
"hashes": [
"sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
"sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
],
"version": "==18.2.0"
},
"beautifulsoup4": {
"hashes": [
"sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858",
"sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348",
"sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718"
],
"index": "pypi",
"version": "==4.7.1"
},
"blockchain": {
"hashes": [
"sha256:dbaa3eebb6f81b4245005739da802c571b09f98d97eb66520afd95d9ccafebe2"
],
"index": "pypi",
"version": "==1.4.4"
},
"certifi": {
"hashes": [
"sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
"sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
],
"version": "==2018.11.29"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"click": {
"hashes": [
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
"sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
],
"version": "==7.0"
},
"click-plugins": {
"hashes": [
"sha256:b1ee1ccc9421c73007fe290680d97984eb6eaf5f4512b7620c6aa46031d6cb6b",
"sha256:dfed74b5063546a137de99baaaf742b4de4337ad2b3e1df5ec7c8a256adc0847"
],
"version": "==1.0.4"
},
"colorama": {
"hashes": [
"sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d",
"sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"
],
"version": "==0.4.1"
},
"dnspython": {
"hashes": [
"sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
"sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d"
],
"index": "pypi",
"version": "==1.16.0"
},
"domaintools-api": {
"hashes": [
"sha256:f567f407b8997e947df5badf7c2bea64fdfd33c54ade24eab36ef575fb71ccb7"
],
"index": "pypi",
"version": "==0.3.3"
},
"enum-compat": {
"hashes": [
"sha256:939ceff18186a5762ae4db9fa7bfe017edbd03b66526b798dd8245394c8a4192"
],
"version": "==0.0.2"
},
"ez-setup": {
"hashes": [
"sha256:303c5b17d552d1e3fb0505d80549f8579f557e13d8dc90e5ecef3c07d7f58642"
],
"version": "==0.9"
},
"future": {
"hashes": [
"sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8"
],
"version": "==0.17.1"
},
"httplib2": {
"hashes": [
"sha256:f61fb838a94ce3b349aa32c92fd8430f7e3511afdb18bf9640d647e30c90a6d6"
],
"version": "==0.12.0"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
],
"version": "==2.8"
},
"idna-ssl": {
"hashes": [
"sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c"
],
"markers": "python_version < '3.7'",
"version": "==1.1.0"
},
"isodate": {
"hashes": [
"sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8",
"sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81"
],
"version": "==0.6.0"
},
"jsonschema": {
"hashes": [
"sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08",
"sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02"
],
"version": "==2.6.0"
},
"maclookup": {
"hashes": [
"sha256:33bf8eaebe3b1e4ab4ae9277dd93c78024e0ebf6b3c42f76c37695bc26ce287a",
"sha256:795e792cd3e03c9bdad77e52904d43ff71d3ac03b360443f99d4bae08a6bffef"
],
"index": "pypi",
"version": "==1.0.3"
},
"misp-modules": {
"editable": true,
"path": "."
},
"multidict": {
"hashes": [
"sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f",
"sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3",
"sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef",
"sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b",
"sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73",
"sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc",
"sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3",
"sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd",
"sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351",
"sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941",
"sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d",
"sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1",
"sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b",
"sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a",
"sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3",
"sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7",
"sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0",
"sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0",
"sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014",
"sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5",
"sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036",
"sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d",
"sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a",
"sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce",
"sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1",
"sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a",
"sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9",
"sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7",
"sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b"
],
"version": "==4.5.2"
},
"oauth2": {
"hashes": [
"sha256:15b5c42301f46dd63113f1214b0d81a8b16254f65a86d3c32a1b52297f3266e6",
"sha256:c006a85e7c60107c7cc6da1b184b5c719f6dd7202098196dfa6e55df669b59bf"
],
"index": "pypi",
"version": "==1.9.0.post1"
},
"passivetotal": {
"hashes": [
"sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d"
],
"index": "pypi",
"version": "==1.0.30"
},
"pillow": {
"hashes": [
"sha256:051de330a06c99d6f84bcf582960487835bcae3fc99365185dc2d4f65a390c0e",
"sha256:0ae5289948c5e0a16574750021bd8be921c27d4e3527800dc9c2c1d2abc81bf7",
"sha256:0b1efce03619cdbf8bcc61cfae81fcda59249a469f31c6735ea59badd4a6f58a",
"sha256:163136e09bd1d6c6c6026b0a662976e86c58b932b964f255ff384ecc8c3cefa3",
"sha256:18e912a6ccddf28defa196bd2021fe33600cbe5da1aa2f2e2c6df15f720b73d1",
"sha256:24ec3dea52339a610d34401d2d53d0fb3c7fd08e34b20c95d2ad3973193591f1",
"sha256:267f8e4c0a1d7e36e97c6a604f5b03ef58e2b81c1becb4fccecddcb37e063cc7",
"sha256:3273a28734175feebbe4d0a4cde04d4ed20f620b9b506d26f44379d3c72304e1",
"sha256:4c678e23006798fc8b6f4cef2eaad267d53ff4c1779bd1af8725cc11b72a63f3",
"sha256:4d4bc2e6bb6861103ea4655d6b6f67af8e5336e7216e20fff3e18ffa95d7a055",
"sha256:505738076350a337c1740a31646e1de09a164c62c07db3b996abdc0f9d2e50cf",
"sha256:5233664eadfa342c639b9b9977190d64ad7aca4edc51a966394d7e08e7f38a9f",
"sha256:5d95cb9f6cced2628f3e4de7e795e98b2659dfcc7176ab4a01a8b48c2c2f488f",
"sha256:7eda4c737637af74bac4b23aa82ea6fbb19002552be85f0b89bc27e3a762d239",
"sha256:801ddaa69659b36abf4694fed5aa9f61d1ecf2daaa6c92541bbbbb775d97b9fe",
"sha256:825aa6d222ce2c2b90d34a0ea31914e141a85edefc07e17342f1d2fdf121c07c",
"sha256:9c215442ff8249d41ff58700e91ef61d74f47dfd431a50253e1a1ca9436b0697",
"sha256:a3d90022f2202bbb14da991f26ca7a30b7e4c62bf0f8bf9825603b22d7e87494",
"sha256:a631fd36a9823638fe700d9225f9698fb59d049c942d322d4c09544dc2115356",
"sha256:a6523a23a205be0fe664b6b8747a5c86d55da960d9586db039eec9f5c269c0e6",
"sha256:a756ecf9f4b9b3ed49a680a649af45a8767ad038de39e6c030919c2f443eb000",
"sha256:b117287a5bdc81f1bac891187275ec7e829e961b8032c9e5ff38b70fd036c78f",
"sha256:ba04f57d1715ca5ff74bb7f8a818bf929a204b3b3c2c2826d1e1cc3b1c13398c",
"sha256:cd878195166723f30865e05d87cbaf9421614501a4bd48792c5ed28f90fd36ca",
"sha256:cee815cc62d136e96cf76771b9d3eb58e0777ec18ea50de5cfcede8a7c429aa8",
"sha256:d1722b7aa4b40cf93ac3c80d3edd48bf93b9208241d166a14ad8e7a20ee1d4f3",
"sha256:d7c1c06246b05529f9984435fc4fa5a545ea26606e7f450bdbe00c153f5aeaad",
"sha256:e9c8066249c040efdda84793a2a669076f92a301ceabe69202446abb4c5c5ef9",
"sha256:f227d7e574d050ff3996049e086e1f18c7bd2d067ef24131e50a1d3fe5831fbc",
"sha256:fc9a12aad714af36cf3ad0275a96a733526571e52710319855628f476dcb144e"
],
"index": "pypi",
"version": "==5.4.1"
},
"psutil": {
"hashes": [
"sha256:1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736",
"sha256:1c71b9716790e202a00ab0931a6d1e25db1aa1198bcacaea2f5329f75d257fff",
"sha256:3b7a4daf4223dae171a67a89314ac5ca0738e94064a78d99cfd751c55d05f315",
"sha256:3e19be3441134445347af3767fa7770137d472a484070840eee6653b94ac5576",
"sha256:6e265c8f3da00b015d24b842bfeb111f856b13d24f2c57036582568dc650d6c3",
"sha256:809c9cef0402e3e48b5a1dddc390a8a6ff58b15362ea5714494073fa46c3d293",
"sha256:b4d1b735bf5b120813f4c89db8ac22d89162c558cbd7fdd298866125fe906219",
"sha256:bbffac64cfd01c6bcf90eb1bedc6c80501c4dae8aef4ad6d6dd49f8f05f6fc5a",
"sha256:bfcea4f189177b2d2ce4a34b03c4ac32c5b4c22e21f5b093d9d315e6e253cd81"
],
"version": "==5.4.8"
},
"pybgpranking": {
"editable": true,
"git": "https://github.com/D4-project/BGP-Ranking.git/",
"ref": "7e698f87366e6f99b4d0d11852737db28e3ddc62",
"subdirectory": "client"
},
"pydnstrails": {
"editable": true,
"git": "https://github.com/sebdraven/pydnstrails",
"ref": "48c1f740025c51289f43a24863d1845ff12fd21a"
},
"pyeupi": {
"hashes": [
"sha256:35b0e6b430f23ecd303f7cc7a8fe5147cf2509a5b2254eaf9695392c0af02901"
],
"index": "pypi",
"version": "==1.0"
},
"pygeoip": {
"hashes": [
"sha256:1938b9dac7b00d77f94d040b9465ea52c938f3fcdcd318b5537994f3c16aef96",
"sha256:f22c4e00ddf1213e0fae36dc60b46ee7c25a6339941ec1a975539014c1f9a96d"
],
"index": "pypi",
"version": "==0.3.2"
},
"pyintel471": {
"editable": true,
"git": "https://github.com/MISP/PyIntel471.git",
"ref": "0df8d51f1c1425de66714b3a5a45edb69b8cc2fc"
},
"pyipasnhistory": {
"editable": true,
"git": "https://github.com/D4-project/IPASN-History.git/",
"ref": "e846cd36fe1ed6b22f60890bba89f84e61b62e59",
"subdirectory": "client"
},
"pymisp": {
"editable": true,
"git": "https://github.com/MISP/PyMISP.git",
"ref": "d4934cdf5f537c9f42ae37be7878de1848961de0"
},
"pyonyphe": {
"editable": true,
"git": "https://github.com/sebdraven/pyonyphe",
"ref": "66329baeee7cab844f2203c047c2551828eaf14d"
},
"pyparsing": {
"hashes": [
"sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a",
"sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3"
],
"version": "==2.3.1"
},
"pypdns": {
"hashes": [
"sha256:0356360156dd26d2cf27a415a10ff2bd1ff1d2eb3b2dd51b35553d60b87fd328"
],
"index": "pypi",
"version": "==1.3"
},
"pypssl": {
"hashes": [
"sha256:4dbe772aefdf4ab18934d83cde79e2fc5d5ba9d2b4153dc419a63faab3432643"
],
"index": "pypi",
"version": "==2.1"
},
"pytesseract": {
"hashes": [
"sha256:11c20321595b6e2e904b594633edf1a717212b13bac7512986a2d807b8849770"
],
"index": "pypi",
"version": "==0.2.6"
},
"python-dateutil": {
"hashes": [
"sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
"sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
],
"version": "==2.7.5"
},
"pyyaml": {
"hashes": [
"sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b",
"sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf",
"sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a",
"sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3",
"sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1",
"sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1",
"sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613",
"sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04",
"sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f",
"sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537",
"sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531"
],
"version": "==3.13"
},
"rdflib": {
"hashes": [
"sha256:58d5994610105a457cff7fdfe3d683d87786c5028a45ae032982498a7e913d6f",
"sha256:da1df14552555c5c7715d8ce71c08f404c988c58a1ecd38552d0da4fc261280d"
],
"version": "==4.2.2"
},
"redis": {
"hashes": [
"sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf",
"sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a"
],
"version": "==3.0.1"
},
"requests": {
"hashes": [
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
],
"index": "pypi",
"version": "==2.21.0"
},
"requests-cache": {
"hashes": [
"sha256:e9270030becc739b0a7f7f834234c73a878b2d794122bf76f40055a22419eb67",
"sha256:fe561ca119879bbcfb51f03a35e35b425e18f338248e59fd5cf2166c77f457a2"
],
"version": "==0.4.13"
},
"shodan": {
"hashes": [
"sha256:c40abb6ff2fd66bdee9f773746fb961eefdfaa8e720a07cb12fb70def136268d"
],
"index": "pypi",
"version": "==1.10.4"
},
"sigmatools": {
"hashes": [
"sha256:98c9897f27e7c99f398bff537bb6b0259599177d955f8b60a22db1b246f9cb0b"
],
"index": "pypi",
"version": "==0.7.1"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
},
"soupsieve": {
"hashes": [
"sha256:10687fc53eeb3518e01a0ac84d3d711da623d3298a3039459d3f649927c4a270",
"sha256:b23a0d7da0247200fe83c67c34de9d7599ad404106367313d8e65e04174d0b4b"
],
"version": "==1.7.2"
},
"sparqlwrapper": {
"hashes": [
"sha256:2a95fdede2833be660b81092934c4a0054ff85f2693098556762a2759ea486f1",
"sha256:7f4c8d38ea1bfcffbc358c9a05de35a3fd7152cc3e8ea57963ee7a0a242f7a5e",
"sha256:acf6d60f0a3684cb673653b07871acb0c350a974b891f20f8ac94926ff9eb2ff"
],
"index": "pypi",
"version": "==1.8.2"
},
"stix2-patterns": {
"hashes": [
"sha256:137cbe28d29af774d526a49d60b3a40af7c19fe1e5f252e741bb25f253d5616f"
],
"index": "pypi",
"version": "==1.1.0"
},
"tornado": {
"hashes": [
"sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d",
"sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409",
"sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f",
"sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f",
"sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5",
"sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb",
"sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444"
],
"version": "==5.1.1"
},
"url-normalize": {
"hashes": [
"sha256:3468d64cb22a9092a2c086e46c781f741dc9a1689b24e9b48ab5e8244ffa6c02",
"sha256:51e0f14050c79e732d175c33d12167f5e642cc23e0cb23275236af843faf884f"
],
"version": "==1.4.1"
},
"urlarchiver": {
"hashes": [
"sha256:652e0890dab58bf62a759656671dcfb9a40eb4a77aac8a8d93154f00360238b5"
],
"index": "pypi",
"version": "==0.2"
},
"urllib3": {
"hashes": [
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
],
"version": "==1.24.1"
},
"uwhois": {
"editable": true,
"git": "https://github.com/Rafiot/uwhoisd.git",
"ref": "f6f035e52213c8abc20f2084d28cfffb399457cb",
"subdirectory": "client"
},
"vulners": {
"hashes": [
"sha256:8b468db8f8b0bad39ae51ebd4247f6ead90b6f53699e03b91ff9d63da70554d7",
"sha256:ad72378c842096cad9ebf83aa53d330117ece5d208ed7c419a21c70a8d5e2236",
"sha256:ffc92a099eeddea840fd199665992c0eb6d7ad69ac3a6730a286d00600bc5f2c"
],
"index": "pypi",
"version": "==1.3.6"
},
"wand": {
"hashes": [
"sha256:3e59e4bda9ef9d643d90e881cc950c8eee1508ec2cde1c150a1cbd5a12c1c007",
"sha256:52763dbf65d00cf98d7bc910b49329eea15896249c5555d47e169f2b6efbe166"
],
"index": "pypi",
"version": "==0.5.0"
},
"xlsxwriter": {
"hashes": [
"sha256:7cc07619760641b67112dbe0df938399d4d915d9b9924bb58eb5c17384d29cc6",
"sha256:ae22658a0fc5b9e875fa97c213d1ffd617d86dc49bf08be99ebdac814db7bf36"
],
"version": "==1.1.2"
},
"yara-python": {
"hashes": [
"sha256:03e5c5e333c8572e7994b0b11964d515d61a393f23c5e272f8d0e4229f368c58",
"sha256:0423e08bd618752a028ac0405ff8e0103f3a8fd607dde7618a64a4c010c3757b",
"sha256:0a0dd632dcdb347d1a9a8b1f6a83b3a77d5e63f691357ea4021fb1cf1d7ff0a4",
"sha256:728b99627a8072a877eaaa4dafb4eff39d1b14ff4fd70d39f18899ce81e29625",
"sha256:7cb0d5724eccfa52e1bcd352a56cb4dc422aa51f5f6d0945d4f830783927513b",
"sha256:8c76531e89806c0309586dd4863a972d12f1d5d63261c6d4b9331a99859fd1d8",
"sha256:9472676583e212bc4e17c2236634e02273d53c872b350f0571b48e06183de233",
"sha256:9735b680a7d95c1d3f255c351bb067edc62cdb3c0999f7064278cb2c85245405",
"sha256:997f104590167220a9af5564c042ec4d6534261e7b8a5b49655d8dffecc6b8a2",
"sha256:a48e071d02a3699363e628ac899b5b7237803bcb4b512c92ebcb4fb9b1488497",
"sha256:b67c0d75a6519ca357b4b85ede9768c96a81fff20fbc169bd805ff009ddee561"
],
"index": "pypi",
"version": "==3.8.1"
},
"yarl": {
"hashes": [
"sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9",
"sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f",
"sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb",
"sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320",
"sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842",
"sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0",
"sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829",
"sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310",
"sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4",
"sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8",
"sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1"
],
"version": "==1.3.0"
}
},
"develop": {
"atomicwrites": {
"hashes": [
"sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0",
"sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee"
],
"version": "==1.2.1"
},
"attrs": {
"hashes": [
"sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69",
"sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb"
],
"version": "==18.2.0"
},
"certifi": {
"hashes": [
"sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
"sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
],
"version": "==2018.11.29"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"codecov": {
"hashes": [
"sha256:8ed8b7c6791010d359baed66f84f061bba5bd41174bf324c31311e8737602788",
"sha256:ae00d68e18d8a20e9c3288ba3875ae03db3a8e892115bf9b83ef20507732bed4"
],
"index": "pypi",
"version": "==2.0.15"
},
"coverage": {
"hashes": [
"sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f",
"sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe",
"sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d",
"sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0",
"sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607",
"sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d",
"sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b",
"sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3",
"sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e",
"sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815",
"sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36",
"sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1",
"sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14",
"sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c",
"sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794",
"sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b",
"sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840",
"sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd",
"sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82",
"sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952",
"sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389",
"sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f",
"sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4",
"sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da",
"sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647",
"sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d",
"sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42",
"sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478",
"sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b",
"sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb",
"sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9"
],
"version": "==4.5.2"
},
"flake8": {
"hashes": [
"sha256:6a35f5b8761f45c5513e3405f110a86bea57982c3b75b766ce7b65217abe1670",
"sha256:c01f8a3963b3571a8e6bd7a4063359aff90749e160778e03817cd9b71c9e07d2"
],
"index": "pypi",
"version": "==3.6.0"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
],
"version": "==2.8"
},
"mccabe": {
"hashes": [
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
],
"version": "==0.6.1"
},
"more-itertools": {
"hashes": [
"sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4",
"sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc",
"sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"
],
"version": "==5.0.0"
},
"nose": {
"hashes": [
"sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac",
"sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a",
"sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98"
],
"index": "pypi",
"version": "==1.3.7"
},
"pluggy": {
"hashes": [
"sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616",
"sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a"
],
"version": "==0.8.1"
},
"py": {
"hashes": [
"sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
"sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
],
"version": "==1.7.0"
},
"pycodestyle": {
"hashes": [
"sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
"sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
],
"version": "==2.4.0"
},
"pyflakes": {
"hashes": [
"sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49",
"sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae"
],
"version": "==2.0.0"
},
"pytest": {
"hashes": [
"sha256:41568ea7ecb4a68d7f63837cf65b92ce8d0105e43196ff2b26622995bb3dc4b2",
"sha256:c3c573a29d7c9547fb90217ece8a8843aa0c1328a797e200290dc3d0b4b823be"
],
"index": "pypi",
"version": "==4.1.1"
},
"requests": {
"hashes": [
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
],
"index": "pypi",
"version": "==2.21.0"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
},
"urllib3": {
"hashes": [
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
],
"version": "==1.24.1"
}
}
}

2
README.md

@ -17,7 +17,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
### Expansion modules
* [ASN History](misp_modules/modules/expansion/asn_history.py) - a hover and expansion module to expand an AS number with the ASN description and its history.
* [BGP Ranking](misp_modules/modules/expansion/bgpranking.py) - a hover and expansion module to expand an AS number with the ASN description, its history, and position in BGP Ranking.
* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate seen.

94
REQUIREMENTS

@ -1,31 +1,63 @@
tornado
dnspython
requests
urlarchiver
passivetotal
PyPDNS
pypssl
redis
pyeupi
ipasn-redis
asnhistory
git+https://github.com/Rafiot/uwhoisd.git@testing#egg=uwhois&subdirectory=client
git+https://github.com/MISP/PyMISP.git#egg=pymisp
git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe
git+https://github.com/sebdraven/pydnstrails#egg=pydnstrails
pillow
pytesseract
wand
SPARQLWrapper
domaintools_api
pygeoip
bs4
oauth2
yara-python==3.8.0
sigmatools
stix2-patterns
maclookup
vulners
psutil
blockchain
git+https://github.com/MISP/PyIntel471.git
-i https://pypi.org/simple
-e .
-e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client
-e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client
-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
-e git+https://github.com/MISP/PyMISP.git@d4934cdf5f537c9f42ae37be7878de1848961de0#egg=pymisp
-e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
-e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe
aiohttp==3.4.4
antlr4-python3-runtime==4.7.2 ; python_version >= '3'
async-timeout==3.0.1
attrs==18.2.0
beautifulsoup4==4.7.1
blockchain==1.4.4
certifi==2018.11.29
chardet==3.0.4
click-plugins==1.0.4
click==7.0
colorama==0.4.1
dnspython==1.16.0
domaintools-api==0.3.3
enum-compat==0.0.2
ez-setup==0.9
future==0.17.1
httplib2==0.12.0
idna-ssl==1.1.0 ; python_version < '3.7'
idna==2.8
isodate==0.6.0
jsonschema==2.6.0
maclookup==1.0.3
multidict==4.5.2
oauth2==1.9.0.post1
passivetotal==1.0.30
pillow==5.4.1
psutil==5.4.8
pyeupi==1.0
pygeoip==0.3.2
pyparsing==2.3.1
pypdns==1.3
pypssl==2.1
pytesseract==0.2.6
python-dateutil==2.7.5
pyyaml==3.13
rdflib==4.2.2
redis==3.0.1
requests-cache==0.4.13
requests==2.21.0
shodan==1.10.4
sigmatools==0.7.1
six==1.12.0
soupsieve==1.7.2
sparqlwrapper==1.8.2
stix2-patterns==1.1.0
tornado==5.1.1
url-normalize==1.4.1
urlarchiver==0.2
urllib3==1.24.1
vulners==1.3.6
wand==0.5.0
xlsxwriter==1.1.2
yara-python==3.8.1
yarl==1.3.0

26
doc/documentation.md

@ -2,25 +2,21 @@
## Expansion Modules
#### [asn_history](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/asn_history.py)
#### [bgpranking](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/bgpranking.py)
Query an ASN description history service (https://github.com/CIRCL/ASN-Description-History.git).
Query BGP Ranking (https://bgpranking-ng.circl.lu/).
- **features**:
>The module takes an AS number attribute as input and displays its description and history.
>The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking.
>
>For a proper working, a communication with a redis database is needed, thus 3 parameters are needed:
>- host, the address of the redis server
>- port, the port used by redis
>- db, the index of the database used
>
- **input**:
>Autonomous system number.
- **output**:
>Text containing a description of the ASN and its history.
>Text containing a description of the ASN, its history, and the position in BGP Ranking.
- **references**:
>https://github.com/CIRCL/ASN-Description-History.git
>https://github.com/D4-project/BGP-Ranking/
- **requirements**:
>asnhistory python library
>pybgpranking python library
-----
@ -331,17 +327,17 @@ Module to access intelmqs eventdb.
#### [ipasn](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/ipasn.py)
Module to query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git).
Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).
- **features**:
>This module takes an IP address attribute as input and queries the CIRCL IP ASN service to get additional information about the input.
>This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input.
- **input**:
>An IP address MISP attribute.
- **output**:
>Text describing additional information about the input after a query on the IP-ASN-history database.
>Text describing additional information about the input after a query on the IPASN-history database.
- **references**:
>https://www.circl.lu/services/ip-asn-history/
>https://github.com/D4-project/IPASN-History
- **requirements**:
>ipasn_redis: Python library to access IP-ASN-history instance via redis, An IP-ASN-history instance information (host, port and database index)
>pyipasnhistory: Python library to access IPASN-history instance
-----

8
doc/expansion/asn_history.json

@ -1,8 +0,0 @@
{
"description": "Query an ASN description history service (https://github.com/CIRCL/ASN-Description-History.git).",
"requirements": ["asnhistory python library"],
"features": "The module takes an AS number attribute as input and displays its description and history.\n\nFor a proper working, a communication with a redis database is needed, thus 3 parameters are needed:\n- host, the address of the redis server\n- port, the port used by redis\n- db, the index of the database used\n",
"references": ["https://github.com/CIRCL/ASN-Description-History.git"],
"input": "Autonomous system number.",
"output": "Text containing a description of the ASN and its history."
}

8
doc/expansion/bgpranking.json

@ -0,0 +1,8 @@
{
"description": "Query BGP Ranking (https://bgpranking-ng.circl.lu/).",
"requirements": ["pybgpranking python library"],
"features": "The module takes an AS number attribute as input and displays its description and history, and position in BGP Ranking.\n\n",
"references": ["https://github.com/D4-project/BGP-Ranking/"],
"input": "Autonomous system number.",
"output": "Text containing a description of the ASN, its history, and the position in BGP Ranking."
}

10
doc/expansion/ipasn.json

@ -1,8 +1,8 @@
{
"description": "Module to query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git).",
"requirements": ["ipasn_redis: Python library to access IP-ASN-history instance via redis", "An IP-ASN-history instance information (host, port and database index)"],
"description": "Module to query an IP ASN history service (https://github.com/D4-project/IPASN-History).",
"requirements": ["pyipasnhistory: Python library to access IPASN-history instance"],
"input": "An IP address MISP attribute.",
"output": "Text describing additional information about the input after a query on the IP-ASN-history database.",
"references": ["https://www.circl.lu/services/ip-asn-history/"],
"features": "This module takes an IP address attribute as input and queries the CIRCL IP ASN service to get additional information about the input."
"output": "Text describing additional information about the input after a query on the IPASN-history database.",
"references": ["https://github.com/D4-project/IPASN-History"],
"features": "This module takes an IP address attribute as input and queries the CIRCL IPASN service to get additional information about the input."
}

6
doc/generate_documentation.py

@ -4,9 +4,10 @@ import json
module_types = ['expansion', 'export_mod', 'import_mod']
titles = ['Expansion Modules', 'Export Modules', 'Import Modules']
markdown= ["# MISP modules documentation\n"]
markdown = ["# MISP modules documentation\n"]
githublink = 'https://github.com/MISP/misp-modules/tree/master/misp_modules/modules'
def generate_doc(root_path):
for _path, title in zip(module_types, titles):
markdown.append('\n## {}\n'.format(title))
@ -18,7 +19,7 @@ def generate_doc(root_path):
githubref = '{}/{}.py'.format(githubpath, modulename)
markdown.append('\n#### [{}]({})\n'.format(modulename, githubref))
filename = os.path.join(current_path, _file)
with open(filename, 'rt', encoding='utf-8') as f:
with open(filename, 'rt') as f:
definition = json.loads(f.read())
if 'logo' in definition:
markdown.append('\n<img src={} height=60>\n'.format(definition.pop('logo')))
@ -32,6 +33,7 @@ def generate_doc(root_path):
with open('documentation.md', 'w') as w:
w.write(''.join(markdown))
if __name__ == '__main__':
root_path = os.path.dirname(os.path.realpath(__file__))
generate_doc(root_path)

10
misp_modules/helpers/cache.py

@ -30,7 +30,7 @@ db = 5
def selftest(enable=True):
if not enable:
return False
r = redis.StrictRedis(host=hostname, port=port, db=db)
r = redis.Redis(host=hostname, port=port, db=db)
try:
r.ping()
except Exception:
@ -40,11 +40,11 @@ def selftest(enable=True):
def get(modulename=None, query=None, value=None, debug=False):
if (modulename is None or query is None):
return False
r = redis.StrictRedis(host=hostname, port=port, db=db)
r = redis.Redis(host=hostname, port=port, db=db, decode_responses=True)
h = hashlib.sha1()
h.update(query.encode('UTF-8'))
hv = h.hexdigest()
key = "m:" + modulename + ":" + hv
key = "m:{}:{}".format(modulename, hv)
if not r.exists(key):
if debug:
@ -58,7 +58,7 @@ def get(modulename=None, query=None, value=None, debug=False):
def flush():
r = redis.StrictRedis(host=hostname, port=port, db=db)
r = redis.StrictRedis(host=hostname, port=port, db=db, decode_responses=True)
returncode = r.flushdb()
return returncode
@ -70,7 +70,7 @@ if __name__ == "__main__":
else:
print("Selftest ok")
v = get(modulename="testmodule", query="abcdef", value="barfoo", debug=True)
if v == b'barfoo':
if v == 'barfoo':
print("Cache ok")
v = get(modulename="testmodule", query="abcdef")
print(v)

20
misp_modules/modules/expansion/asn_history.py → misp_modules/modules/expansion/bgpranking.py

@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
import json
from asnhistory import ASNHistory
from datetime import date, timedelta
from pybgpranking import BGPRanking
misperrors = {'error': 'Error'}
mispattributes = {'input': ['AS'], 'output': ['freetext']}
@ -9,8 +10,6 @@ moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot',
'description': 'Query an ASN Description history service (https://github.com/CIRCL/ASN-Description-History.git)',
'module-type': ['expansion', 'hover']}
moduleconfig = ['host', 'port', 'db']
def handler(q=False):
if q is False:
@ -22,19 +21,11 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host')
and request['config'].get('port')
and request['config'].get('db')):
misperrors['error'] = 'ASN description history configuration is missing'
return misperrors
asnhistory = ASNHistory(host=request['config'].get('host'),
port=request['config'].get('port'), db=request['config'].get('db'))
values = ['{} {}'.format(date.isoformat(), description) for date, description in asnhistory.get_all_descriptions(toquery)]
bgpranking = BGPRanking()
values = bgpranking.query(toquery, date=(date.today() - timedelta(1)).isoformat())
if not values:
misperrors['error'] = 'Unable to find descriptions for this ASN'
misperrors['error'] = 'Unable to find the ASN in BGP Ranking'
return misperrors
return {'results': [{'types': mispattributes['output'], 'values': values}]}
@ -44,5 +35,4 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

19
misp_modules/modules/expansion/ipasn.py

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
import json
from ipasn_redis import IPASN
from pyipasnhistory import IPASNHistory
misperrors = {'error': 'Error'}
mispattributes = {'input': ['ip-src', 'ip-dst'], 'output': ['freetext']}
@ -9,8 +9,6 @@ moduleinfo = {'version': '0.1', 'author': 'Raphaël Vinot',
'description': 'Query an IP ASN history service (https://github.com/CIRCL/IP-ASN-history.git)',
'module-type': ['expansion', 'hover']}
moduleconfig = ['host', 'port', 'db']
def handler(q=False):
if q is False:
@ -24,18 +22,8 @@ def handler(q=False):
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host')
and request['config'].get('port')
and request['config'].get('db')):
misperrors['error'] = 'IP ASN history configuration is missing'
return misperrors
ipasn = IPASN(host=request['config'].get('host'),
port=request['config'].get('port'), db=request['config'].get('db'))
values = []
for first_seen, last_seen, asn, block in ipasn.aggregate_history(toquery):
values.append('{} {} {} {}'.format(first_seen.decode(), last_seen.decode(), asn.decode(), block))
ipasn = IPASNHistory()
values = ipasn.query(toquery)
if not values:
misperrors['error'] = 'Unable to find the history of this IP'
@ -48,5 +36,4 @@ def introspection():
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

15
setup.py

@ -23,18 +23,7 @@ setup(
],
install_requires=[
'tornado',
'dnspython3',
'requests',
'urlarchiver',
'passivetotal',
'PyPDNS',
'pypssl',
'redis',
'pyeupi',
'ipasn-redis',
'asnhistory',
'pillow',
'pytesseract',
'shodan',
'psutil',
'redis>=3'
],
)

Loading…
Cancel
Save