chg: Migrate to new framework
Raphaël Vinot
parent
aa7741710e
commit
80b5c045ac
|
|
@ -1,3 +1,8 @@
|
|||
# Local exclude
|
||||
scraped/
|
||||
*.swp
|
||||
lookyloo/ete3_webserver/webapi.py
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
|
@ -100,29 +105,20 @@ ENV/
|
|||
# mypy
|
||||
.mypy_cache/
|
||||
|
||||
# Redis
|
||||
|
||||
# web
|
||||
secret_key
|
||||
|
||||
cache.pid
|
||||
*.rdb
|
||||
|
||||
# Storage
|
||||
# Local config files
|
||||
config/*.json
|
||||
config/*.json.bkp
|
||||
|
||||
rawdata
|
||||
|
||||
# ardb
|
||||
storage/ardb.pid
|
||||
storage/data
|
||||
storage/repl
|
||||
|
||||
# Config file shadow server with password
|
||||
bgpranking/config/shadowserver.json
|
||||
# Ths shadow server config files are dynamically generated
|
||||
bgpranking/config/modules/shadowserver_*.json
|
||||
|
||||
# Do not store the d3 lib in the repo
|
||||
website/web/static/d3*.js
|
||||
|
||||
# Same got bootstrap-select
|
||||
website/web/static/bootstrap-select*
|
||||
|
||||
# Session key
|
||||
website/secret_key
|
||||
|
||||
*.swp
|
||||
storage/db/
|
||||
storage/kvrocks*
|
||||
website/web/static/d3.v5.js
|
||||
website/web/static/bootstrap-select.min.*
|
||||
|
|
|
|||
32
Pipfile
32
Pipfile
|
|
@ -1,32 +0,0 @@
|
|||
[[source]]
|
||||
name = "pypi"
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[packages]
|
||||
pybgpranking = {editable = true,path = "./client"}
|
||||
bgpranking = {editable = true,path = "."}
|
||||
redis = ">=3"
|
||||
hiredis = "*"
|
||||
python-dateutil = "*"
|
||||
aiohttp = "*"
|
||||
requests = "*"
|
||||
simplejson = "*"
|
||||
beautifulsoup4 = "*"
|
||||
flask = "*"
|
||||
flask-bootstrap = "*"
|
||||
gunicorn = {extras = ["gevent"],version = "<20"}
|
||||
pyipasnhistory = {editable = true,git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"}
|
||||
pycountry = "*"
|
||||
pid = {editable = true,git = "https://github.com/trbs/pid.git/"}
|
||||
pytaxonomies = {editable = true,git = "https://github.com/MISP/PyTaxonomies.git"}
|
||||
pymispgalaxies = {editable = true,git = "https://github.com/MISP/PyMISPGalaxies.git"}
|
||||
Jinja2 = ">=2.10.1" # CVE-2019-10906
|
||||
idna-ssl = {markers = "python_version < '3.7'"}
|
||||
typing-extensions = {markers = "python_version < '3.7'"}
|
||||
werkzeug = ">=0.15.3" # CVE-2019-14806
|
||||
|
||||
[requires]
|
||||
python_version = "3"
|
||||
|
|
@ -1,627 +0,0 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "8408df42fa1da47b55611862c09307fd4e2cb77b9dd45aef412c6b99b99f6b10"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
"url": "https://pypi.org/simple",
|
||||
"verify_ssl": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"aiohttp": {
|
||||
"hashes": [
|
||||
"sha256:119feb2bd551e58d83d1b38bfa4cb921af8ddedec9fad7183132db334c3133e0",
|
||||
"sha256:16d0683ef8a6d803207f02b899c928223eb219111bd52420ef3d7a8aa76227b6",
|
||||
"sha256:2eb3efe243e0f4ecbb654b08444ae6ffab37ac0ef8f69d3a2ffb958905379daf",
|
||||
"sha256:2ffea7904e70350da429568113ae422c88d2234ae776519549513c8f217f58a9",
|
||||
"sha256:40bd1b101b71a18a528ffce812cc14ff77d4a2a1272dfb8b11b200967489ef3e",
|
||||
"sha256:418597633b5cd9639e514b1d748f358832c08cd5d9ef0870026535bd5eaefdd0",
|
||||
"sha256:481d4b96969fbfdcc3ff35eea5305d8565a8300410d3d269ccac69e7256b1329",
|
||||
"sha256:4c1bdbfdd231a20eee3e56bd0ac1cd88c4ff41b64ab679ed65b75c9c74b6c5c2",
|
||||
"sha256:5563ad7fde451b1986d42b9bb9140e2599ecf4f8e42241f6da0d3d624b776f40",
|
||||
"sha256:58c62152c4c8731a3152e7e650b29ace18304d086cb5552d317a54ff2749d32a",
|
||||
"sha256:5b50e0b9460100fe05d7472264d1975f21ac007b35dcd6fd50279b72925a27f4",
|
||||
"sha256:5d84ecc73141d0a0d61ece0742bb7ff5751b0657dab8405f899d3ceb104cc7de",
|
||||
"sha256:5dde6d24bacac480be03f4f864e9a67faac5032e28841b00533cd168ab39cad9",
|
||||
"sha256:5e91e927003d1ed9283dee9abcb989334fc8e72cf89ebe94dc3e07e3ff0b11e9",
|
||||
"sha256:62bc216eafac3204877241569209d9ba6226185aa6d561c19159f2e1cbb6abfb",
|
||||
"sha256:6c8200abc9dc5f27203986100579fc19ccad7a832c07d2bc151ce4ff17190076",
|
||||
"sha256:6ca56bdfaf825f4439e9e3673775e1032d8b6ea63b8953d3812c71bd6a8b81de",
|
||||
"sha256:71680321a8a7176a58dfbc230789790639db78dad61a6e120b39f314f43f1907",
|
||||
"sha256:7c7820099e8b3171e54e7eedc33e9450afe7cd08172632d32128bd527f8cb77d",
|
||||
"sha256:7dbd087ff2f4046b9b37ba28ed73f15fd0bc9f4fdc8ef6781913da7f808d9536",
|
||||
"sha256:822bd4fd21abaa7b28d65fc9871ecabaddc42767884a626317ef5b75c20e8a2d",
|
||||
"sha256:8ec1a38074f68d66ccb467ed9a673a726bb397142c273f90d4ba954666e87d54",
|
||||
"sha256:950b7ef08b2afdab2488ee2edaff92a03ca500a48f1e1aaa5900e73d6cf992bc",
|
||||
"sha256:99c5a5bf7135607959441b7d720d96c8e5c46a1f96e9d6d4c9498be8d5f24212",
|
||||
"sha256:b84ad94868e1e6a5e30d30ec419956042815dfaea1b1df1cef623e4564c374d9",
|
||||
"sha256:bc3d14bf71a3fb94e5acf5bbf67331ab335467129af6416a437bd6024e4f743d",
|
||||
"sha256:c2a80fd9a8d7e41b4e38ea9fe149deed0d6aaede255c497e66b8213274d6d61b",
|
||||
"sha256:c44d3c82a933c6cbc21039326767e778eface44fca55c65719921c4b9661a3f7",
|
||||
"sha256:cc31e906be1cc121ee201adbdf844522ea3349600dd0a40366611ca18cd40e81",
|
||||
"sha256:d5d102e945ecca93bcd9801a7bb2fa703e37ad188a2f81b1e65e4abe4b51b00c",
|
||||
"sha256:dd7936f2a6daa861143e376b3a1fb56e9b802f4980923594edd9ca5670974895",
|
||||
"sha256:dee68ec462ff10c1d836c0ea2642116aba6151c6880b688e56b4c0246770f297",
|
||||
"sha256:e76e78863a4eaec3aee5722d85d04dcbd9844bc6cd3bfa6aa880ff46ad16bfcb",
|
||||
"sha256:eab51036cac2da8a50d7ff0ea30be47750547c9aa1aa2cf1a1b710a1827e7dbe",
|
||||
"sha256:f4496d8d04da2e98cc9133e238ccebf6a13ef39a93da2e87146c8c8ac9768242",
|
||||
"sha256:fbd3b5e18d34683decc00d9a360179ac1e7a320a5fee10ab8053ffd6deab76e0",
|
||||
"sha256:feb24ff1226beeb056e247cf2e24bba5232519efb5645121c4aea5b6ad74c1f2"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.7.4"
|
||||
},
|
||||
"async-timeout": {
|
||||
"hashes": [
|
||||
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
|
||||
"sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
|
||||
],
|
||||
"version": "==3.0.1"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
|
||||
"sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
|
||||
],
|
||||
"version": "==20.3.0"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
|
||||
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
|
||||
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.8.1"
|
||||
},
|
||||
"bgpranking": {
|
||||
"editable": true,
|
||||
"path": "."
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
|
||||
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
|
||||
],
|
||||
"version": "==2020.12.5"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||
],
|
||||
"version": "==3.0.4"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
"sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
|
||||
"sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
|
||||
],
|
||||
"version": "==7.1.2"
|
||||
},
|
||||
"dominate": {
|
||||
"hashes": [
|
||||
"sha256:76ec2cde23700a6fc4fee098168b9dee43b99c2f1dd0ca6a711f683e8eb7e1e4",
|
||||
"sha256:84b5f71ed30021193cb0faa45d7776e1083f392cfe67a49f44e98cb2ed76c036"
|
||||
],
|
||||
"version": "==2.6.0"
|
||||
},
|
||||
"flask": {
|
||||
"hashes": [
|
||||
"sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52",
|
||||
"sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.1.1"
|
||||
},
|
||||
"flask-bootstrap": {
|
||||
"hashes": [
|
||||
"sha256:cb08ed940183f6343a64e465e83b3a3f13c53e1baabb8d72b5da4545ef123ac8"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.3.7.1"
|
||||
},
|
||||
"gevent": {
|
||||
"hashes": [
|
||||
"sha256:16574e4aa902ebc7bad564e25aa9740a82620fdeb61e0bbf5cbc32e84c13cb6a",
|
||||
"sha256:188c3c6da67e17ffa28f960fc80f8b7e4ba0f4efdc7519822c9d3a1784ca78ea",
|
||||
"sha256:1e5af63e452cc1758924528a2ba6d3e472f5338e1534b7233cd01d3429fc1082",
|
||||
"sha256:242e32cc011ad7127525ca9181aef3379ce4ad9c733aefe311ecf90248ad9a6f",
|
||||
"sha256:2a9ae0a0fd956cbbc9c326b8f290dcad2b58acfb2e2732855fe1155fb110a04d",
|
||||
"sha256:33741e3cd51b90483b14f73b6a3b32b779acf965aeb91d22770c0c8e0c937b73",
|
||||
"sha256:3694f393ab08372bd337b9bc8eebef3ccab3c1623ef94536762a1eee68821449",
|
||||
"sha256:464ec84001ba5108a9022aded4c5e69ea4d13ef11a2386d3ec37c1d08f3074c9",
|
||||
"sha256:520cc2a029a9eef436e4e56b007af7859315cafa21937d43c1d5269f12f2c981",
|
||||
"sha256:77b65a68c83e1c680f52dc39d5e5406763dd10a18ce08420665504b6f047962e",
|
||||
"sha256:7bdfee07be5eee4f687bf90c54c2a65c909bcf2b6c4878faee51218ffa5d5d3e",
|
||||
"sha256:969743debf89d6409423aaeae978437cc042247f91f5801e946a07a0a3b59148",
|
||||
"sha256:96f704561a9dd9a817c67f2e279e23bfad6166cf95d63d35c501317e17f68bcf",
|
||||
"sha256:9f99c3ec61daed54dc074fbcf1a86bcf795b9dfac2f6d4cdae6dfdb8a9125692",
|
||||
"sha256:a130a1885603eabd8cea11b3e1c3c7333d4341b537eca7f0c4794cb5c7120db1",
|
||||
"sha256:a54b9c7516c211045d7897a73a4ccdc116b3720c9ad3c591ef9592b735202a3b",
|
||||
"sha256:ac98570649d9c276e39501a1d1cbf6c652b78f57a0eb1445c5ff25ff80336b63",
|
||||
"sha256:afaeda9a7e8e93d0d86bf1d65affe912366294913fe43f0d107145dc32cd9545",
|
||||
"sha256:b6ffc1131e017aafa70d7ec19cc24010b19daa2f11d5dc2dc191a79c3c9ea147",
|
||||
"sha256:ba0c6ad94614e9af4240affbe1b4839c54da5a0a7e60806c6f7f69c1a7f5426e",
|
||||
"sha256:bdb3677e77ab4ebf20c4752ac49f3b1e47445678dd69f82f9905362c68196456",
|
||||
"sha256:c2c4326bb507754ef354635c05f560a217c171d80f26ca65bea81aa59b1ac179",
|
||||
"sha256:cfb2878c2ecf27baea436bb9c4d8ab8c2fa7763c3916386d5602992b6a056ff3",
|
||||
"sha256:e370e0a861db6f63c75e74b6ee56a40f5cdac90212ec404621445afa12bfc94b",
|
||||
"sha256:e8a5d9fcf5d031f2e4c499f5f4b53262face416e22e8769078354f641255a663",
|
||||
"sha256:ecff28416c99e0f73137f35849c3027cc3edde9dc13b7707825ebbf728623928",
|
||||
"sha256:f0498df97a303da77e180a9368c9228b0fc94d10dd2ce79fc5ebb63fec0d2fc9",
|
||||
"sha256:f91fd07b9cf642f24e58ed381e19ec33e28b8eee8726c19b026ea24fcc9ff897"
|
||||
],
|
||||
"version": "==21.1.2"
|
||||
},
|
||||
"greenlet": {
|
||||
"hashes": [
|
||||
"sha256:0a77691f0080c9da8dfc81e23f4e3cffa5accf0f5b56478951016d7cfead9196",
|
||||
"sha256:0ddd77586553e3daf439aa88b6642c5f252f7ef79a39271c25b1d4bf1b7cbb85",
|
||||
"sha256:111cfd92d78f2af0bc7317452bd93a477128af6327332ebf3c2be7df99566683",
|
||||
"sha256:122c63ba795fdba4fc19c744df6277d9cfd913ed53d1a286f12189a0265316dd",
|
||||
"sha256:181300f826625b7fd1182205b830642926f52bd8cdb08b34574c9d5b2b1813f7",
|
||||
"sha256:1a1ada42a1fd2607d232ae11a7b3195735edaa49ea787a6d9e6a53afaf6f3476",
|
||||
"sha256:1bb80c71de788b36cefb0c3bb6bfab306ba75073dbde2829c858dc3ad70f867c",
|
||||
"sha256:1d1d4473ecb1c1d31ce8fd8d91e4da1b1f64d425c1dc965edc4ed2a63cfa67b2",
|
||||
"sha256:292e801fcb3a0b3a12d8c603c7cf340659ea27fd73c98683e75800d9fd8f704c",
|
||||
"sha256:2c65320774a8cd5fdb6e117c13afa91c4707548282464a18cf80243cf976b3e6",
|
||||
"sha256:4365eccd68e72564c776418c53ce3c5af402bc526fe0653722bc89efd85bf12d",
|
||||
"sha256:5352c15c1d91d22902582e891f27728d8dac3bd5e0ee565b6a9f575355e6d92f",
|
||||
"sha256:58ca0f078d1c135ecf1879d50711f925ee238fe773dfe44e206d7d126f5bc664",
|
||||
"sha256:5d4030b04061fdf4cbc446008e238e44936d77a04b2b32f804688ad64197953c",
|
||||
"sha256:5d69bbd9547d3bc49f8a545db7a0bd69f407badd2ff0f6e1a163680b5841d2b0",
|
||||
"sha256:5f297cb343114b33a13755032ecf7109b07b9a0020e841d1c3cedff6602cc139",
|
||||
"sha256:62afad6e5fd70f34d773ffcbb7c22657e1d46d7fd7c95a43361de979f0a45aef",
|
||||
"sha256:647ba1df86d025f5a34043451d7c4a9f05f240bee06277a524daad11f997d1e7",
|
||||
"sha256:719e169c79255816cdcf6dccd9ed2d089a72a9f6c42273aae12d55e8d35bdcf8",
|
||||
"sha256:7cd5a237f241f2764324396e06298b5dee0df580cf06ef4ada0ff9bff851286c",
|
||||
"sha256:875d4c60a6299f55df1c3bb870ebe6dcb7db28c165ab9ea6cdc5d5af36bb33ce",
|
||||
"sha256:90b6a25841488cf2cb1c8623a53e6879573010a669455046df5f029d93db51b7",
|
||||
"sha256:94620ed996a7632723a424bccb84b07e7b861ab7bb06a5aeb041c111dd723d36",
|
||||
"sha256:b5f1b333015d53d4b381745f5de842f19fe59728b65f0fbb662dafbe2018c3a5",
|
||||
"sha256:c5b22b31c947ad8b6964d4ed66776bcae986f73669ba50620162ba7c832a6b6a",
|
||||
"sha256:c93d1a71c3fe222308939b2e516c07f35a849c5047f0197442a4d6fbcb4128ee",
|
||||
"sha256:cdb90267650c1edb54459cdb51dab865f6c6594c3a47ebd441bc493360c7af70",
|
||||
"sha256:cfd06e0f0cc8db2a854137bd79154b61ecd940dce96fad0cba23fe31de0b793c",
|
||||
"sha256:d3789c1c394944084b5e57c192889985a9f23bd985f6d15728c745d380318128",
|
||||
"sha256:da7d09ad0f24270b20f77d56934e196e982af0d0a2446120cb772be4e060e1a2",
|
||||
"sha256:df3e83323268594fa9755480a442cabfe8d82b21aba815a71acf1bb6c1776218",
|
||||
"sha256:df8053867c831b2643b2c489fe1d62049a98566b1646b194cc815f13e27b90df",
|
||||
"sha256:e1128e022d8dce375362e063754e129750323b67454cac5600008aad9f54139e",
|
||||
"sha256:e6e9fdaf6c90d02b95e6b0709aeb1aba5affbbb9ccaea5502f8638e4323206be",
|
||||
"sha256:eac8803c9ad1817ce3d8d15d1bb82c2da3feda6bee1153eec5c58fa6e5d3f770",
|
||||
"sha256:eb333b90036358a0e2c57373f72e7648d7207b76ef0bd00a4f7daad1f79f5203",
|
||||
"sha256:ed1d1351f05e795a527abc04a0d82e9aecd3bdf9f46662c36ff47b0b00ecaf06",
|
||||
"sha256:f3dc68272990849132d6698f7dc6df2ab62a88b0d36e54702a8fd16c0490e44f",
|
||||
"sha256:f59eded163d9752fd49978e0bab7a1ff21b1b8d25c05f0995d140cc08ac83379",
|
||||
"sha256:f5e2d36c86c7b03c94b8459c3bd2c9fe2c7dab4b258b8885617d44a22e453fb7",
|
||||
"sha256:f6f65bf54215e4ebf6b01e4bb94c49180a589573df643735107056f7a910275b",
|
||||
"sha256:f8450d5ef759dbe59f84f2c9f77491bb3d3c44bc1a573746daf086e70b14c243",
|
||||
"sha256:f97d83049715fd9dec7911860ecf0e17b48d8725de01e45de07d8ac0bd5bc378"
|
||||
],
|
||||
"markers": "platform_python_implementation == 'CPython'",
|
||||
"version": "==1.0.0"
|
||||
},
|
||||
"gunicorn": {
|
||||
"extras": [
|
||||
"gevent"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471",
|
||||
"sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==19.9.0"
|
||||
},
|
||||
"hiredis": {
|
||||
"hashes": [
|
||||
"sha256:01b577f84c20ecc9c07fc4c184231b08e3c3942de096fa99978e053de231c423",
|
||||
"sha256:01ff0900134166961c9e339df77c33b72f7edc5cb41739f0babcd9faa345926e",
|
||||
"sha256:03ed34a13316d0c34213c4fd46e0fa3a5299073f4d4f08e93fed8c2108b399b3",
|
||||
"sha256:040436e91df5143aff9e0debb49530d0b17a6bd52200ce568621c31ef581b10d",
|
||||
"sha256:091eb38fbf968d1c5b703e412bbbd25f43a7967d8400842cee33a5a07b33c27b",
|
||||
"sha256:102f9b9dc6ed57feb3a7c9bdf7e71cb7c278fe8df1edfcfe896bc3e0c2be9447",
|
||||
"sha256:2b4b392c7e3082860c8371fab3ae762139090f9115819e12d9f56060f9ede05d",
|
||||
"sha256:2c9cc0b986397b833073f466e6b9e9c70d1d4dc2c2c1b3e9cae3a23102ff296c",
|
||||
"sha256:2fa65a9df683bca72073cd77709ddeb289ea2b114d3775d225fbbcc5faf808c5",
|
||||
"sha256:38437a681f17c975fd22349e72c29bc643f8e7eb2d6dc5df419eac59afa4d7ce",
|
||||
"sha256:3b3428fa3cf1ee178807b52c9bee8950ab94cd4eaa9bfae8c1bbae3c49501d34",
|
||||
"sha256:3dd8c2fae7f5494978facb0e93297dd627b1a3f536f3b070cf0a7d9157a07dcb",
|
||||
"sha256:4414a96c212e732723b5c3d7c04d386ebbb2ec359e1de646322cbc3f875cbd0d",
|
||||
"sha256:48c627581ad4ef60adbac980981407939acf13a0e18f093502c7b542223c4f19",
|
||||
"sha256:4a60e71625a2d78d8ab84dfb2fa2cfd9458c964b6e6c04fea76d9ade153fb371",
|
||||
"sha256:585ace09f434e43d8a8dbeb366865b1a044d7c06319b3c7372a0a00e63b860f4",
|
||||
"sha256:74b364b3f06c9cf0a53f7df611045bc9437ed972a283fa1f0b12537236d23ddc",
|
||||
"sha256:75c65c3850e89e9daa68d1b9bedd5806f177d60aa5a7b0953b4829481cfc1f72",
|
||||
"sha256:7f052de8bf744730a9120dbdc67bfeb7605a01f69fb8e7ba5c475af33c24e145",
|
||||
"sha256:8113a7d5e87ecf57cd4ae263cc9e429adb9a3e59f5a7768da5d3312a8d0a051a",
|
||||
"sha256:84857ce239eb8ed191ac78e77ff65d52902f00f30f4ee83bf80eb71da73b70e6",
|
||||
"sha256:8644a48ddc4a40b3e3a6b9443f396c2ee353afb2d45656c4fc68d04a82e8e3f7",
|
||||
"sha256:936aa565e673536e8a211e43ec43197406f24cd1f290138bd143765079c8ba00",
|
||||
"sha256:9afeb88c67bbc663b9f27385c496da056d06ad87f55df6e393e1516cfecb0461",
|
||||
"sha256:9d62cc7880110e4f83b0a51d218f465d3095e2751fbddd34e553dbd106a929ff",
|
||||
"sha256:a1fadd062fc8d647ff39220c57ea2b48c99bb73f18223828ec97f88fc27e7898",
|
||||
"sha256:a7754a783b1e5d6f627c19d099b178059c62f782ab62b4d8ba165b9fbc2ee34c",
|
||||
"sha256:aa59dd63bb3f736de4fc2d080114429d5d369dfb3265f771778e8349d67a97a4",
|
||||
"sha256:ae2ee0992f8de249715435942137843a93db204dd7db1e7cc9bdc5a8436443e8",
|
||||
"sha256:b36842d7cf32929d568f37ec5b3173b72b2ec6572dec4d6be6ce774762215aee",
|
||||
"sha256:bcbf9379c553b5facc6c04c1e5569b44b38ff16bcbf354676287698d61ee0c92",
|
||||
"sha256:cbccbda6f1c62ab460449d9c85fdf24d0d32a6bf45176581151e53cc26a5d910",
|
||||
"sha256:d0caf98dfb8af395d6732bd16561c0a2458851bea522e39f12f04802dbf6f502",
|
||||
"sha256:d6456afeddba036def1a36d8a2758eca53202308d83db20ab5d0b66590919627",
|
||||
"sha256:dbaef9a21a4f10bc281684ee4124f169e62bb533c2a92b55f8c06f64f9af7b8f",
|
||||
"sha256:dce84916c09aaece006272b37234ae84a8ed13abb3a4d341a23933b8701abfb5",
|
||||
"sha256:eb8c9c8b9869539d58d60ff4a28373a22514d40495911451343971cb4835b7a9",
|
||||
"sha256:efc98b14ee3a8595e40b1425e8d42f5fd26f11a7b215a81ef9259068931754f4",
|
||||
"sha256:fa2dc05b87d97acc1c6ae63f3e0f39eae5246565232484b08db6bf2dc1580678",
|
||||
"sha256:fe7d6ce9f6a5fbe24f09d95ea93e9c7271abc4e1565da511e1449b107b4d7848"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
|
||||
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
|
||||
],
|
||||
"version": "==2.8"
|
||||
},
|
||||
"idna-ssl": {
|
||||
"hashes": [
|
||||
"sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version < '3.7'",
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"itsdangerous": {
|
||||
"hashes": [
|
||||
"sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
|
||||
"sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"
|
||||
],
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"jinja2": {
|
||||
"hashes": [
|
||||
"sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419",
|
||||
"sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.11.3"
|
||||
},
|
||||
"markupsafe": {
|
||||
"hashes": [
|
||||
"sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
|
||||
"sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161",
|
||||
"sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
|
||||
"sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
|
||||
"sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",
|
||||
"sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f",
|
||||
"sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39",
|
||||
"sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
|
||||
"sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
|
||||
"sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014",
|
||||
"sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f",
|
||||
"sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
|
||||
"sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
|
||||
"sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
|
||||
"sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66",
|
||||
"sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b",
|
||||
"sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",
|
||||
"sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",
|
||||
"sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
|
||||
"sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85",
|
||||
"sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1",
|
||||
"sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
|
||||
"sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
|
||||
"sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",
|
||||
"sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850",
|
||||
"sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0",
|
||||
"sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",
|
||||
"sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d",
|
||||
"sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb",
|
||||
"sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e",
|
||||
"sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d",
|
||||
"sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c",
|
||||
"sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1",
|
||||
"sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2",
|
||||
"sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21",
|
||||
"sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2",
|
||||
"sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5",
|
||||
"sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7",
|
||||
"sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b",
|
||||
"sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8",
|
||||
"sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",
|
||||
"sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193",
|
||||
"sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",
|
||||
"sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b",
|
||||
"sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",
|
||||
"sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2",
|
||||
"sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5",
|
||||
"sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c",
|
||||
"sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032",
|
||||
"sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",
|
||||
"sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be",
|
||||
"sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621"
|
||||
],
|
||||
"version": "==1.1.1"
|
||||
},
|
||||
"multidict": {
|
||||
"hashes": [
|
||||
"sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a",
|
||||
"sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93",
|
||||
"sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632",
|
||||
"sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656",
|
||||
"sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79",
|
||||
"sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7",
|
||||
"sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d",
|
||||
"sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5",
|
||||
"sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224",
|
||||
"sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26",
|
||||
"sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea",
|
||||
"sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348",
|
||||
"sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6",
|
||||
"sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76",
|
||||
"sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1",
|
||||
"sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f",
|
||||
"sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952",
|
||||
"sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a",
|
||||
"sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37",
|
||||
"sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9",
|
||||
"sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359",
|
||||
"sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8",
|
||||
"sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da",
|
||||
"sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3",
|
||||
"sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d",
|
||||
"sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf",
|
||||
"sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841",
|
||||
"sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d",
|
||||
"sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93",
|
||||
"sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f",
|
||||
"sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647",
|
||||
"sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635",
|
||||
"sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456",
|
||||
"sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda",
|
||||
"sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5",
|
||||
"sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281",
|
||||
"sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80"
|
||||
],
|
||||
"version": "==5.1.0"
|
||||
},
|
||||
"pid": {
|
||||
"editable": true,
|
||||
"git": "https://github.com/trbs/pid.git/",
|
||||
"ref": "6a5d43a57149f6d70c61ad10f5622f88a3ae663c"
|
||||
},
|
||||
"pybgpranking": {
|
||||
"editable": true,
|
||||
"path": "./client"
|
||||
},
|
||||
"pycountry": {
|
||||
"hashes": [
|
||||
"sha256:3c57aa40adcf293d59bebaffbe60d8c39976fba78d846a018dc0c2ec9c6cb3cb"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==19.8.18"
|
||||
},
|
||||
"pyipasnhistory": {
|
||||
"editable": true,
|
||||
"git": "https://github.com/D4-project/IPASN-History.git/",
|
||||
"ref": "98ed95eeb33dff69b350bbab638da5634614d685",
|
||||
"subdirectory": "client"
|
||||
},
|
||||
"pymispgalaxies": {
|
||||
"editable": true,
|
||||
"git": "https://github.com/MISP/PyMISPGalaxies.git",
|
||||
"ref": "a43655ba85f00ba3a212a7f38e89380a2480adac"
|
||||
},
|
||||
"pytaxonomies": {
|
||||
"editable": true,
|
||||
"git": "https://github.com/MISP/PyTaxonomies.git",
|
||||
"ref": "01d18a50fd786d359df0a448200f10d64c06d175"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
|
||||
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.8.1"
|
||||
},
|
||||
"redis": {
|
||||
"hashes": [
|
||||
"sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62",
|
||||
"sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.3.11"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
|
||||
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.22.0"
|
||||
},
|
||||
"simplejson": {
|
||||
"hashes": [
|
||||
"sha256:067a7177ddfa32e1483ba5169ebea1bc2ea27f224853211ca669325648ca5642",
|
||||
"sha256:2fc546e6af49fb45b93bbe878dea4c48edc34083729c0abd09981fe55bdf7f91",
|
||||
"sha256:354fa32b02885e6dae925f1b5bbf842c333c1e11ea5453ddd67309dc31fdb40a",
|
||||
"sha256:37e685986cf6f8144607f90340cff72d36acf654f3653a6c47b84c5c38d00df7",
|
||||
"sha256:3af610ee72efbe644e19d5eaad575c73fb83026192114e5f6719f4901097fce2",
|
||||
"sha256:3b919fc9cf508f13b929a9b274c40786036b31ad28657819b3b9ba44ba651f50",
|
||||
"sha256:3dd289368bbd064974d9a5961101f080e939cbe051e6689a193c99fb6e9ac89b",
|
||||
"sha256:6c3258ffff58712818a233b9737fe4be943d306c40cf63d14ddc82ba563f483a",
|
||||
"sha256:75e3f0b12c28945c08f54350d91e624f8dd580ab74fd4f1bbea54bc6b0165610",
|
||||
"sha256:b1f329139ba647a9548aa05fb95d046b4a677643070dc2afc05fa2e975d09ca5",
|
||||
"sha256:ee9625fc8ee164902dfbb0ff932b26df112da9f871c32f0f9c1bcf20c350fe2a",
|
||||
"sha256:fb2530b53c28f0d4d84990e945c2ebb470edb469d63e389bf02ff409012fe7c5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.16.0"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
|
||||
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
|
||||
],
|
||||
"version": "==1.15.0"
|
||||
},
|
||||
"soupsieve": {
|
||||
"hashes": [
|
||||
"sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc",
|
||||
"sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"
|
||||
],
|
||||
"version": "==2.2.1"
|
||||
},
|
||||
"typing-extensions": {
|
||||
"hashes": [
|
||||
"sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2",
|
||||
"sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d",
|
||||
"sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version < '3.7'",
|
||||
"version": "==3.7.4.1"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
|
||||
"sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
|
||||
],
|
||||
"version": "==1.25.11"
|
||||
},
|
||||
"visitor": {
|
||||
"hashes": [
|
||||
"sha256:2c737903b2b6864ebc6167eef7cf3b997126f1aa94bdf590f90f1436d23e480a"
|
||||
],
|
||||
"version": "==0.1.3"
|
||||
},
|
||||
"werkzeug": {
|
||||
"hashes": [
|
||||
"sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7",
|
||||
"sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.16.0"
|
||||
},
|
||||
"yarl": {
|
||||
"hashes": [
|
||||
"sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e",
|
||||
"sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434",
|
||||
"sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366",
|
||||
"sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3",
|
||||
"sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec",
|
||||
"sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959",
|
||||
"sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e",
|
||||
"sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c",
|
||||
"sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6",
|
||||
"sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a",
|
||||
"sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6",
|
||||
"sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424",
|
||||
"sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e",
|
||||
"sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f",
|
||||
"sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50",
|
||||
"sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2",
|
||||
"sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc",
|
||||
"sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4",
|
||||
"sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970",
|
||||
"sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10",
|
||||
"sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0",
|
||||
"sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406",
|
||||
"sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896",
|
||||
"sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643",
|
||||
"sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721",
|
||||
"sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478",
|
||||
"sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724",
|
||||
"sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e",
|
||||
"sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8",
|
||||
"sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96",
|
||||
"sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25",
|
||||
"sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76",
|
||||
"sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2",
|
||||
"sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2",
|
||||
"sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c",
|
||||
"sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a",
|
||||
"sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71"
|
||||
],
|
||||
"version": "==1.6.3"
|
||||
},
|
||||
"zope.event": {
|
||||
"hashes": [
|
||||
"sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42",
|
||||
"sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330"
|
||||
],
|
||||
"version": "==4.5.0"
|
||||
},
|
||||
"zope.interface": {
|
||||
"hashes": [
|
||||
"sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
|
||||
"sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d",
|
||||
"sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123",
|
||||
"sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232",
|
||||
"sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549",
|
||||
"sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102",
|
||||
"sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5",
|
||||
"sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45",
|
||||
"sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00",
|
||||
"sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc",
|
||||
"sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7",
|
||||
"sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104",
|
||||
"sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034",
|
||||
"sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3",
|
||||
"sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3",
|
||||
"sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4",
|
||||
"sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86",
|
||||
"sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96",
|
||||
"sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546",
|
||||
"sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb",
|
||||
"sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3",
|
||||
"sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b",
|
||||
"sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b",
|
||||
"sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec",
|
||||
"sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae",
|
||||
"sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e",
|
||||
"sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386",
|
||||
"sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2",
|
||||
"sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a",
|
||||
"sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d",
|
||||
"sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a",
|
||||
"sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24",
|
||||
"sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d",
|
||||
"sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b",
|
||||
"sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50",
|
||||
"sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523",
|
||||
"sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a",
|
||||
"sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095",
|
||||
"sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a",
|
||||
"sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520",
|
||||
"sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65",
|
||||
"sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11",
|
||||
"sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c",
|
||||
"sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7",
|
||||
"sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332",
|
||||
"sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e",
|
||||
"sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c",
|
||||
"sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7",
|
||||
"sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20",
|
||||
"sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc",
|
||||
"sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd",
|
||||
"sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537"
|
||||
],
|
||||
"version": "==5.2.0"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
import logging
|
||||
|
||||
from .libs.helpers import long_sleep, shutdown_requested
|
||||
|
||||
|
||||
class AbstractManager(ABC):
|
||||
|
||||
def __init__(self, loglevel: int=logging.DEBUG):
|
||||
self.loglevel = loglevel
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||
self.logger.setLevel(loglevel)
|
||||
self.logger.info(f'Initializing {self.__class__.__name__}')
|
||||
|
||||
@abstractmethod
|
||||
def _to_run_forever(self):
|
||||
pass
|
||||
|
||||
def run(self, sleep_in_sec: int):
|
||||
self.logger.info(f'Launching {self.__class__.__name__}')
|
||||
while True:
|
||||
if shutdown_requested():
|
||||
break
|
||||
try:
|
||||
self._to_run_forever()
|
||||
except Exception:
|
||||
self.logger.exception(f'Something went wrong in {self.__class__.__name__}.')
|
||||
if not long_sleep(sleep_in_sec):
|
||||
break
|
||||
self.logger.info(f'Shutting down {self.__class__.__name__}')
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from dateutil import parser
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from collections import defaultdict
|
||||
import zipfile
|
||||
import logging
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
from .libs.helpers import safe_create_dir, set_running, unset_running
|
||||
|
||||
|
||||
class DeepArchive():
|
||||
|
||||
def __init__(self, config_file: Path, storage_directory: Path,
|
||||
loglevel: int=logging.DEBUG):
|
||||
'''Archive everyfile older than 2 month.'''
|
||||
with open(config_file, 'r') as f:
|
||||
module_parameters = json.load(f)
|
||||
self.vendor = module_parameters['vendor']
|
||||
self.listname = module_parameters['name']
|
||||
self.directory = storage_directory / self.vendor / self.listname / 'archive'
|
||||
safe_create_dir(self.directory)
|
||||
self.deep_archive = self.directory / 'deep'
|
||||
safe_create_dir(self.deep_archive)
|
||||
self.__init_logger(loglevel)
|
||||
|
||||
def __init_logger(self, loglevel):
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}-{self.vendor}-{self.listname}')
|
||||
self.logger.setLevel(loglevel)
|
||||
|
||||
def archive(self):
|
||||
set_running(self.__class__.__name__)
|
||||
|
||||
to_archive = defaultdict(list)
|
||||
today = date.today()
|
||||
last_day_to_keep = date(today.year, today.month, 1) - relativedelta(months=2)
|
||||
for p in self.directory.iterdir():
|
||||
if not p.is_file():
|
||||
continue
|
||||
filedate = parser.parse(p.name.split('.')[0]).date()
|
||||
if filedate >= last_day_to_keep:
|
||||
continue
|
||||
to_archive['{}.zip'.format(filedate.strftime('%Y%m'))].append(p)
|
||||
if to_archive:
|
||||
self.logger.info('Found old files. Archiving: {}'.format(', '.join(to_archive.keys())))
|
||||
else:
|
||||
self.logger.debug('No old files.')
|
||||
for archivename, path_list in to_archive.items():
|
||||
with zipfile.ZipFile(self.deep_archive / archivename, 'x', zipfile.ZIP_DEFLATED) as z:
|
||||
for f in path_list:
|
||||
z.write(f, f.name)
|
||||
# Delete all the files if the archiving worked out properly
|
||||
[f.unlink() for f in path_list]
|
||||
unset_running(self.__class__.__name__)
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import requests
|
||||
from redis import StrictRedis
|
||||
from dateutil.parser import parse
|
||||
import re
|
||||
|
||||
from .libs.helpers import set_running, unset_running, get_socket_path, safe_create_dir
|
||||
|
||||
|
||||
class ASNDescriptions():
|
||||
|
||||
def __init__(self, storage_directory: Path, loglevel: int=logging.DEBUG):
|
||||
self.__init_logger(loglevel)
|
||||
self.asn_meta = StrictRedis(unix_socket_path=get_socket_path('storage'), db=2, decode_responses=True)
|
||||
self.logger.debug('Starting ASN History')
|
||||
self.directory = storage_directory / 'asn_descriptions'
|
||||
safe_create_dir(self.directory)
|
||||
self.archives = self.directory / 'archive'
|
||||
safe_create_dir(self.archives)
|
||||
self.url = 'https://www.cidr-report.org/as2.0/autnums.html'
|
||||
|
||||
def __init_logger(self, loglevel):
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||
self.logger.setLevel(loglevel)
|
||||
|
||||
def __update_available(self):
|
||||
r = requests.head(self.url)
|
||||
current_last_modified = parse(r.headers['Last-Modified'])
|
||||
if not self.asn_meta.exists('ans_description_last_update'):
|
||||
return True
|
||||
last_update = parse(self.asn_meta.get('ans_description_last_update'))
|
||||
if last_update < current_last_modified:
|
||||
return True
|
||||
return False
|
||||
|
||||
def load_descriptions(self):
|
||||
if not self.__update_available():
|
||||
self.logger.debug('No new file to import.')
|
||||
return
|
||||
set_running(self.__class__.__name__)
|
||||
self.logger.info('Importing new ASN descriptions.')
|
||||
r = requests.get(self.url)
|
||||
last_modified = parse(r.headers['Last-Modified']).isoformat()
|
||||
p = self.asn_meta.pipeline()
|
||||
new_asn = 0
|
||||
new_description = 0
|
||||
for asn, descr in re.findall('as=AS(.*)&.*</a> (.*)\n', r.text):
|
||||
existing_descriptions = self.asn_meta.hgetall(f'{asn}|descriptions')
|
||||
if not existing_descriptions:
|
||||
self.logger.debug(f'New ASN: {asn} - {descr}')
|
||||
p.hset(f'{asn}|descriptions', last_modified, descr)
|
||||
new_asn += 1
|
||||
else:
|
||||
last_descr = sorted(existing_descriptions.keys(), reverse=True)[0]
|
||||
if descr != existing_descriptions[last_descr]:
|
||||
self.logger.debug(f'New description for {asn}: {existing_descriptions[last_descr]} -> {descr}')
|
||||
p.hset(f'{asn}|descriptions', last_modified, descr)
|
||||
new_description += 1
|
||||
p.set('ans_description_last_update', last_modified)
|
||||
p.execute()
|
||||
self.logger.info(f'Done with import. New ASNs: {new_asn}, new descriptions: {new_description}')
|
||||
if new_asn or new_description:
|
||||
with open(self.archives / f'{last_modified}.html', 'w') as f:
|
||||
f.write(r.text)
|
||||
unset_running(self.__class__.__name__)
|
||||
|
|
@ -1,37 +1,52 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from typing import TypeVar, Union
|
||||
import logging
|
||||
import re
|
||||
|
||||
from redis import ConnectionPool, Redis
|
||||
from redis.connection import UnixDomainSocketConnection
|
||||
|
||||
from .default import get_config, get_socket_path
|
||||
|
||||
from typing import TypeVar, Union, Optional, Dict, Any, List, Tuple
|
||||
import datetime
|
||||
from datetime import timedelta
|
||||
from dateutil.parser import parse
|
||||
from collections import defaultdict
|
||||
|
||||
import logging
|
||||
import json
|
||||
from redis import StrictRedis
|
||||
|
||||
from .libs.helpers import get_socket_path, get_config_path
|
||||
from .libs.exceptions import InvalidDateFormat
|
||||
from .libs.statsripe import StatsRIPE
|
||||
from .default import InvalidDateFormat
|
||||
from .helpers import get_modules
|
||||
from .statsripe import StatsRIPE
|
||||
|
||||
Dates = TypeVar('Dates', datetime.datetime, datetime.date, str)
|
||||
|
||||
|
||||
class Querying():
|
||||
class BGPRanking():
|
||||
|
||||
def __init__(self, loglevel: int=logging.DEBUG):
|
||||
self.__init_logger(loglevel)
|
||||
self.storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
|
||||
self.ranking = StrictRedis(unix_socket_path=get_socket_path('storage'), db=1)
|
||||
self.asn_meta = StrictRedis(unix_socket_path=get_socket_path('storage'), db=2, decode_responses=True)
|
||||
self.cache = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
|
||||
def __init_logger(self, loglevel: int):
|
||||
def __init__(self) -> None:
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||
self.logger.setLevel(loglevel)
|
||||
self.logger.setLevel(get_config('generic', 'loglevel'))
|
||||
|
||||
def __normalize_date(self, date: Dates):
|
||||
self.cache_pool: ConnectionPool = ConnectionPool(connection_class=UnixDomainSocketConnection,
|
||||
path=get_socket_path('cache'), decode_responses=True)
|
||||
|
||||
self.storage = Redis(get_config('generic', 'storage_db_hostname'), get_config('generic', 'storage_db_port'), decode_responses=True)
|
||||
self.asn_meta = Redis(get_config('generic', 'storage_db_hostname'), get_config('generic', 'storage_db_port'), decode_responses=True)
|
||||
self.ranking = Redis(get_config('generic', 'ranking_db_hostname'), get_config('generic', 'ranking_db_port'))
|
||||
|
||||
@property
|
||||
def cache(self):
|
||||
return Redis(connection_pool=self.cache_pool, db=1)
|
||||
|
||||
def check_redis_up(self) -> bool:
|
||||
return self.cache.ping()
|
||||
|
||||
def __normalize_date(self, date: Optional[Dates]) -> str:
|
||||
if not date:
|
||||
return datetime.date.today().isoformat()
|
||||
if isinstance(date, datetime.datetime):
|
||||
return date.date().isoformat()
|
||||
elif isinstance(date, datetime.date):
|
||||
|
|
@ -45,16 +60,19 @@ class Querying():
|
|||
def _ranking_cache_wrapper(self, key):
|
||||
if not self.cache.exists(key):
|
||||
if self.ranking.exists(key):
|
||||
key_dump = self.ranking.dump(key)
|
||||
content: List[Tuple[bytes, float]] = self.ranking.zrangebyscore(key, '-Inf', '+Inf', withscores=True)
|
||||
# Cache for 10 hours
|
||||
self.cache.restore(key, 36000, key_dump, True)
|
||||
self.cache.zadd(key, {value: rank for value, rank in content})
|
||||
self.cache.expire(key, 36000)
|
||||
|
||||
def asns_global_ranking(self, date: Dates=datetime.date.today(), source: Union[list, str]='',
|
||||
def asns_global_ranking(self, date: Optional[Dates]=None, source: Union[list, str]='',
|
||||
ipversion: str='v4', limit: int=100):
|
||||
'''Aggregated ranking of all the ASNs known in the system, weighted by source.'''
|
||||
to_return = {'meta': {'ipversion': ipversion, 'limit': limit}, 'source': source,
|
||||
'response': set()}
|
||||
|
||||
to_return: Dict[str, Any] = {
|
||||
'meta': {'ipversion': ipversion, 'limit': limit},
|
||||
'source': source,
|
||||
'response': set()
|
||||
}
|
||||
d = self.__normalize_date(date)
|
||||
to_return['meta']['date'] = d
|
||||
if source:
|
||||
|
|
@ -76,11 +94,13 @@ class Querying():
|
|||
to_return['response'] = self.cache.zrevrange(key, start=0, end=limit, withscores=True)
|
||||
return to_return
|
||||
|
||||
def asn_details(self, asn: int, date: Dates= datetime.date.today(), source: Union[list, str]='',
|
||||
def asn_details(self, asn: int, date: Optional[Dates]=None, source: Union[list, str]='',
|
||||
ipversion: str='v4'):
|
||||
'''Aggregated ranking of all the prefixes anounced by the given ASN, weighted by source.'''
|
||||
to_return = {'meta': {'asn': asn, 'ipversion': ipversion, 'source': source},
|
||||
'response': set()}
|
||||
to_return: Dict[str, Any] = {
|
||||
'meta': {'asn': asn, 'ipversion': ipversion, 'source': source},
|
||||
'response': set()
|
||||
}
|
||||
|
||||
d = self.__normalize_date(date)
|
||||
to_return['meta']['date'] = d
|
||||
|
|
@ -103,12 +123,14 @@ class Querying():
|
|||
to_return['response'] = self.cache.zrevrange(key, start=0, end=-1, withscores=True)
|
||||
return to_return
|
||||
|
||||
def asn_rank(self, asn: int, date: Dates=datetime.date.today(), source: Union[list, str]='',
|
||||
def asn_rank(self, asn: int, date: Optional[Dates]=None, source: Union[list, str]='',
|
||||
ipversion: str='v4', with_position: bool=False):
|
||||
'''Get the rank of a single ASN, weighted by source.'''
|
||||
to_return = {'meta': {'asn': asn, 'ipversion': ipversion,
|
||||
'source': source, 'with_position': with_position},
|
||||
'response': 0.0}
|
||||
to_return: Dict[str, Any] = {
|
||||
'meta': {'asn': asn, 'ipversion': ipversion,
|
||||
'source': source, 'with_position': with_position},
|
||||
'response': 0.0
|
||||
}
|
||||
|
||||
d = self.__normalize_date(date)
|
||||
to_return['meta']['date'] = d
|
||||
|
|
@ -141,9 +163,9 @@ class Querying():
|
|||
to_return['response'] = float(r)
|
||||
return to_return
|
||||
|
||||
def get_sources(self, date: Dates=datetime.date.today()):
|
||||
def get_sources(self, date: Optional[Dates]=None):
|
||||
'''Get the sources availables for a specific day (default: today).'''
|
||||
to_return = {'meta': {}, 'response': set()}
|
||||
to_return: Dict[str, Any] = {'meta': {}, 'response': set()}
|
||||
|
||||
d = self.__normalize_date(date)
|
||||
to_return['meta']['date'] = d
|
||||
|
|
@ -151,9 +173,11 @@ class Querying():
|
|||
to_return['response'] = self.storage.smembers(key)
|
||||
return to_return
|
||||
|
||||
def get_asn_descriptions(self, asn: int, all_descriptions=False):
|
||||
to_return = {'meta': {'asn': asn, 'all_descriptions': all_descriptions},
|
||||
'response': []}
|
||||
def get_asn_descriptions(self, asn: int, all_descriptions=False) -> Dict[str, Any]:
|
||||
to_return: Dict[str, Union[Dict, List, str]] = {
|
||||
'meta': {'asn': asn, 'all_descriptions': all_descriptions},
|
||||
'response': []
|
||||
}
|
||||
descriptions = self.asn_meta.hgetall(f'{asn}|descriptions')
|
||||
if all_descriptions or not descriptions:
|
||||
to_return['response'] = descriptions
|
||||
|
|
@ -161,11 +185,13 @@ class Querying():
|
|||
to_return['response'] = descriptions[sorted(descriptions.keys(), reverse=True)[0]]
|
||||
return to_return
|
||||
|
||||
def get_prefix_ips(self, asn: int, prefix: str, date: Dates=datetime.date.today(),
|
||||
def get_prefix_ips(self, asn: int, prefix: str, date: Optional[Dates]=None,
|
||||
source: Union[list, str]='', ipversion: str='v4'):
|
||||
to_return = {'meta': {'asn': asn, 'prefix': prefix, 'ipversion': ipversion,
|
||||
'source': source},
|
||||
'response': defaultdict(list)}
|
||||
to_return: Dict[str, Any] = {
|
||||
'meta': {'asn': asn, 'prefix': prefix, 'ipversion': ipversion,
|
||||
'source': source},
|
||||
'response': defaultdict(list)
|
||||
}
|
||||
|
||||
d = self.__normalize_date(date)
|
||||
to_return['meta']['date'] = d
|
||||
|
|
@ -186,33 +212,37 @@ class Querying():
|
|||
return to_return
|
||||
|
||||
def get_asn_history(self, asn: int, period: int=100, source: Union[list, str]='',
|
||||
ipversion: str='v4', date: Dates=datetime.date.today()):
|
||||
to_return = {'meta': {'asn': asn, 'period': period, 'ipversion': ipversion,
|
||||
'source': source},
|
||||
'response': []}
|
||||
ipversion: str='v4', date: Optional[Dates]=None):
|
||||
to_return: Dict[str, Any] = {
|
||||
'meta': {'asn': asn, 'period': period, 'ipversion': ipversion, 'source': source},
|
||||
'response': []
|
||||
}
|
||||
|
||||
if isinstance(date, str):
|
||||
date = parse(date).date()
|
||||
if date + timedelta(days=period / 3) > datetime.date.today():
|
||||
# the period to display will be around the date passed at least 2/3 before the date, at most 1/3 after
|
||||
# FIXME: That is not doing what it is supposed to...
|
||||
date = datetime.date.today()
|
||||
if date is None:
|
||||
python_date: datetime.date = datetime.date.today()
|
||||
elif isinstance(date, str):
|
||||
python_date = parse(date).date()
|
||||
elif isinstance(date, datetime.datetime):
|
||||
python_date = date.date()
|
||||
else:
|
||||
python_date = date
|
||||
|
||||
to_return['meta']['date'] = date.isoformat()
|
||||
to_return['meta']['date'] = python_date.isoformat()
|
||||
|
||||
for i in range(period):
|
||||
d = date - timedelta(days=i)
|
||||
d = python_date - timedelta(days=i)
|
||||
rank = self.asn_rank(asn, d, source, ipversion)
|
||||
if 'response' not in rank:
|
||||
rank = 0
|
||||
to_return['response'].insert(0, (d.isoformat(), rank['response']))
|
||||
return to_return
|
||||
|
||||
def country_rank(self, country: str, date: Dates=datetime.date.today(), source: Union[list, str]='',
|
||||
def country_rank(self, country: str, date: Optional[Dates]=None, source: Union[list, str]='',
|
||||
ipversion: str='v4'):
|
||||
to_return = {'meta': {'country': country, 'ipversion': ipversion,
|
||||
'source': source},
|
||||
'response': []}
|
||||
to_return: Dict[str, Any] = {
|
||||
'meta': {'country': country, 'ipversion': ipversion, 'source': source},
|
||||
'response': []
|
||||
}
|
||||
|
||||
d = self.__normalize_date(date)
|
||||
to_return['meta']['date'] = d
|
||||
|
|
@ -224,29 +254,31 @@ class Querying():
|
|||
logging.warning(f'Invalid response: {response}')
|
||||
# FIXME: return something
|
||||
return 0, [(0, 0)]
|
||||
routed_asns = response['data']['countries'][0]['routed']
|
||||
routed_asns = re.findall(r"AsnSingle\(([\d]*)\)", response['data']['countries'][0]['routed'])
|
||||
ranks = [self.asn_rank(asn, d, source, ipversion)['response'] for asn in routed_asns]
|
||||
to_return['response'] = [sum(ranks), zip(routed_asns, ranks)]
|
||||
return to_return
|
||||
|
||||
def country_history(self, country: Union[list, str], period: int=30, source: Union[list, str]='',
|
||||
ipversion: str='v4', date: Dates=datetime.date.today()):
|
||||
to_return = {}
|
||||
to_return = {'meta': {'country': country, 'ipversion': ipversion,
|
||||
'source': source},
|
||||
'response': defaultdict(list)}
|
||||
|
||||
if isinstance(date, str):
|
||||
date = parse(date).date()
|
||||
if date + timedelta(days=period / 3) > datetime.date.today():
|
||||
# the period to display will be around the date passed at least 2/3 before the date, at most 1/3 after
|
||||
date = datetime.date.today()
|
||||
ipversion: str='v4', date: Optional[Dates]=None):
|
||||
to_return: Dict[str, Any] = {
|
||||
'meta': {'country': country, 'ipversion': ipversion, 'source': source},
|
||||
'response': defaultdict(list)
|
||||
}
|
||||
if date is None:
|
||||
python_date: datetime.date = datetime.date.today()
|
||||
elif isinstance(date, str):
|
||||
python_date = parse(date).date()
|
||||
elif isinstance(date, datetime.datetime):
|
||||
python_date = date.date()
|
||||
else:
|
||||
python_date = date
|
||||
|
||||
if isinstance(country, str):
|
||||
country = [country]
|
||||
for c in country:
|
||||
for i in range(period):
|
||||
d = date - timedelta(days=i)
|
||||
d = python_date - timedelta(days=i)
|
||||
rank, details = self.country_rank(c, d, source, ipversion)['response']
|
||||
if rank is None:
|
||||
rank = 0
|
||||
|
|
@ -257,9 +289,8 @@ class Querying():
|
|||
pass
|
||||
|
||||
def get_sources_configs(self):
|
||||
config_dir = get_config_path() / 'modules'
|
||||
loaded = []
|
||||
for modulepath in config_dir.glob('*.json'):
|
||||
for modulepath in get_modules():
|
||||
with open(modulepath) as f:
|
||||
loaded.append(json.load(f))
|
||||
return {'{}-{}'.format(config['vendor'], config['name']): config for config in loaded}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
{
|
||||
"ipasnhistory_url": "https://ipasnhistory.circl.lu/"
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
{
|
||||
"url": "http://www.nothink.org/blacklist/blacklist_snmp_day.txt",
|
||||
"vendor": "nothink",
|
||||
"name": "snmp",
|
||||
"impact": 5,
|
||||
"parser": ".parsers.nothink"
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
{
|
||||
"url": "http://www.nothink.org/blacklist/blacklist_ssh_day.txt",
|
||||
"vendor": "nothink",
|
||||
"name": "ssh",
|
||||
"impact": 5,
|
||||
"parser": ".parsers.nothink"
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
{
|
||||
"url": "http://www.nothink.org/blacklist/blacklist_telnet_day.txt",
|
||||
"vendor": "nothink",
|
||||
"name": "telnet",
|
||||
"impact": 5,
|
||||
"parser": ".parsers.nothink"
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
{
|
||||
"url": "https://palevotracker.abuse.ch/blocklists.php?download=ipblocklist",
|
||||
"vendor": "palevotracker",
|
||||
"name": "ipblocklist",
|
||||
"impact": 5
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
{
|
||||
"url": "https://www.openbl.org/lists/base.txt",
|
||||
"vendor": "sshbl",
|
||||
"name": "base",
|
||||
"impact": 5
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
{
|
||||
"url": "https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist",
|
||||
"vendor": "zeustracker",
|
||||
"name": "ipblocklist",
|
||||
"impact": 5
|
||||
}
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import time
|
||||
from redis import StrictRedis
|
||||
from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path, get_ipasn, sanity_check_ipasn
|
||||
|
||||
|
||||
class DatabaseInsert():
|
||||
|
||||
def __init__(self, loglevel: int=logging.DEBUG):
|
||||
self.__init_logger(loglevel)
|
||||
self.ardb_storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
|
||||
self.redis_sanitized = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
|
||||
self.ipasn = get_ipasn()
|
||||
self.logger.debug('Starting import')
|
||||
|
||||
def __init_logger(self, loglevel):
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||
self.logger.setLevel(loglevel)
|
||||
|
||||
def insert(self):
|
||||
ready, message = sanity_check_ipasn(self.ipasn)
|
||||
if not ready:
|
||||
# Try again later.
|
||||
self.logger.warning(message)
|
||||
return
|
||||
self.logger.debug(message)
|
||||
|
||||
set_running(self.__class__.__name__)
|
||||
while True:
|
||||
if shutdown_requested():
|
||||
break
|
||||
try:
|
||||
if not self.ipasn.is_up:
|
||||
break
|
||||
except Exception:
|
||||
self.logger.warning('Unable to query ipasnhistory')
|
||||
time.sleep(10)
|
||||
continue
|
||||
uuids = self.redis_sanitized.spop('to_insert', 100)
|
||||
if not uuids:
|
||||
break
|
||||
p = self.redis_sanitized.pipeline(transaction=False)
|
||||
[p.hgetall(uuid) for uuid in uuids]
|
||||
sanitized_data = p.execute()
|
||||
|
||||
for_query = []
|
||||
for i, uuid in enumerate(uuids):
|
||||
data = sanitized_data[i]
|
||||
if not data:
|
||||
self.logger.warning(f'No data for UUID {uuid}. This should not happen, but lets move on.')
|
||||
continue
|
||||
for_query.append({'ip': data['ip'], 'address_family': data['address_family'], 'source': 'caida',
|
||||
'date': data['datetime'], 'precision_delta': {'days': 3}})
|
||||
try:
|
||||
responses = self.ipasn.mass_query(for_query)
|
||||
except Exception:
|
||||
self.logger.exception('Mass query in IPASN History failed, trying again later.')
|
||||
# Rollback the spop
|
||||
self.redis_sanitized.sadd('to_insert', *uuids)
|
||||
time.sleep(10)
|
||||
continue
|
||||
retry = []
|
||||
done = []
|
||||
ardb_pipeline = self.ardb_storage.pipeline(transaction=False)
|
||||
for i, uuid in enumerate(uuids):
|
||||
data = sanitized_data[i]
|
||||
if not data:
|
||||
self.logger.warning(f'No data for UUID {uuid}. This should not happen, but lets move on.')
|
||||
continue
|
||||
routing_info = responses['responses'][i]['response'] # our queries are on one single date, not a range
|
||||
# Data gathered from IPASN History:
|
||||
# * IP Block of the IP
|
||||
# * AS number
|
||||
if 'error' in routing_info:
|
||||
self.logger.warning(f"Unable to find routing information for {data['ip']} - {data['datetime']}: {routing_info['error']}")
|
||||
continue
|
||||
# Single date query, getting from the object
|
||||
datetime_routing = list(routing_info.keys())[0]
|
||||
entry = routing_info[datetime_routing]
|
||||
if not entry:
|
||||
# routing info is missing, need to try again later.
|
||||
retry.append(uuid)
|
||||
continue
|
||||
if 'asn' in entry and entry['asn'] is None:
|
||||
self.logger.warning(f"Unable to find the AS number associated to {data['ip']} - {data['datetime']} (got None). This should not happen...")
|
||||
continue
|
||||
if 'prefix' in entry and entry['prefix'] is None:
|
||||
self.logger.warning(f"Unable to find the prefix associated to {data['ip']} - {data['datetime']} (got None). This should not happen...")
|
||||
continue
|
||||
|
||||
# Format: <YYYY-MM-DD>|sources -> set([<source>, ...])
|
||||
ardb_pipeline.sadd(f"{data['date']}|sources", data['source'])
|
||||
|
||||
# Format: <YYYY-MM-DD>|<source> -> set([<asn>, ...])
|
||||
ardb_pipeline.sadd(f"{data['date']}|{data['source']}", entry['asn'])
|
||||
# Format: <YYYY-MM-DD>|<source>|<asn> -> set([<prefix>, ...])
|
||||
ardb_pipeline.sadd(f"{data['date']}|{data['source']}|{entry['asn']}", entry['prefix'])
|
||||
|
||||
# Format: <YYYY-MM-DD>|<source>|<asn>|<prefix> -> set([<ip>|<datetime>, ...])
|
||||
ardb_pipeline.sadd(f"{data['date']}|{data['source']}|{entry['asn']}|{entry['prefix']}",
|
||||
f"{data['ip']}|{data['datetime']}")
|
||||
done.append(uuid)
|
||||
ardb_pipeline.execute()
|
||||
p = self.redis_sanitized.pipeline(transaction=False)
|
||||
if done:
|
||||
p.delete(*done)
|
||||
if retry:
|
||||
p.sadd('to_insert', *retry)
|
||||
p.execute()
|
||||
unset_running(self.__class__.__name__)
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
env_global_name: str = 'BGPRANKING_HOME'
|
||||
|
||||
from .exceptions import (BGPRankingException, FetcherException, ArchiveException, # noqa
|
||||
CreateDirectoryException, MissingEnv, InvalidDateFormat, # noqa
|
||||
MissingConfigFile, MissingConfigEntry, ThirdPartyUnreachable) # noqa
|
||||
|
||||
# NOTE: the imports below are there to avoid too long paths when importing the
|
||||
# classes/methods in the rest of the project while keeping all that in a subdirectory
|
||||
# and allow to update them easily.
|
||||
# You should not have to change anything in this file below this line.
|
||||
|
||||
from .abstractmanager import AbstractManager # noqa
|
||||
|
||||
from .exceptions import MissingEnv, CreateDirectoryException, ConfigError # noqa
|
||||
|
||||
from .helpers import get_homedir, load_configs, get_config, safe_create_dir, get_socket_path, try_make_file # noqa
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
from abc import ABC
|
||||
from datetime import datetime, timedelta
|
||||
from subprocess import Popen
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from redis import Redis
|
||||
from redis.exceptions import ConnectionError
|
||||
|
||||
from .helpers import get_socket_path
|
||||
|
||||
|
||||
class AbstractManager(ABC):
|
||||
|
||||
script_name: str
|
||||
|
||||
def __init__(self, loglevel: int=logging.DEBUG):
|
||||
self.loglevel = loglevel
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||
self.logger.setLevel(loglevel)
|
||||
self.logger.info(f'Initializing {self.__class__.__name__}')
|
||||
self.process: Optional[Popen] = None
|
||||
self.__redis = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
|
||||
@staticmethod
|
||||
def is_running() -> List[Tuple[str, float]]:
|
||||
try:
|
||||
r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
return r.zrangebyscore('running', '-inf', '+inf', withscores=True)
|
||||
except ConnectionError:
|
||||
print('Unable to connect to redis, the system is down.')
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def force_shutdown():
|
||||
try:
|
||||
r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
r.set('shutdown', 1)
|
||||
except ConnectionError:
|
||||
print('Unable to connect to redis, the system is down.')
|
||||
|
||||
def set_running(self) -> None:
|
||||
self.__redis.zincrby('running', 1, self.script_name)
|
||||
|
||||
def unset_running(self) -> None:
|
||||
current_running = self.__redis.zincrby('running', -1, self.script_name)
|
||||
if int(current_running) <= 0:
|
||||
self.__redis.zrem('running', self.script_name)
|
||||
|
||||
def long_sleep(self, sleep_in_sec: int, shutdown_check: int=10) -> bool:
|
||||
if shutdown_check > sleep_in_sec:
|
||||
shutdown_check = sleep_in_sec
|
||||
sleep_until = datetime.now() + timedelta(seconds=sleep_in_sec)
|
||||
while sleep_until > datetime.now():
|
||||
time.sleep(shutdown_check)
|
||||
if self.shutdown_requested():
|
||||
return False
|
||||
return True
|
||||
|
||||
async def long_sleep_async(self, sleep_in_sec: int, shutdown_check: int=10) -> bool:
|
||||
if shutdown_check > sleep_in_sec:
|
||||
shutdown_check = sleep_in_sec
|
||||
sleep_until = datetime.now() + timedelta(seconds=sleep_in_sec)
|
||||
while sleep_until > datetime.now():
|
||||
await asyncio.sleep(shutdown_check)
|
||||
if self.shutdown_requested():
|
||||
return False
|
||||
return True
|
||||
|
||||
def shutdown_requested(self) -> bool:
|
||||
try:
|
||||
return True if self.__redis.exists('shutdown') else False
|
||||
except ConnectionRefusedError:
|
||||
return True
|
||||
except ConnectionError:
|
||||
return True
|
||||
|
||||
def _to_run_forever(self) -> None:
|
||||
pass
|
||||
|
||||
def run(self, sleep_in_sec: int) -> None:
|
||||
self.logger.info(f'Launching {self.__class__.__name__}')
|
||||
try:
|
||||
while True:
|
||||
if self.shutdown_requested():
|
||||
break
|
||||
try:
|
||||
if self.process:
|
||||
if self.process.poll() is not None:
|
||||
self.logger.critical(f'Unable to start {self.script_name}.')
|
||||
break
|
||||
else:
|
||||
self.set_running()
|
||||
self._to_run_forever()
|
||||
except Exception:
|
||||
self.logger.exception(f'Something went terribly wrong in {self.__class__.__name__}.')
|
||||
finally:
|
||||
if not self.process:
|
||||
# self.process means we run an external script, all the time,
|
||||
# do not unset between sleep.
|
||||
self.unset_running()
|
||||
if not self.long_sleep(sleep_in_sec):
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
self.logger.warning(f'{self.script_name} killed by user.')
|
||||
finally:
|
||||
if self.process:
|
||||
try:
|
||||
# Killing everything if possible.
|
||||
self.process.send_signal(signal.SIGWINCH)
|
||||
self.process.send_signal(signal.SIGTERM)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.unset_running()
|
||||
except Exception:
|
||||
# the services can already be down at that point.
|
||||
pass
|
||||
self.logger.info(f'Shutting down {self.__class__.__name__}')
|
||||
|
||||
async def _to_run_forever_async(self) -> None:
|
||||
pass
|
||||
|
||||
async def run_async(self, sleep_in_sec: int) -> None:
|
||||
self.logger.info(f'Launching {self.__class__.__name__}')
|
||||
try:
|
||||
while True:
|
||||
if self.shutdown_requested():
|
||||
break
|
||||
try:
|
||||
if self.process:
|
||||
if self.process.poll() is not None:
|
||||
self.logger.critical(f'Unable to start {self.script_name}.')
|
||||
break
|
||||
else:
|
||||
self.set_running()
|
||||
await self._to_run_forever_async()
|
||||
except Exception:
|
||||
self.logger.exception(f'Something went terribly wrong in {self.__class__.__name__}.')
|
||||
finally:
|
||||
if not self.process:
|
||||
# self.process means we run an external script, all the time,
|
||||
# do not unset between sleep.
|
||||
self.unset_running()
|
||||
if not await self.long_sleep_async(sleep_in_sec):
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
self.logger.warning(f'{self.script_name} killed by user.')
|
||||
finally:
|
||||
if self.process:
|
||||
try:
|
||||
# Killing everything if possible.
|
||||
self.process.send_signal(signal.SIGWINCH)
|
||||
self.process.send_signal(signal.SIGTERM)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.unset_running()
|
||||
except Exception:
|
||||
# the services can already be down at that point.
|
||||
pass
|
||||
self.logger.info(f'Shutting down {self.__class__.__name__}')
|
||||
|
|
@ -36,3 +36,7 @@ class MissingConfigEntry(BGPRankingException):
|
|||
|
||||
class ThirdPartyUnreachable(BGPRankingException):
|
||||
pass
|
||||
|
||||
|
||||
class ConfigError(BGPRankingException):
|
||||
pass
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
from . import env_global_name
|
||||
from .exceptions import ConfigError, CreateDirectoryException, MissingEnv
|
||||
|
||||
configs: Dict[str, Dict[str, Any]] = {}
|
||||
logger = logging.getLogger('Helpers')
|
||||
|
||||
|
||||
@lru_cache(64)
|
||||
def get_homedir() -> Path:
|
||||
if not os.environ.get(env_global_name):
|
||||
# Try to open a .env file in the home directory if it exists.
|
||||
if (Path(__file__).resolve().parent.parent.parent / '.env').exists():
|
||||
with (Path(__file__).resolve().parent.parent.parent / '.env').open() as f:
|
||||
for line in f:
|
||||
key, value = line.strip().split('=', 1)
|
||||
if value[0] in ['"', "'"]:
|
||||
value = value[1:-1]
|
||||
os.environ[key] = value
|
||||
|
||||
if not os.environ.get(env_global_name):
|
||||
guessed_home = Path(__file__).resolve().parent.parent.parent
|
||||
raise MissingEnv(f"{env_global_name} is missing. \
|
||||
Run the following command (assuming you run the code from the clonned repository):\
|
||||
export {env_global_name}='{guessed_home}'")
|
||||
return Path(os.environ[env_global_name])
|
||||
|
||||
|
||||
@lru_cache(64)
|
||||
def load_configs(path_to_config_files: Optional[Union[str, Path]]=None):
|
||||
global configs
|
||||
if configs:
|
||||
return
|
||||
if path_to_config_files:
|
||||
if isinstance(path_to_config_files, str):
|
||||
config_path = Path(path_to_config_files)
|
||||
else:
|
||||
config_path = path_to_config_files
|
||||
else:
|
||||
config_path = get_homedir() / 'config'
|
||||
if not config_path.exists():
|
||||
raise ConfigError(f'Configuration directory {config_path} does not exists.')
|
||||
elif not config_path.is_dir():
|
||||
raise ConfigError(f'Configuration directory {config_path} is not a directory.')
|
||||
|
||||
configs = {}
|
||||
for path in config_path.glob('*.json'):
|
||||
with path.open() as _c:
|
||||
configs[path.stem] = json.load(_c)
|
||||
|
||||
|
||||
@lru_cache(64)
|
||||
def get_config(config_type: str, entry: str, quiet: bool=False) -> Any:
|
||||
"""Get an entry from the given config_type file. Automatic fallback to the sample file"""
|
||||
global configs
|
||||
if not configs:
|
||||
load_configs()
|
||||
if config_type in configs:
|
||||
if entry in configs[config_type]:
|
||||
return configs[config_type][entry]
|
||||
else:
|
||||
if not quiet:
|
||||
logger.warning(f'Unable to find {entry} in config file.')
|
||||
else:
|
||||
if not quiet:
|
||||
logger.warning(f'No {config_type} config file available.')
|
||||
if not quiet:
|
||||
logger.warning(f'Falling back on sample config, please initialize the {config_type} config file.')
|
||||
with (get_homedir() / 'config' / f'{config_type}.json.sample').open() as _c:
|
||||
sample_config = json.load(_c)
|
||||
return sample_config[entry]
|
||||
|
||||
|
||||
def safe_create_dir(to_create: Path) -> None:
|
||||
if to_create.exists() and not to_create.is_dir():
|
||||
raise CreateDirectoryException(f'The path {to_create} already exists and is not a directory')
|
||||
to_create.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def get_socket_path(name: str) -> str:
|
||||
mapping = {
|
||||
'cache': Path('cache', 'cache.sock'),
|
||||
'intake': Path('temp', 'intake.sock'),
|
||||
'prepare': Path('temp', 'prepare.sock')
|
||||
}
|
||||
return str(get_homedir() / mapping[name])
|
||||
|
||||
|
||||
def try_make_file(filename: Path):
|
||||
try:
|
||||
filename.touch(exist_ok=False)
|
||||
return True
|
||||
except FileExistsError:
|
||||
return False
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
import requests
|
||||
|
||||
from pyipasnhistory import IPASNHistory
|
||||
|
||||
from .default import get_homedir, get_config, ThirdPartyUnreachable, safe_create_dir
|
||||
|
||||
|
||||
@lru_cache(64)
|
||||
def get_data_dir() -> Path:
|
||||
capture_dir = get_homedir() / 'rawdata'
|
||||
safe_create_dir(capture_dir)
|
||||
return capture_dir
|
||||
|
||||
|
||||
@lru_cache(64)
|
||||
def get_modules_dir() -> Path:
|
||||
modules_dir = get_homedir() / 'config' / 'modules'
|
||||
safe_create_dir(modules_dir)
|
||||
return modules_dir
|
||||
|
||||
|
||||
@lru_cache(64)
|
||||
def get_modules() -> List[Path]:
|
||||
return [modulepath for modulepath in get_modules_dir().glob('*.json')]
|
||||
|
||||
|
||||
@lru_cache(64)
|
||||
def load_all_modules_configs() -> Dict[str, Dict]:
|
||||
configs = {}
|
||||
for p in get_modules():
|
||||
with p.open() as f:
|
||||
j = json.load(f)
|
||||
configs[f"{j['vendor']}-{j['name']}"] = j
|
||||
return configs
|
||||
|
||||
|
||||
def get_ipasn():
|
||||
ipasnhistory_url = get_config('generic', 'ipasnhistory_url')
|
||||
ipasn = IPASNHistory(ipasnhistory_url)
|
||||
if not ipasn.is_up:
|
||||
raise ThirdPartyUnreachable(f"Unable to reach IPASNHistory on {ipasnhistory_url}")
|
||||
return ipasn
|
||||
|
||||
|
||||
def sanity_check_ipasn(ipasn):
|
||||
try:
|
||||
meta = ipasn.meta()
|
||||
except requests.exceptions.ConnectionError:
|
||||
return False, "IP ASN History is not reachable, try again later."
|
||||
|
||||
if 'error' in meta:
|
||||
raise ThirdPartyUnreachable(f'IP ASN History has a problem: {meta["error"]}')
|
||||
|
||||
v4_percent = meta['cached_dates']['caida']['v4']['percent']
|
||||
v6_percent = meta['cached_dates']['caida']['v6']['percent']
|
||||
if v4_percent < 90 or v6_percent < 90: # (this way it works if we only load 10 days)
|
||||
# Try again later.
|
||||
return False, f"IP ASN History is not ready: v4 {v4_percent}% / v6 {v6_percent}% loaded"
|
||||
return True, f"IP ASN History is ready: v4 {v4_percent}% / v6 {v6_percent}% loaded"
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from .exceptions import CreateDirectoryException, MissingEnv, MissingConfigFile, MissingConfigEntry, ThirdPartyUnreachable
|
||||
from redis import StrictRedis
|
||||
from redis.exceptions import ConnectionError
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
from pyipasnhistory import IPASNHistory
|
||||
|
||||
|
||||
def load_config_files(config_dir: Path=None) -> dict:
|
||||
if not config_dir:
|
||||
config_dir = get_config_path()
|
||||
modules_config = config_dir / 'modules'
|
||||
modules_paths = [modulepath for modulepath in modules_config.glob('*.json')]
|
||||
configs = {}
|
||||
for p in modules_paths:
|
||||
with open(p, 'r') as f:
|
||||
j = json.load(f)
|
||||
configs[f"{j['vendor']}-{j['name']}"] = j
|
||||
return configs
|
||||
|
||||
|
||||
def get_config_path():
|
||||
if Path('bgpranking', 'config').exists():
|
||||
# Running from the repository
|
||||
return Path('bgpranking', 'config')
|
||||
return Path(sys.modules['bgpranking'].__file__).parent / 'config'
|
||||
|
||||
|
||||
def get_list_storage_path():
|
||||
if not os.environ.get('VIRTUAL_ENV'):
|
||||
raise MissingEnv("VIRTUAL_ENV is missing. This project really wants to run from a virtual envoronment.")
|
||||
return Path(os.environ['VIRTUAL_ENV'])
|
||||
|
||||
|
||||
def get_homedir():
|
||||
if not os.environ.get('BGPRANKING_HOME'):
|
||||
guessed_home = Path(__file__).resolve().parent.parent.parent
|
||||
raise MissingEnv(f"BGPRANKING_HOME is missing. \
|
||||
Run the following command (assuming you run the code from the clonned repository):\
|
||||
export BGPRANKING_HOME='{guessed_home}'")
|
||||
return Path(os.environ['BGPRANKING_HOME'])
|
||||
|
||||
|
||||
def safe_create_dir(to_create: Path):
|
||||
if to_create.exists() and not to_create.is_dir():
|
||||
raise CreateDirectoryException(f'The path {to_create} already exists and is not a directory')
|
||||
os.makedirs(to_create, exist_ok=True)
|
||||
|
||||
|
||||
def set_running(name: str):
|
||||
r = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
r.hset('running', name, 1)
|
||||
|
||||
|
||||
def unset_running(name: str):
|
||||
r = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
r.hdel('running', name)
|
||||
|
||||
|
||||
def is_running():
|
||||
r = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
return r.hgetall('running')
|
||||
|
||||
|
||||
def get_socket_path(name: str):
|
||||
mapping = {
|
||||
'cache': Path('cache', 'cache.sock'),
|
||||
'storage': Path('storage', 'storage.sock'),
|
||||
'intake': Path('temp', 'intake.sock'),
|
||||
'prepare': Path('temp', 'prepare.sock'),
|
||||
}
|
||||
return str(get_homedir() / mapping[name])
|
||||
|
||||
|
||||
def load_general_config():
|
||||
general_config_file = get_config_path() / 'bgpranking.json'
|
||||
if not general_config_file.exists():
|
||||
raise MissingConfigFile(f'The general configuration file ({general_config_file}) does not exists.')
|
||||
with open(general_config_file) as f:
|
||||
config = json.load(f)
|
||||
return config, general_config_file
|
||||
|
||||
|
||||
def get_ipasn():
|
||||
config, general_config_file = load_general_config()
|
||||
if 'ipasnhistory_url' not in config:
|
||||
raise MissingConfigEntry(f'"ipasnhistory_url" is missing in {general_config_file}.')
|
||||
ipasn = IPASNHistory(config['ipasnhistory_url'])
|
||||
if not ipasn.is_up:
|
||||
raise ThirdPartyUnreachable(f"Unable to reach IPASNHistory on {config['ipasnhistory_url']}")
|
||||
return ipasn
|
||||
|
||||
|
||||
def sanity_check_ipasn(ipasn):
|
||||
meta = ipasn.meta()
|
||||
if 'error' in meta:
|
||||
raise ThirdPartyUnreachable(f'IP ASN History has a problem: meta["error"]')
|
||||
|
||||
v4_percent = meta['cached_dates']['caida']['v4']['percent']
|
||||
v6_percent = meta['cached_dates']['caida']['v6']['percent']
|
||||
if v4_percent < 90 or v6_percent < 90: # (this way it works if we only load 10 days)
|
||||
# Try again later.
|
||||
return False, f"IP ASN History is not ready: v4 {v4_percent}% / v6 {v6_percent}% loaded"
|
||||
return True, f"IP ASN History is ready: v4 {v4_percent}% / v6 {v6_percent}% loaded"
|
||||
|
||||
|
||||
def check_running(name: str):
|
||||
socket_path = get_socket_path(name)
|
||||
try:
|
||||
r = StrictRedis(unix_socket_path=socket_path)
|
||||
return r.ping()
|
||||
except ConnectionError:
|
||||
return False
|
||||
|
||||
|
||||
def shutdown_requested():
|
||||
try:
|
||||
r = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
return r.exists('shutdown')
|
||||
except ConnectionRefusedError:
|
||||
return True
|
||||
except ConnectionError:
|
||||
return True
|
||||
|
||||
|
||||
def long_sleep(sleep_in_sec: int, shutdown_check: int=10):
|
||||
sleep_until = datetime.now() + timedelta(seconds=sleep_in_sec)
|
||||
while sleep_until > datetime.now():
|
||||
time.sleep(shutdown_check)
|
||||
if shutdown_requested():
|
||||
return False
|
||||
return True
|
||||
|
|
@ -1,150 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import aiohttp
|
||||
from dateutil import parser
|
||||
from datetime import datetime, date
|
||||
from hashlib import sha512 # Faster than sha256 on 64b machines.
|
||||
from pathlib import Path
|
||||
import logging
|
||||
from pid import PidFile, PidFileError
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
from .libs.helpers import safe_create_dir, set_running, unset_running
|
||||
|
||||
|
||||
class Fetcher():
|
||||
|
||||
def __init__(self, config_file: Path, storage_directory: Path,
|
||||
loglevel: int=logging.DEBUG):
|
||||
'''Load `config_file`, and store the fetched data into `storage_directory`
|
||||
Note: if the `config_file` does not provide a URL (the file is
|
||||
gathered by some oter mean), the fetcher is automatically stoped.'''
|
||||
with open(config_file, 'r') as f:
|
||||
module_parameters = json.load(f)
|
||||
self.vendor = module_parameters['vendor']
|
||||
self.listname = module_parameters['name']
|
||||
self.__init_logger(loglevel)
|
||||
self.fetcher = True
|
||||
if 'url' not in module_parameters:
|
||||
self.logger.info('No URL to fetch, breaking.')
|
||||
self.fetcher = False
|
||||
return
|
||||
self.url = module_parameters['url']
|
||||
self.logger.debug(f'Starting fetcher on {self.url}')
|
||||
self.directory = storage_directory / self.vendor / self.listname
|
||||
safe_create_dir(self.directory)
|
||||
self.meta = self.directory / 'meta'
|
||||
safe_create_dir(self.meta)
|
||||
self.archive_dir = self.directory / 'archive'
|
||||
safe_create_dir(self.archive_dir)
|
||||
self.first_fetch = True
|
||||
|
||||
def __init_logger(self, loglevel):
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}-{self.vendor}-{self.listname}')
|
||||
self.logger.setLevel(loglevel)
|
||||
|
||||
async def __get_last_modified(self):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.head(self.url) as r:
|
||||
headers = r.headers
|
||||
if 'Last-Modified' in headers:
|
||||
return parser.parse(headers['Last-Modified'])
|
||||
return None
|
||||
|
||||
async def __newer(self):
|
||||
'''Check if the file available for download is newed than the one
|
||||
already downloaded by checking the `Last-Modified` header.
|
||||
Note: return False if the file containing the last header content
|
||||
is not existing, or the header doesn't have this key.
|
||||
'''
|
||||
last_modified_path = self.meta / 'lastmodified'
|
||||
if not last_modified_path.exists():
|
||||
# The file doesn't exists
|
||||
if not self.first_fetch:
|
||||
# The URL has no Last-Modified header, we cannot use it.
|
||||
self.logger.debug('No Last-Modified header available')
|
||||
return True
|
||||
self.first_fetch = False
|
||||
last_modified = await self.__get_last_modified()
|
||||
if last_modified:
|
||||
self.logger.debug('Last-Modified header available')
|
||||
with last_modified_path.open('w') as f:
|
||||
f.write(last_modified.isoformat())
|
||||
else:
|
||||
self.logger.debug('No Last-Modified header available')
|
||||
return True
|
||||
with last_modified_path.open() as f:
|
||||
file_content = f.read()
|
||||
last_modified_file = parser.parse(file_content)
|
||||
last_modified = await self.__get_last_modified()
|
||||
if not last_modified:
|
||||
# No more Last-Modified header Oo
|
||||
self.logger.warning(f'{self.listname}: Last-Modified header was present, isn\'t anymore!')
|
||||
last_modified_path.unlink()
|
||||
return True
|
||||
if last_modified > last_modified_file:
|
||||
self.logger.info('Got a new file.')
|
||||
with last_modified_path.open('w') as f:
|
||||
f.write(last_modified.isoformat())
|
||||
return True
|
||||
return False
|
||||
|
||||
def __same_as_last(self, downloaded):
|
||||
'''Figure out the last downloaded file, check if it is the same as the
|
||||
newly downloaded one. Returns true if both files have been downloaded the
|
||||
same day.
|
||||
Note: we check the new and the archive directory because we may have backlog
|
||||
and the newest file is always the first one we process
|
||||
'''
|
||||
to_check = []
|
||||
to_check_new = sorted([f for f in self.directory.iterdir() if f.is_file()])
|
||||
if to_check_new:
|
||||
# we have files waiting to be processed
|
||||
self.logger.debug('{} file(s) are waiting to be processed'.format(len(to_check_new)))
|
||||
to_check.append(to_check_new[-1])
|
||||
to_check_archive = sorted([f for f in self.archive_dir.iterdir() if f.is_file()])
|
||||
if to_check_archive:
|
||||
# we have files already processed, in the archive
|
||||
self.logger.debug('{} file(s) have been processed'.format(len(to_check_archive)))
|
||||
to_check.append(to_check_archive[-1])
|
||||
if not to_check:
|
||||
self.logger.debug('New list, no hisorical files')
|
||||
# nothing has been downloaded ever, moving on
|
||||
return False
|
||||
dl_hash = sha512(downloaded)
|
||||
for last_file in to_check:
|
||||
with last_file.open('rb') as f:
|
||||
last_hash = sha512(f.read())
|
||||
if (dl_hash.digest() == last_hash.digest() and
|
||||
parser.parse(last_file.name.split('.')[0]).date() == date.today()):
|
||||
self.logger.debug('Same file already downloaded today.')
|
||||
return True
|
||||
return False
|
||||
|
||||
async def fetch_list(self):
|
||||
'''Fetch & store the list'''
|
||||
if not self.fetcher:
|
||||
return
|
||||
set_running(f'{self.__class__.__name__}-{self.vendor}-{self.listname}')
|
||||
try:
|
||||
with PidFile(f'{self.listname}.pid', piddir=self.meta):
|
||||
if not await self.__newer():
|
||||
unset_running(f'{self.__class__.__name__}-{self.vendor}-{self.listname}')
|
||||
return
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(self.url) as r:
|
||||
content = await r.content.read()
|
||||
if self.__same_as_last(content):
|
||||
return
|
||||
self.logger.info('Got a new file \o/')
|
||||
with (self.directory / '{}.txt'.format(datetime.now().isoformat())).open('wb') as f:
|
||||
f.write(content)
|
||||
unset_running(f'{self.__class__.__name__}-{self.vendor}-{self.listname}')
|
||||
except PidFileError:
|
||||
self.logger.info('Fetcher already running')
|
||||
finally:
|
||||
unset_running(f'{self.__class__.__name__}-{self.vendor}-{self.listname}')
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import logging
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
import re
|
||||
from redis import StrictRedis
|
||||
from uuid import uuid4
|
||||
from io import BytesIO
|
||||
import importlib
|
||||
|
||||
from typing import List
|
||||
import types
|
||||
|
||||
from .libs.helpers import safe_create_dir, set_running, unset_running, get_socket_path
|
||||
|
||||
|
||||
class RawFilesParser():
|
||||
|
||||
def __init__(self, config_file: Path, storage_directory: Path,
|
||||
loglevel: int=logging.DEBUG) -> None:
|
||||
with open(config_file, 'r') as f:
|
||||
module_parameters = json.load(f)
|
||||
self.vendor = module_parameters['vendor']
|
||||
self.listname = module_parameters['name']
|
||||
if 'parser' in module_parameters:
|
||||
self.parse_raw_file = types.MethodType(importlib.import_module(module_parameters['parser'], 'bgpranking').parse_raw_file, self)
|
||||
self.source = f'{self.vendor}-{self.listname}'
|
||||
self.directory = storage_directory / self.vendor / self.listname
|
||||
safe_create_dir(self.directory)
|
||||
self.unparsable_dir = self.directory / 'unparsable'
|
||||
safe_create_dir(self.unparsable_dir)
|
||||
self.__init_logger(loglevel)
|
||||
self.redis_intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0)
|
||||
self.logger.debug(f'Starting intake on {self.source}')
|
||||
|
||||
def __init_logger(self, loglevel) -> None:
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}-{self.vendor}-{self.listname}')
|
||||
self.logger.setLevel(loglevel)
|
||||
|
||||
@property
|
||||
def files_to_parse(self) -> List[Path]:
|
||||
return sorted([f for f in self.directory.iterdir() if f.is_file()], reverse=True)
|
||||
|
||||
def extract_ipv4(self, bytestream: bytes) -> List[bytes]:
|
||||
return re.findall(rb'[0-9]+(?:\.[0-9]+){3}', bytestream)
|
||||
|
||||
def strip_leading_zeros(self, ips: List[bytes]) -> List[bytes]:
|
||||
'''Helper to get rid of leading 0s in an IP list.
|
||||
Only run it when needed, it is nasty and slow'''
|
||||
return ['.'.join(str(int(part)) for part in ip.split(b'.')).encode() for ip in ips]
|
||||
|
||||
def parse_raw_file(self, f: BytesIO) -> List[bytes]:
|
||||
# If the list doesn't provide a time, fallback to current day, midnight
|
||||
self.datetime = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
return self.extract_ipv4(f.getvalue())
|
||||
|
||||
def parse_raw_files(self) -> None:
|
||||
set_running(f'{self.__class__.__name__}-{self.source}')
|
||||
nb_unparsable_files = len([f for f in self.unparsable_dir.iterdir() if f.is_file()])
|
||||
if nb_unparsable_files:
|
||||
self.logger.warning(f'Was unable to parse {nb_unparsable_files} files.')
|
||||
try:
|
||||
for filepath in self.files_to_parse:
|
||||
self.logger.debug('Parsing {}, {} to go.'.format(filepath, len(self.files_to_parse) - 1))
|
||||
with open(filepath, 'rb') as f:
|
||||
to_parse = BytesIO(f.read())
|
||||
p = self.redis_intake.pipeline()
|
||||
for ip in self.parse_raw_file(to_parse):
|
||||
if isinstance(ip, tuple):
|
||||
ip, datetime = ip
|
||||
else:
|
||||
datetime = self.datetime
|
||||
uuid = uuid4()
|
||||
p.hmset(str(uuid), {'ip': ip, 'source': self.source,
|
||||
'datetime': datetime.isoformat()})
|
||||
p.sadd('intake', str(uuid))
|
||||
p.execute()
|
||||
self._archive(filepath)
|
||||
except Exception as e:
|
||||
self.logger.exception("That didn't go well")
|
||||
self._unparsable(filepath)
|
||||
finally:
|
||||
unset_running(f'{self.__class__.__name__}-{self.source}')
|
||||
|
||||
def _archive(self, filepath: Path) -> None:
|
||||
'''After processing, move file to the archive directory'''
|
||||
filepath.rename(self.directory / 'archive' / filepath.name)
|
||||
|
||||
def _unparsable(self, filepath: Path) -> None:
|
||||
'''After processing, move file to the archive directory'''
|
||||
filepath.rename(self.unparsable_dir / filepath.name)
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from dateutil.parser import parse
|
||||
import re
|
||||
from io import BytesIO
|
||||
|
||||
from typing import List
|
||||
|
||||
|
||||
def parse_raw_file(self, f: BytesIO) -> List[bytes]:
|
||||
if re.findall(b'This feed is not generated for this family', f.getvalue()):
|
||||
return []
|
||||
|
||||
self.datetime = parse(re.findall(b'## Feed generated at: (.*)\n', f.getvalue())[0])
|
||||
return self.extract_ipv4(f.getvalue())
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
from redis import StrictRedis
|
||||
from .libs.helpers import set_running, unset_running, get_socket_path, load_config_files, get_ipasn, sanity_check_ipasn
|
||||
from datetime import datetime, date, timedelta
|
||||
from ipaddress import ip_network
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Ranking():
|
||||
|
||||
def __init__(self, config_dir: Path=None, loglevel: int=logging.DEBUG):
|
||||
self.__init_logger(loglevel)
|
||||
self.storage = StrictRedis(unix_socket_path=get_socket_path('storage'), decode_responses=True)
|
||||
self.ranking = StrictRedis(unix_socket_path=get_socket_path('storage'), db=1, decode_responses=True)
|
||||
self.ipasn = get_ipasn()
|
||||
self.config_dir = config_dir
|
||||
|
||||
def __init_logger(self, loglevel):
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||
self.logger.setLevel(loglevel)
|
||||
|
||||
def rank_a_day(self, day: str, config_files: dict):
|
||||
asns_aggregation_key_v4 = f'{day}|asns|v4'
|
||||
asns_aggregation_key_v6 = f'{day}|asns|v6'
|
||||
to_delete = set([asns_aggregation_key_v4, asns_aggregation_key_v6])
|
||||
r_pipeline = self.ranking.pipeline()
|
||||
cached_meta = {}
|
||||
for source in self.storage.smembers(f'{day}|sources'):
|
||||
self.logger.info(f'{day} - Ranking source: {source}')
|
||||
source_aggregation_key_v4 = f'{day}|{source}|asns|v4'
|
||||
source_aggregation_key_v6 = f'{day}|{source}|asns|v6'
|
||||
to_delete.update([source_aggregation_key_v4, source_aggregation_key_v6])
|
||||
for asn in self.storage.smembers(f'{day}|{source}'):
|
||||
prefixes_aggregation_key_v4 = f'{day}|{asn}|v4'
|
||||
prefixes_aggregation_key_v6 = f'{day}|{asn}|v6'
|
||||
to_delete.update([prefixes_aggregation_key_v4, prefixes_aggregation_key_v6])
|
||||
if asn == '0':
|
||||
# Default ASN when no matches. Probably spoofed.
|
||||
continue
|
||||
self.logger.debug(f'{day} - Ranking source: {source} / ASN: {asn}')
|
||||
asn_rank_v4 = 0.0
|
||||
asn_rank_v6 = 0.0
|
||||
for prefix in self.storage.smembers(f'{day}|{source}|{asn}'):
|
||||
if prefix == 'None':
|
||||
# This should not happen and requires a DB cleanup.
|
||||
self.logger.critical(f'Fucked up prefix in "{day}|{source}|{asn}"')
|
||||
continue
|
||||
ips = set([ip_ts.split('|')[0]
|
||||
for ip_ts in self.storage.smembers(f'{day}|{source}|{asn}|{prefix}')])
|
||||
py_prefix = ip_network(prefix)
|
||||
prefix_rank = float(len(ips)) / py_prefix.num_addresses
|
||||
r_pipeline.zadd(f'{day}|{source}|{asn}|v{py_prefix.version}|prefixes', {prefix: prefix_rank})
|
||||
if py_prefix.version == 4:
|
||||
asn_rank_v4 += len(ips) * config_files[source]['impact']
|
||||
r_pipeline.zincrby(prefixes_aggregation_key_v4, prefix_rank * config_files[source]['impact'], prefix)
|
||||
else:
|
||||
asn_rank_v6 += len(ips) * config_files[source]['impact']
|
||||
r_pipeline.zincrby(prefixes_aggregation_key_v6, prefix_rank * config_files[source]['impact'], prefix)
|
||||
if asn in cached_meta:
|
||||
v4info = cached_meta[asn]['v4']
|
||||
v6info = cached_meta[asn]['v6']
|
||||
else:
|
||||
v4info = self.ipasn.asn_meta(asn=asn, source='caida', address_family='v4', date=day)
|
||||
v6info = self.ipasn.asn_meta(asn=asn, source='caida', address_family='v6', date=day)
|
||||
cached_meta[asn] = {'v4': v4info, 'v6': v6info}
|
||||
ipasnhistory_date_v4 = list(v4info['response'].keys())[0]
|
||||
v4count = v4info['response'][ipasnhistory_date_v4][asn]['ipcount']
|
||||
ipasnhistory_date_v6 = list(v6info['response'].keys())[0]
|
||||
v6count = v6info['response'][ipasnhistory_date_v6][asn]['ipcount']
|
||||
if v4count:
|
||||
asn_rank_v4 /= float(v4count)
|
||||
if asn_rank_v4:
|
||||
r_pipeline.set(f'{day}|{source}|{asn}|v4', asn_rank_v4)
|
||||
r_pipeline.zincrby(asns_aggregation_key_v4, asn_rank_v4, asn)
|
||||
r_pipeline.zadd(source_aggregation_key_v4, {asn: asn_rank_v4})
|
||||
if v6count:
|
||||
asn_rank_v6 /= float(v6count)
|
||||
if asn_rank_v6:
|
||||
r_pipeline.set(f'{day}|{source}|{asn}|v6', asn_rank_v6)
|
||||
r_pipeline.zincrby(asns_aggregation_key_v6, asn_rank_v6, asn)
|
||||
r_pipeline.zadd(source_aggregation_key_v6, {asn: asn_rank_v6})
|
||||
self.ranking.delete(*to_delete)
|
||||
r_pipeline.execute()
|
||||
|
||||
def compute(self):
|
||||
config_files = load_config_files(self.config_dir)
|
||||
ready, message = sanity_check_ipasn(self.ipasn)
|
||||
if not ready:
|
||||
# Try again later.
|
||||
self.logger.warning(message)
|
||||
return
|
||||
self.logger.debug(message)
|
||||
|
||||
self.logger.info('Start ranking')
|
||||
set_running(self.__class__.__name__)
|
||||
today = date.today()
|
||||
now = datetime.now()
|
||||
today12am = now.replace(hour=12, minute=0, second=0, microsecond=0)
|
||||
if now < today12am:
|
||||
# Compute yesterday and today's ranking (useful when we have lists generated only once a day)
|
||||
self.rank_a_day((today - timedelta(days=1)).isoformat(), config_files)
|
||||
self.rank_a_day(today.isoformat(), config_files)
|
||||
unset_running(self.__class__.__name__)
|
||||
self.logger.info('Ranking done.')
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from datetime import timezone
|
||||
from dateutil import parser
|
||||
import logging
|
||||
from redis import StrictRedis
|
||||
import ipaddress
|
||||
|
||||
from .libs.helpers import shutdown_requested, set_running, unset_running, get_socket_path, get_ipasn, sanity_check_ipasn
|
||||
|
||||
|
||||
class Sanitizer():
|
||||
|
||||
def __init__(self, loglevel: int=logging.DEBUG):
|
||||
self.__init_logger(loglevel)
|
||||
self.redis_intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
|
||||
self.redis_sanitized = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
|
||||
self.ipasn = get_ipasn()
|
||||
self.logger.debug('Starting import')
|
||||
|
||||
def __init_logger(self, loglevel):
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||
self.logger.setLevel(loglevel)
|
||||
|
||||
def sanitize(self):
|
||||
ready, message = sanity_check_ipasn(self.ipasn)
|
||||
if not ready:
|
||||
# Try again later.
|
||||
self.logger.warning(message)
|
||||
return
|
||||
self.logger.debug(message)
|
||||
|
||||
set_running(self.__class__.__name__)
|
||||
while True:
|
||||
if shutdown_requested() or not self.ipasn.is_up:
|
||||
break
|
||||
uuids = self.redis_intake.spop('intake', 100)
|
||||
if not uuids:
|
||||
break
|
||||
for_cache = []
|
||||
pipeline = self.redis_sanitized.pipeline(transaction=False)
|
||||
for uuid in uuids:
|
||||
data = self.redis_intake.hgetall(uuid)
|
||||
try:
|
||||
ip = ipaddress.ip_address(data['ip'])
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
address_family = 'v6'
|
||||
else:
|
||||
address_family = 'v4'
|
||||
except ValueError:
|
||||
self.logger.info(f"Invalid IP address: {data['ip']}")
|
||||
continue
|
||||
except KeyError:
|
||||
self.logger.info(f"Invalid entry {data}")
|
||||
continue
|
||||
|
||||
if not ip.is_global:
|
||||
self.logger.info(f"The IP address {data['ip']} is not global")
|
||||
continue
|
||||
|
||||
datetime = parser.parse(data['datetime'])
|
||||
if datetime.tzinfo:
|
||||
# Make sure the datetime isn't TZ aware, and UTC.
|
||||
datetime = datetime.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
for_cache.append({'ip': str(ip), 'address_family': address_family, 'source': 'caida',
|
||||
'date': datetime.isoformat(), 'precision_delta': {'days': 3}})
|
||||
|
||||
# Add to temporay DB for further processing
|
||||
pipeline.hmset(uuid, {'ip': str(ip), 'source': data['source'], 'address_family': address_family,
|
||||
'date': datetime.date().isoformat(), 'datetime': datetime.isoformat()})
|
||||
pipeline.sadd('to_insert', uuid)
|
||||
pipeline.execute()
|
||||
self.redis_intake.delete(*uuids)
|
||||
|
||||
try:
|
||||
# Just cache everything so the lookup scripts can do their thing.
|
||||
self.ipasn.mass_cache(for_cache)
|
||||
except Exception:
|
||||
self.logger.exception('Mass cache in IPASN History failed, trying again later.')
|
||||
# Rollback the spop
|
||||
self.redis_intake.sadd('intake', *uuids)
|
||||
break
|
||||
unset_running(self.__class__.__name__)
|
||||
|
|
@ -1,171 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import aiohttp
|
||||
import logging
|
||||
from bs4 import BeautifulSoup
|
||||
from dateutil.parser import parse
|
||||
from typing import Tuple
|
||||
from datetime import datetime, date
|
||||
from pathlib import Path
|
||||
from .libs.helpers import safe_create_dir, set_running, unset_running
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
|
||||
class ShadowServerFetcher():
|
||||
|
||||
def __init__(self, user, password, config_path_modules: Path, storage_directory: Path,
|
||||
loglevel: int=logging.DEBUG) -> None:
|
||||
self.__init_logger(loglevel)
|
||||
self.storage_directory = storage_directory
|
||||
self.config_path_modules = config_path_modules
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.index_page = 'https://dl.shadowserver.org/reports/index.php'
|
||||
self.vendor = 'shadowserver'
|
||||
self.known_list_types = ('blacklist', 'botnet', 'cc', 'cisco', 'cwsandbox', 'drone',
|
||||
'microsoft', 'scan', 'sinkhole6', 'sinkhole', 'outdated',
|
||||
'compromised', 'hp', 'darknet', 'ddos')
|
||||
self.first_available_day = None
|
||||
self.last_available_day = None
|
||||
self.available_entries = {}
|
||||
|
||||
def __init_logger(self, loglevel):
|
||||
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||
self.logger.setLevel(loglevel)
|
||||
|
||||
async def __get_index(self):
|
||||
auth_details = {'user': self.user, 'password': self.password, 'login': 'Login'}
|
||||
async with aiohttp.ClientSession() as s:
|
||||
self.logger.debug('Fetching the index.')
|
||||
async with s.post(self.index_page, data=auth_details) as r:
|
||||
return await r.text()
|
||||
|
||||
async def __build_daily_dict(self):
|
||||
html_index = await self.__get_index()
|
||||
soup = BeautifulSoup(html_index, 'html.parser')
|
||||
treeview = soup.find(id='treemenu1')
|
||||
for y in treeview.select(':scope > li'):
|
||||
year = y.contents[0]
|
||||
for m in y.contents[1].select(':scope > li'):
|
||||
month = m.contents[0]
|
||||
for d in m.contents[1].select(':scope > li'):
|
||||
day = d.contents[0]
|
||||
date = parse(f'{year} {month} {day}').date()
|
||||
self.available_entries[date.isoformat()] = []
|
||||
for a in d.contents[1].find_all('a', href=True):
|
||||
if not self.first_available_day:
|
||||
self.first_available_day = date
|
||||
self.last_available_day = date
|
||||
self.available_entries[date.isoformat()].append((a['href'], a.string))
|
||||
self.logger.debug('Dictionary created.')
|
||||
|
||||
def __normalize_day(self, day: Tuple[str, date, datetime]=None) -> str:
|
||||
if not day:
|
||||
if not self.last_available_day:
|
||||
raise Exception('Unable to figure out the last available day. You need to run build_daily_dict first')
|
||||
day = self.last_available_day
|
||||
else:
|
||||
if isinstance(day, str):
|
||||
day = parse(day).date()
|
||||
elif isinstance(day, datetime):
|
||||
day = day.date()
|
||||
return day.isoformat()
|
||||
|
||||
def __split_name(self, name):
|
||||
type_content, country, list_type = name.split('-')
|
||||
if '_' in type_content:
|
||||
type_content, details_type = type_content.split('_', maxsplit=1)
|
||||
if '_' in details_type:
|
||||
details_type, sub = details_type.split('_', maxsplit=1)
|
||||
return list_type, country, (type_content, details_type, sub)
|
||||
return list_type, country, (type_content, details_type)
|
||||
return list_type, country, (type_content)
|
||||
|
||||
def __check_config(self, filename: str) -> Path:
|
||||
self.logger.debug(f'Working on config for {filename}.')
|
||||
config = {'vendor': 'shadowserver', 'parser': '.parsers.shadowserver'}
|
||||
type_content, _, type_details = self.__split_name(filename)
|
||||
prefix = type_content.split('.')[0]
|
||||
|
||||
if isinstance(type_details, str):
|
||||
main_type = type_details
|
||||
config['name'] = '{}-{}'.format(prefix, type_details)
|
||||
else:
|
||||
main_type = type_details[0]
|
||||
config['name'] = '{}-{}'.format(prefix, '_'.join(type_details))
|
||||
|
||||
if main_type not in self.known_list_types:
|
||||
self.logger.warning(f'Unknown type: {main_type}. Please update the config creator script.')
|
||||
return None
|
||||
|
||||
if main_type == 'blacklist':
|
||||
config['impact'] = 5
|
||||
elif main_type == 'botnet':
|
||||
config['impact'] = 2
|
||||
elif main_type == 'cc':
|
||||
config['impact'] = 5
|
||||
elif main_type == 'cisco':
|
||||
config['impact'] = 3
|
||||
elif main_type == 'cwsandbox':
|
||||
config['impact'] = 5
|
||||
elif main_type == 'drone':
|
||||
config['impact'] = 2
|
||||
elif main_type == 'microsoft':
|
||||
config['impact'] = 3
|
||||
elif main_type == 'scan':
|
||||
config['impact'] = 1
|
||||
elif main_type == 'sinkhole6':
|
||||
config['impact'] = 2
|
||||
elif main_type == 'sinkhole':
|
||||
config['impact'] = 2
|
||||
else:
|
||||
config['impact'] = 1
|
||||
|
||||
if not (self.config_path_modules / f"{config['vendor']}_{config['name']}.json").exists():
|
||||
self.logger.debug(f'Creating config file for {filename}.')
|
||||
with open(self.config_path_modules / f"{config['vendor']}_{config['name']}.json", 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
else:
|
||||
with open(self.config_path_modules / f"{config['vendor']}_{config['name']}.json", 'r') as f:
|
||||
# Validate new config file with old
|
||||
config_current = json.load(f)
|
||||
if config_current != config:
|
||||
self.logger.warning('The config file created by this script is different from the one on disk: \n{}\n{}'.format(json.dumps(config), json.dumps(config_current)))
|
||||
# Init list directory
|
||||
directory = self.storage_directory / config['vendor'] / config['name']
|
||||
safe_create_dir(directory)
|
||||
meta = directory / 'meta'
|
||||
safe_create_dir(meta)
|
||||
archive_dir = directory / 'archive'
|
||||
safe_create_dir(archive_dir)
|
||||
self.logger.debug(f'Done with config for {filename}.')
|
||||
return directory
|
||||
|
||||
async def download_daily_entries(self, day: Tuple[str, date, datetime]=None):
|
||||
set_running(f'{self.__class__.__name__}')
|
||||
await self.__build_daily_dict()
|
||||
for url, filename in self.available_entries[self.__normalize_day(day)]:
|
||||
storage_dir = self.__check_config(filename)
|
||||
if not storage_dir:
|
||||
continue
|
||||
# Check if the file we're trying to download has already been downloaded. Skip if True.
|
||||
uuid = url.split('/')[-1]
|
||||
if (storage_dir / 'meta' / 'last_download').exists():
|
||||
with open(storage_dir / 'meta' / 'last_download') as f:
|
||||
last_download_uuid = f.read()
|
||||
if last_download_uuid == uuid:
|
||||
self.logger.debug(f'Already downloaded: {url}.')
|
||||
continue
|
||||
async with aiohttp.ClientSession() as s:
|
||||
async with s.get(url) as r:
|
||||
self.logger.info(f'Downloading {url}.')
|
||||
content = await r.content.read()
|
||||
with (storage_dir / '{}.txt'.format(datetime.now().isoformat())).open('wb') as f:
|
||||
f.write(content)
|
||||
with open(storage_dir / 'meta' / 'last_download', 'w') as f:
|
||||
f.write(uuid)
|
||||
unset_running(f'{self.__class__.__name__}')
|
||||
|
|
@ -1,18 +1,19 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import requests
|
||||
from enum import Enum
|
||||
from datetime import datetime, timedelta
|
||||
from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network
|
||||
from typing import TypeVar
|
||||
from .helpers import get_homedir, safe_create_dir
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
from dateutil.parser import parse
|
||||
import copy
|
||||
import json
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network
|
||||
from typing import TypeVar, Optional, Dict, Any
|
||||
|
||||
import requests
|
||||
|
||||
from dateutil.parser import parse
|
||||
|
||||
from .helpers import get_homedir, safe_create_dir
|
||||
|
||||
IPTypes = TypeVar('IPTypes', IPv4Address, IPv6Address, 'str')
|
||||
PrefixTypes = TypeVar('PrefixTypes', IPv4Network, IPv6Network, 'str')
|
||||
|
|
@ -84,7 +85,7 @@ class StatsRIPE():
|
|||
with open(c_path, 'w') as f:
|
||||
json.dump(response, f, indent=2)
|
||||
|
||||
def _get(self, method: str, parameters: dict) -> dict:
|
||||
def _get(self, method: str, parameters: Dict) -> Dict:
|
||||
parameters['sourceapp'] = self.sourceapp
|
||||
cached = self._get_cache(method, parameters)
|
||||
if cached:
|
||||
|
|
@ -100,8 +101,8 @@ class StatsRIPE():
|
|||
return self._get('network-info', parameters)
|
||||
|
||||
def prefix_overview(self, prefix: PrefixTypes, min_peers_seeing: int= 0,
|
||||
max_related: int=0, query_time: TimeTypes=None) -> dict:
|
||||
parameters = {'resource': prefix}
|
||||
max_related: int=0, query_time: Optional[TimeTypes]=None) -> dict:
|
||||
parameters: Dict[str, Any] = {'resource': prefix}
|
||||
if min_peers_seeing:
|
||||
parameters['min_peers_seeing'] = min_peers_seeing
|
||||
if max_related:
|
||||
|
|
@ -110,8 +111,8 @@ class StatsRIPE():
|
|||
parameters['query_time'] = self.__time_to_text(query_time)
|
||||
return self._get('prefix-overview', parameters)
|
||||
|
||||
def ris_asns(self, query_time: TimeTypes=None, list_asns: bool=False, asn_types: ASNsTypes=ASNsTypes.undefined):
|
||||
parameters = {}
|
||||
def ris_asns(self, query_time: Optional[TimeTypes]=None, list_asns: bool=False, asn_types: ASNsTypes=ASNsTypes.undefined):
|
||||
parameters: Dict[str, Any] = {}
|
||||
if list_asns:
|
||||
parameters['list_asns'] = list_asns
|
||||
if asn_types:
|
||||
|
|
@ -120,10 +121,10 @@ class StatsRIPE():
|
|||
parameters['query_time'] = self.__time_to_text(query_time)
|
||||
return self._get('ris-asns', parameters)
|
||||
|
||||
def ris_prefixes(self, asn: int, query_time: TimeTypes=None,
|
||||
def ris_prefixes(self, asn: int, query_time: Optional[TimeTypes]=None,
|
||||
list_prefixes: bool=False, types: ASNsTypes=ASNsTypes.undefined,
|
||||
af: AddressFamilies=AddressFamilies.undefined, noise: Noise=Noise.keep):
|
||||
parameters = {'resource': str(asn)}
|
||||
parameters: Dict[str, Any] = {'resource': str(asn)}
|
||||
if query_time:
|
||||
parameters['query_time'] = self.__time_to_text(query_time)
|
||||
if list_prefixes:
|
||||
|
|
@ -136,8 +137,8 @@ class StatsRIPE():
|
|||
parameters['noise'] = noise.value
|
||||
return self._get('ris-prefixes', parameters)
|
||||
|
||||
def country_asns(self, country: str, details: int=0, query_time: TimeTypes=None):
|
||||
parameters = {'resource': country}
|
||||
def country_asns(self, country: str, details: int=0, query_time: Optional[TimeTypes]=None):
|
||||
parameters: Dict[str, Any] = {'resource': country}
|
||||
if details:
|
||||
parameters['lod'] = details
|
||||
if query_time:
|
||||
|
|
@ -1,42 +1,79 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from bgpranking.archive import DeepArchive
|
||||
import json
|
||||
import logging
|
||||
import zipfile
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import date
|
||||
from logging import Logger
|
||||
from pathlib import Path
|
||||
from bgpranking.libs.helpers import get_config_path, get_homedir
|
||||
from pid import PidFile, PidFileError
|
||||
|
||||
from dateutil import parser
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
from bgpranking.default import safe_create_dir, AbstractManager
|
||||
from bgpranking.helpers import get_modules, get_data_dir
|
||||
|
||||
|
||||
logger = logging.getLogger('Archiver')
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
||||
# NOTE:
|
||||
# * Supposed to run once every ~2 months
|
||||
|
||||
class DeepArchive():
|
||||
|
||||
class ModulesArchiver():
|
||||
|
||||
def __init__(self, config_dir: Path=None, storage_directory: Path=None, loglevel: int=logging.INFO):
|
||||
if not config_dir:
|
||||
config_dir = get_config_path()
|
||||
if not storage_directory:
|
||||
self.storage_directory = get_homedir() / 'rawdata'
|
||||
modules_config = config_dir / 'modules'
|
||||
modules_paths = [modulepath for modulepath in modules_config.glob('*.json')]
|
||||
self.modules = [DeepArchive(path, self.storage_directory, loglevel) for path in modules_paths]
|
||||
def __init__(self, config_file: Path, logger: Logger):
|
||||
'''Archive everyfile older than 2 month.'''
|
||||
with config_file.open() as f:
|
||||
module_parameters = json.load(f)
|
||||
self.logger = logger
|
||||
self.vendor = module_parameters['vendor']
|
||||
self.listname = module_parameters['name']
|
||||
self.directory = get_data_dir() / self.vendor / self.listname / 'archive'
|
||||
safe_create_dir(self.directory)
|
||||
self.deep_archive = self.directory / 'deep'
|
||||
safe_create_dir(self.deep_archive)
|
||||
|
||||
def archive(self):
|
||||
to_archive = defaultdict(list)
|
||||
today = date.today()
|
||||
last_day_to_keep = date(today.year, today.month, 1) - relativedelta(months=2)
|
||||
for p in self.directory.iterdir():
|
||||
if not p.is_file():
|
||||
continue
|
||||
filedate = parser.parse(p.name.split('.')[0]).date()
|
||||
if filedate >= last_day_to_keep:
|
||||
continue
|
||||
to_archive['{}.zip'.format(filedate.strftime('%Y%m'))].append(p)
|
||||
if to_archive:
|
||||
self.logger.info('Found old files. Archiving: {}'.format(', '.join(to_archive.keys())))
|
||||
else:
|
||||
self.logger.debug('No old files.')
|
||||
for archivename, path_list in to_archive.items():
|
||||
with zipfile.ZipFile(self.deep_archive / archivename, 'x', zipfile.ZIP_DEFLATED) as z:
|
||||
for f in path_list:
|
||||
z.write(f, f.name)
|
||||
# Delete all the files if the archiving worked out properly
|
||||
[f.unlink() for f in path_list]
|
||||
|
||||
|
||||
class ModulesArchiver(AbstractManager):
|
||||
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
super().__init__(loglevel)
|
||||
self.script_name = 'archiver'
|
||||
self.modules = [DeepArchive(path, self.logger) for path in get_modules()]
|
||||
|
||||
def _to_run_forever(self):
|
||||
[module.archive() for module in self.modules]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
def main():
|
||||
archiver = ModulesArchiver()
|
||||
try:
|
||||
with PidFile(piddir=archiver.storage_directory):
|
||||
logger.info('Archiving...')
|
||||
archiver.archive()
|
||||
logger.info('... done.')
|
||||
except PidFileError:
|
||||
logger.warning('Archiver already running, skip.')
|
||||
archiver.run(sleep_in_sec=360000)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -2,28 +2,80 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import re
|
||||
import requests
|
||||
|
||||
from bgpranking.abstractmanager import AbstractManager
|
||||
from bgpranking.asn_descriptions import ASNDescriptions
|
||||
from bgpranking.libs.helpers import get_homedir
|
||||
from dateutil.parser import parse
|
||||
from redis import Redis
|
||||
|
||||
from bgpranking.default import get_socket_path, safe_create_dir, AbstractManager, get_config
|
||||
from bgpranking.helpers import get_data_dir
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
||||
|
||||
class ASNDescriptionsManager(AbstractManager):
|
||||
class ASNDescriptions(AbstractManager):
|
||||
|
||||
def __init__(self, storage_directory: Path=None, loglevel: int=logging.DEBUG):
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
super().__init__(loglevel)
|
||||
if not storage_directory:
|
||||
storage_directory = get_homedir() / 'rawdata'
|
||||
self.asn_descr = ASNDescriptions(storage_directory, loglevel)
|
||||
self.script_name = 'asn_descr'
|
||||
self.asn_meta = Redis(get_config('generic', 'storage_db_hostname'), get_config('generic', 'storage_db_port'), db=2, decode_responses=True)
|
||||
self.logger.debug('Starting ASN History')
|
||||
self.directory = get_data_dir() / 'asn_descriptions'
|
||||
safe_create_dir(self.directory)
|
||||
self.archives = self.directory / 'archive'
|
||||
safe_create_dir(self.archives)
|
||||
self.url = 'https://www.cidr-report.org/as2.0/autnums.html'
|
||||
|
||||
def __update_available(self):
|
||||
r = requests.head(self.url)
|
||||
print(r.headers)
|
||||
current_last_modified = parse(r.headers['Last-Modified'])
|
||||
if not self.asn_meta.exists('ans_description_last_update'):
|
||||
return True
|
||||
last_update = parse(self.asn_meta.get('ans_description_last_update')) # type: ignore
|
||||
if last_update < current_last_modified:
|
||||
return True
|
||||
return False
|
||||
|
||||
def load_descriptions(self):
|
||||
if not self.__update_available():
|
||||
self.logger.debug('No new file to import.')
|
||||
return
|
||||
self.logger.info('Importing new ASN descriptions.')
|
||||
r = requests.get(self.url)
|
||||
last_modified = parse(r.headers['Last-Modified']).isoformat()
|
||||
p = self.asn_meta.pipeline()
|
||||
new_asn = 0
|
||||
new_description = 0
|
||||
for asn, descr in re.findall('as=AS(.*)&.*</a> (.*)\n', r.text):
|
||||
existing_descriptions = self.asn_meta.hgetall(f'{asn}|descriptions')
|
||||
if not existing_descriptions:
|
||||
self.logger.debug(f'New ASN: {asn} - {descr}')
|
||||
p.hset(f'{asn}|descriptions', last_modified, descr)
|
||||
new_asn += 1
|
||||
else:
|
||||
last_descr = sorted(existing_descriptions.keys(), reverse=True)[0]
|
||||
if descr != existing_descriptions[last_descr]:
|
||||
self.logger.debug(f'New description for {asn}: {existing_descriptions[last_descr]} -> {descr}')
|
||||
p.hset(f'{asn}|descriptions', last_modified, descr)
|
||||
new_description += 1
|
||||
p.set('ans_description_last_update', last_modified)
|
||||
p.execute()
|
||||
self.logger.info(f'Done with import. New ASNs: {new_asn}, new descriptions: {new_description}')
|
||||
if new_asn or new_description:
|
||||
with open(self.archives / f'{last_modified}.html', 'w') as f:
|
||||
f.write(r.text)
|
||||
|
||||
def _to_run_forever(self):
|
||||
self.asn_descr.load_descriptions()
|
||||
self.load_descriptions()
|
||||
|
||||
|
||||
def main():
|
||||
asnd_manager = ASNDescriptions()
|
||||
asnd_manager.run(sleep_in_sec=3600)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asnd_manager = ASNDescriptionsManager()
|
||||
asnd_manager.run(sleep_in_sec=3600)
|
||||
main()
|
||||
|
|
|
|||
114
bin/dbinsert.py
114
bin/dbinsert.py
|
|
@ -2,8 +2,15 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
from bgpranking.abstractmanager import AbstractManager
|
||||
from bgpranking.dbinsert import DatabaseInsert
|
||||
import time
|
||||
|
||||
from typing import List
|
||||
|
||||
from redis import Redis
|
||||
|
||||
from bgpranking.default import get_socket_path, AbstractManager, get_config
|
||||
from bgpranking.helpers import get_ipasn, sanity_check_ipasn
|
||||
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
|
@ -13,12 +20,107 @@ class DBInsertManager(AbstractManager):
|
|||
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
super().__init__(loglevel)
|
||||
self.dbinsert = DatabaseInsert(loglevel)
|
||||
self.script_name = 'db_insert'
|
||||
self.kvrocks_storage = Redis(get_config('generic', 'storage_db_hostname'), get_config('generic', 'storage_db_port'), decode_responses=True)
|
||||
self.redis_sanitized = Redis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
|
||||
self.ipasn = get_ipasn()
|
||||
self.logger.debug('Starting import')
|
||||
|
||||
def _to_run_forever(self):
|
||||
self.dbinsert.insert()
|
||||
ready, message = sanity_check_ipasn(self.ipasn)
|
||||
if not ready:
|
||||
# Try again later.
|
||||
self.logger.warning(message)
|
||||
return
|
||||
self.logger.debug(message)
|
||||
|
||||
while True:
|
||||
if self.shutdown_requested():
|
||||
break
|
||||
try:
|
||||
if not self.ipasn.is_up:
|
||||
break
|
||||
except Exception:
|
||||
self.logger.warning('Unable to query ipasnhistory')
|
||||
time.sleep(10)
|
||||
continue
|
||||
uuids: List[str] = self.redis_sanitized.spop('to_insert', 100) # type: ignore
|
||||
if not uuids:
|
||||
break
|
||||
p = self.redis_sanitized.pipeline(transaction=False)
|
||||
[p.hgetall(uuid) for uuid in uuids]
|
||||
sanitized_data = p.execute()
|
||||
|
||||
for_query = []
|
||||
for i, uuid in enumerate(uuids):
|
||||
data = sanitized_data[i]
|
||||
if not data:
|
||||
self.logger.warning(f'No data for UUID {uuid}. This should not happen, but lets move on.')
|
||||
continue
|
||||
for_query.append({'ip': data['ip'], 'address_family': data['address_family'], 'source': 'caida',
|
||||
'date': data['datetime'], 'precision_delta': {'days': 3}})
|
||||
try:
|
||||
responses = self.ipasn.mass_query(for_query)
|
||||
except Exception:
|
||||
self.logger.exception('Mass query in IPASN History failed, trying again later.')
|
||||
# Rollback the spop
|
||||
self.redis_sanitized.sadd('to_insert', *uuids)
|
||||
time.sleep(10)
|
||||
continue
|
||||
retry = []
|
||||
done = []
|
||||
ardb_pipeline = self.kvrocks_storage.pipeline(transaction=False)
|
||||
for i, uuid in enumerate(uuids):
|
||||
data = sanitized_data[i]
|
||||
if not data:
|
||||
self.logger.warning(f'No data for UUID {uuid}. This should not happen, but lets move on.')
|
||||
continue
|
||||
routing_info = responses['responses'][i]['response'] # our queries are on one single date, not a range
|
||||
# Data gathered from IPASN History:
|
||||
# * IP Block of the IP
|
||||
# * AS number
|
||||
if 'error' in routing_info:
|
||||
self.logger.warning(f"Unable to find routing information for {data['ip']} - {data['datetime']}: {routing_info['error']}")
|
||||
continue
|
||||
# Single date query, getting from the object
|
||||
datetime_routing = list(routing_info.keys())[0]
|
||||
entry = routing_info[datetime_routing]
|
||||
if not entry:
|
||||
# routing info is missing, need to try again later.
|
||||
retry.append(uuid)
|
||||
continue
|
||||
if 'asn' in entry and entry['asn'] is None:
|
||||
self.logger.warning(f"Unable to find the AS number associated to {data['ip']} - {data['datetime']} (got None). This should not happen...")
|
||||
continue
|
||||
if 'prefix' in entry and entry['prefix'] is None:
|
||||
self.logger.warning(f"Unable to find the prefix associated to {data['ip']} - {data['datetime']} (got None). This should not happen...")
|
||||
continue
|
||||
|
||||
# Format: <YYYY-MM-DD>|sources -> set([<source>, ...])
|
||||
ardb_pipeline.sadd(f"{data['date']}|sources", data['source'])
|
||||
|
||||
# Format: <YYYY-MM-DD>|<source> -> set([<asn>, ...])
|
||||
ardb_pipeline.sadd(f"{data['date']}|{data['source']}", entry['asn'])
|
||||
# Format: <YYYY-MM-DD>|<source>|<asn> -> set([<prefix>, ...])
|
||||
ardb_pipeline.sadd(f"{data['date']}|{data['source']}|{entry['asn']}", entry['prefix'])
|
||||
|
||||
# Format: <YYYY-MM-DD>|<source>|<asn>|<prefix> -> set([<ip>|<datetime>, ...])
|
||||
ardb_pipeline.sadd(f"{data['date']}|{data['source']}|{entry['asn']}|{entry['prefix']}",
|
||||
f"{data['ip']}|{data['datetime']}")
|
||||
done.append(uuid)
|
||||
ardb_pipeline.execute()
|
||||
p = self.redis_sanitized.pipeline(transaction=False)
|
||||
if done:
|
||||
p.delete(*done)
|
||||
if retry:
|
||||
p.sadd('to_insert', *retry)
|
||||
p.execute()
|
||||
|
||||
|
||||
def main():
|
||||
dbinsert = DBInsertManager()
|
||||
dbinsert.run(sleep_in_sec=120)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dbinsert = DBInsertManager()
|
||||
dbinsert.run(sleep_in_sec=120)
|
||||
main()
|
||||
|
|
|
|||
181
bin/fetcher.py
181
bin/fetcher.py
|
|
@ -1,50 +1,177 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import json
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import aiohttp
|
||||
import logging
|
||||
|
||||
from datetime import datetime, date
|
||||
from hashlib import sha512 # Faster than sha256 on 64b machines.
|
||||
from logging import Logger
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from dateutil import parser
|
||||
from pid import PidFile, PidFileError # type: ignore
|
||||
|
||||
from bgpranking.default import AbstractManager, safe_create_dir
|
||||
from bgpranking.helpers import get_modules, get_data_dir, get_modules_dir
|
||||
|
||||
from bgpranking.abstractmanager import AbstractManager
|
||||
from bgpranking.modulesfetcher import Fetcher
|
||||
from bgpranking.libs.helpers import get_config_path, get_homedir
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
||||
|
||||
class Fetcher():
|
||||
|
||||
def __init__(self, config_file: Path, logger: Logger):
|
||||
'''Load `config_file`, and store the fetched data into `storage_directory`
|
||||
Note: if the `config_file` does not provide a URL (the file is
|
||||
gathered by some oter mean), the fetcher is automatically stoped.'''
|
||||
with open(config_file, 'r') as f:
|
||||
module_parameters = json.load(f)
|
||||
self.vendor = module_parameters['vendor']
|
||||
self.listname = module_parameters['name']
|
||||
self.logger = logger
|
||||
self.fetcher = True
|
||||
if 'url' not in module_parameters:
|
||||
self.logger.info(f'{self.vendor}-{self.listname}: No URL to fetch, breaking.')
|
||||
self.fetcher = False
|
||||
return
|
||||
self.url = module_parameters['url']
|
||||
self.logger.debug(f'{self.vendor}-{self.listname}: Starting fetcher on {self.url}')
|
||||
self.directory = get_data_dir() / self.vendor / self.listname
|
||||
safe_create_dir(self.directory)
|
||||
self.meta = self.directory / 'meta'
|
||||
safe_create_dir(self.meta)
|
||||
self.archive_dir = self.directory / 'archive'
|
||||
safe_create_dir(self.archive_dir)
|
||||
self.first_fetch = True
|
||||
|
||||
async def __get_last_modified(self):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.head(self.url) as r:
|
||||
headers = r.headers
|
||||
if 'Last-Modified' in headers:
|
||||
return parser.parse(headers['Last-Modified'])
|
||||
return None
|
||||
|
||||
async def __newer(self):
|
||||
'''Check if the file available for download is newed than the one
|
||||
already downloaded by checking the `Last-Modified` header.
|
||||
Note: return False if the file containing the last header content
|
||||
is not existing, or the header doesn't have this key.
|
||||
'''
|
||||
last_modified_path = self.meta / 'lastmodified'
|
||||
if not last_modified_path.exists():
|
||||
# The file doesn't exists
|
||||
if not self.first_fetch:
|
||||
# The URL has no Last-Modified header, we cannot use it.
|
||||
self.logger.debug(f'{self.vendor}-{self.listname}: No Last-Modified header available')
|
||||
return True
|
||||
self.first_fetch = False
|
||||
last_modified = await self.__get_last_modified()
|
||||
if last_modified:
|
||||
self.logger.debug(f'{self.vendor}-{self.listname}: Last-Modified header available')
|
||||
with last_modified_path.open('w') as f:
|
||||
f.write(last_modified.isoformat())
|
||||
else:
|
||||
self.logger.debug(f'{self.vendor}-{self.listname}: No Last-Modified header available')
|
||||
return True
|
||||
with last_modified_path.open() as f:
|
||||
file_content = f.read()
|
||||
last_modified_file = parser.parse(file_content)
|
||||
last_modified = await self.__get_last_modified()
|
||||
if not last_modified:
|
||||
# No more Last-Modified header Oo
|
||||
self.logger.warning(f'{self.vendor}-{self.listname}: Last-Modified header was present, isn\'t anymore!')
|
||||
last_modified_path.unlink()
|
||||
return True
|
||||
if last_modified > last_modified_file:
|
||||
self.logger.info(f'{self.vendor}-{self.listname}: Got a new file.')
|
||||
with last_modified_path.open('w') as f:
|
||||
f.write(last_modified.isoformat())
|
||||
return True
|
||||
return False
|
||||
|
||||
def __same_as_last(self, downloaded):
|
||||
'''Figure out the last downloaded file, check if it is the same as the
|
||||
newly downloaded one. Returns true if both files have been downloaded the
|
||||
same day.
|
||||
Note: we check the new and the archive directory because we may have backlog
|
||||
and the newest file is always the first one we process
|
||||
'''
|
||||
to_check = []
|
||||
to_check_new = sorted([f for f in self.directory.iterdir() if f.is_file()])
|
||||
if to_check_new:
|
||||
# we have files waiting to be processed
|
||||
self.logger.debug(f'{self.vendor}-{self.listname}: {len(to_check_new)} file(s) are waiting to be processed')
|
||||
to_check.append(to_check_new[-1])
|
||||
to_check_archive = sorted([f for f in self.archive_dir.iterdir() if f.is_file()])
|
||||
if to_check_archive:
|
||||
# we have files already processed, in the archive
|
||||
self.logger.debug(f'{self.vendor}-{self.listname}: {len(to_check_archive)} file(s) have been processed')
|
||||
to_check.append(to_check_archive[-1])
|
||||
if not to_check:
|
||||
self.logger.debug(f'{self.vendor}-{self.listname}: New list, no hisorical files')
|
||||
# nothing has been downloaded ever, moving on
|
||||
return False
|
||||
dl_hash = sha512(downloaded)
|
||||
for last_file in to_check:
|
||||
with last_file.open('rb') as f:
|
||||
last_hash = sha512(f.read())
|
||||
if (dl_hash.digest() == last_hash.digest()
|
||||
and parser.parse(last_file.name.split('.')[0]).date() == date.today()):
|
||||
self.logger.debug(f'{self.vendor}-{self.listname}: Same file already downloaded today.')
|
||||
return True
|
||||
return False
|
||||
|
||||
async def fetch_list(self):
|
||||
'''Fetch & store the list'''
|
||||
if not self.fetcher:
|
||||
return
|
||||
try:
|
||||
with PidFile(f'{self.listname}.pid', piddir=self.meta):
|
||||
if not await self.__newer():
|
||||
return
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(self.url) as r:
|
||||
content = await r.content.read()
|
||||
if self.__same_as_last(content):
|
||||
return
|
||||
self.logger.info(f'{self.vendor}-{self.listname}: Got a new file!')
|
||||
with (self.directory / '{}.txt'.format(datetime.now().isoformat())).open('wb') as f:
|
||||
f.write(content)
|
||||
except PidFileError:
|
||||
self.logger.info(f'{self.vendor}-{self.listname}: Fetcher already running')
|
||||
|
||||
|
||||
class ModulesManager(AbstractManager):
|
||||
|
||||
def __init__(self, config_dir: Path=None, storage_directory: Path=None, loglevel: int=logging.DEBUG):
|
||||
def __init__(self, loglevel: int=logging.DEBUG):
|
||||
super().__init__(loglevel)
|
||||
if not config_dir:
|
||||
config_dir = get_config_path()
|
||||
if not storage_directory:
|
||||
self.storage_directory = get_homedir() / 'rawdata'
|
||||
self.modules_config = config_dir / 'modules'
|
||||
self.modules_paths = [modulepath for modulepath in self.modules_config.glob('*.json')]
|
||||
self.modules = [Fetcher(path, self.storage_directory, loglevel) for path in self.modules_paths]
|
||||
self.script_name = 'modules_manager'
|
||||
self.modules_paths = get_modules()
|
||||
self.modules = [Fetcher(path, self.logger) for path in self.modules_paths]
|
||||
|
||||
def _to_run_forever(self):
|
||||
async def _to_run_forever_async(self):
|
||||
# Check if there are new config files
|
||||
new_modules_paths = [modulepath for modulepath in self.modules_config.glob('*.json') if modulepath not in self.modules_paths]
|
||||
self.modules += [Fetcher(path, self.storage_directory, self.loglevel) for path in new_modules_paths]
|
||||
new_modules_paths = [modulepath for modulepath in get_modules_dir().glob('*.json') if modulepath not in self.modules_paths]
|
||||
self.modules += [Fetcher(path, self.logger) for path in new_modules_paths]
|
||||
self.modules_paths += new_modules_paths
|
||||
|
||||
if self.modules:
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop.run_until_complete(asyncio.gather(
|
||||
*[module.fetch_list() for module in self.modules if module.fetcher],
|
||||
return_exceptions=True)
|
||||
)
|
||||
except aiohttp.client_exceptions.ClientConnectorError as e:
|
||||
self.logger.critical(f'Exception while fetching lists: {e}')
|
||||
for module in self.modules:
|
||||
if module.fetcher:
|
||||
await module.fetch_list()
|
||||
else:
|
||||
self.logger.info('No config files were found so there are no fetchers running yet. Will try again later.')
|
||||
|
||||
|
||||
def main():
|
||||
m = ModulesManager()
|
||||
asyncio.run(m.run_async(sleep_in_sec=3600))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
modules_manager = ModulesManager()
|
||||
modules_manager.run(sleep_in_sec=3600)
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ import logging
|
|||
from dateutil.parser import parse
|
||||
from datetime import timedelta
|
||||
|
||||
from bgpranking.libs.helpers import load_config_files
|
||||
from bgpranking.ranking import Ranking
|
||||
from bgpranking.helpers import load_all_modules_configs
|
||||
from .ranking import Ranking
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.DEBUG, datefmt='%I:%M:%S')
|
||||
|
|
@ -21,13 +21,13 @@ if __name__ == '__main__':
|
|||
args = parser.parse_args()
|
||||
|
||||
ranking = Ranking(loglevel=logging.DEBUG)
|
||||
config_files = load_config_files()
|
||||
config_files = load_all_modules_configs()
|
||||
if args.day:
|
||||
day = parse(args.day).date().isoformat()
|
||||
ranking.rank_a_day(day, config_files)
|
||||
ranking.rank_a_day(day)
|
||||
else:
|
||||
current = parse(args.interval[1]).date()
|
||||
stop_date = parse(args.interval[0]).date()
|
||||
while current >= stop_date:
|
||||
ranking.rank_a_day(current.isoformat(), config_files)
|
||||
ranking.rank_a_day(current.isoformat())
|
||||
current -= timedelta(days=1)
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from bgpranking.monitor import Monitor
|
||||
import logging
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
||||
|
||||
class MonitorManager():
|
||||
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
self.monitor = Monitor()
|
||||
|
||||
def get_values(self):
|
||||
return self.monitor.get_values()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
m = MonitorManager()
|
||||
print(m.get_values())
|
||||
115
bin/parser.py
115
bin/parser.py
|
|
@ -1,41 +1,124 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import re
|
||||
import types
|
||||
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from logging import Logger
|
||||
from pathlib import Path
|
||||
from typing import List, Union, Tuple
|
||||
from uuid import uuid4
|
||||
|
||||
from redis import Redis
|
||||
|
||||
from bgpranking.default import AbstractManager, safe_create_dir, get_socket_path
|
||||
from bgpranking.helpers import get_modules, get_data_dir, get_modules_dir
|
||||
|
||||
from bgpranking.abstractmanager import AbstractManager
|
||||
from bgpranking.parser import RawFilesParser
|
||||
from bgpranking.libs.helpers import get_config_path, get_homedir
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
||||
|
||||
class RawFilesParser():
|
||||
|
||||
def __init__(self, config_file: Path, logger: Logger) -> None:
|
||||
self.logger = logger
|
||||
with open(config_file, 'r') as f:
|
||||
module_parameters = json.load(f)
|
||||
self.vendor = module_parameters['vendor']
|
||||
self.listname = module_parameters['name']
|
||||
if 'parser' in module_parameters:
|
||||
self.parse_raw_file = types.MethodType(importlib.import_module(module_parameters['parser'], 'bgpranking').parse_raw_file, self) # type: ignore
|
||||
self.source = f'{self.vendor}-{self.listname}'
|
||||
self.directory = get_data_dir() / self.vendor / self.listname
|
||||
safe_create_dir(self.directory)
|
||||
self.unparsable_dir = self.directory / 'unparsable'
|
||||
safe_create_dir(self.unparsable_dir)
|
||||
self.redis_intake = Redis(unix_socket_path=get_socket_path('intake'), db=0)
|
||||
self.logger.debug(f'{self.source}: Starting intake.')
|
||||
|
||||
@property
|
||||
def files_to_parse(self) -> List[Path]:
|
||||
return sorted([f for f in self.directory.iterdir() if f.is_file()], reverse=True)
|
||||
|
||||
def extract_ipv4(self, bytestream: bytes) -> List[Union[bytes, Tuple[bytes, datetime]]]:
|
||||
return re.findall(rb'[0-9]+(?:\.[0-9]+){3}', bytestream)
|
||||
|
||||
def strip_leading_zeros(self, ips: List[bytes]) -> List[bytes]:
|
||||
'''Helper to get rid of leading 0s in an IP list.
|
||||
Only run it when needed, it is nasty and slow'''
|
||||
return ['.'.join(str(int(part)) for part in ip.split(b'.')).encode() for ip in ips]
|
||||
|
||||
def parse_raw_file(self, f: BytesIO) -> List[Union[bytes, Tuple[bytes, datetime]]]:
|
||||
# If the list doesn't provide a time, fallback to current day, midnight
|
||||
self.datetime = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
return self.extract_ipv4(f.getvalue())
|
||||
|
||||
def parse_raw_files(self) -> None:
|
||||
nb_unparsable_files = len([f for f in self.unparsable_dir.iterdir() if f.is_file()])
|
||||
if nb_unparsable_files:
|
||||
self.logger.warning(f'{self.source}: Was unable to parse {nb_unparsable_files} files.')
|
||||
try:
|
||||
for filepath in self.files_to_parse:
|
||||
self.logger.debug(f'{self.source}: Parsing {filepath}, {len(self.files_to_parse) - 1} to go.')
|
||||
with open(filepath, 'rb') as f:
|
||||
to_parse = BytesIO(f.read())
|
||||
p = self.redis_intake.pipeline()
|
||||
for line in self.parse_raw_file(to_parse):
|
||||
if isinstance(line, tuple):
|
||||
ip, datetime = line
|
||||
else:
|
||||
ip = line
|
||||
datetime = self.datetime
|
||||
uuid = uuid4()
|
||||
p.hmset(str(uuid), {'ip': ip, 'source': self.source,
|
||||
'datetime': datetime.isoformat()})
|
||||
p.sadd('intake', str(uuid))
|
||||
p.execute()
|
||||
self._archive(filepath)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"{self.source}: That didn't go well: {e}")
|
||||
self._unparsable(filepath)
|
||||
|
||||
def _archive(self, filepath: Path) -> None:
|
||||
'''After processing, move file to the archive directory'''
|
||||
filepath.rename(self.directory / 'archive' / filepath.name)
|
||||
|
||||
def _unparsable(self, filepath: Path) -> None:
|
||||
'''After processing, move file to the archive directory'''
|
||||
filepath.rename(self.unparsable_dir / filepath.name)
|
||||
|
||||
|
||||
class ParserManager(AbstractManager):
|
||||
|
||||
def __init__(self, config_dir: Path=None, storage_directory: Path=None, loglevel: int=logging.DEBUG):
|
||||
def __init__(self, loglevel: int=logging.DEBUG):
|
||||
super().__init__(loglevel)
|
||||
if not config_dir:
|
||||
config_dir = get_config_path()
|
||||
if not storage_directory:
|
||||
self.storage_directory = get_homedir() / 'rawdata'
|
||||
self.modules_config = config_dir / 'modules'
|
||||
self.modules_paths = [modulepath for modulepath in self.modules_config.glob('*.json')]
|
||||
self.modules = [RawFilesParser(path, self.storage_directory, loglevel) for path in self.modules_paths]
|
||||
self.script_name = 'parser'
|
||||
self.modules_paths = get_modules()
|
||||
self.modules = [RawFilesParser(path, self.logger) for path in self.modules_paths]
|
||||
|
||||
def _to_run_forever(self):
|
||||
# Check if there are new config files
|
||||
new_modules_paths = [modulepath for modulepath in self.modules_config.glob('*.json') if modulepath not in self.modules_paths]
|
||||
self.modules += [RawFilesParser(path, self.storage_directory, self.loglevel) for path in new_modules_paths]
|
||||
new_modules_paths = [modulepath for modulepath in get_modules_dir().glob('*.json') if modulepath not in self.modules_paths]
|
||||
self.modules += [RawFilesParser(path, self.logger) for path in new_modules_paths]
|
||||
self.modules_paths += new_modules_paths
|
||||
|
||||
if self.modules:
|
||||
[module.parse_raw_files() for module in self.modules]
|
||||
for module in self.modules:
|
||||
module.parse_raw_files()
|
||||
else:
|
||||
self.logger.warning('No config files were found so there are no parsers running yet. Will try again later.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
def main():
|
||||
parser_manager = ParserManager()
|
||||
parser_manager.run(sleep_in_sec=120)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
125
bin/ranking.py
125
bin/ranking.py
|
|
@ -2,24 +2,131 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
from bgpranking.abstractmanager import AbstractManager
|
||||
from bgpranking.ranking import Ranking
|
||||
from pathlib import Path
|
||||
|
||||
from datetime import datetime, date, timedelta
|
||||
from ipaddress import ip_network
|
||||
from typing import Dict, Any
|
||||
|
||||
from redis import Redis
|
||||
import requests
|
||||
|
||||
from bgpranking.default import AbstractManager, get_config
|
||||
from bgpranking.helpers import get_ipasn, sanity_check_ipasn, load_all_modules_configs
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
||||
|
||||
class RankingManager(AbstractManager):
|
||||
class Ranking(AbstractManager):
|
||||
|
||||
def __init__(self, config_dir: Path=None, loglevel: int=logging.INFO):
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
super().__init__(loglevel)
|
||||
self.ranking = Ranking(config_dir, loglevel)
|
||||
self.script_name = 'ranking'
|
||||
self.storage = Redis(get_config('generic', 'storage_db_hostname'), get_config('generic', 'storage_db_port'), decode_responses=True)
|
||||
self.ranking = Redis(get_config('generic', 'ranking_db_hostname'), get_config('generic', 'ranking_db_port'), decode_responses=True)
|
||||
self.ipasn = get_ipasn()
|
||||
|
||||
def rank_a_day(self, day: str):
|
||||
asns_aggregation_key_v4 = f'{day}|asns|v4'
|
||||
asns_aggregation_key_v6 = f'{day}|asns|v6'
|
||||
to_delete = set([asns_aggregation_key_v4, asns_aggregation_key_v6])
|
||||
r_pipeline = self.ranking.pipeline()
|
||||
cached_meta: Dict[str, Dict[str, Any]] = {}
|
||||
config_files = load_all_modules_configs()
|
||||
for source in self.storage.smembers(f'{day}|sources'):
|
||||
self.logger.info(f'{day} - Ranking source: {source}')
|
||||
source_aggregation_key_v4 = f'{day}|{source}|asns|v4'
|
||||
source_aggregation_key_v6 = f'{day}|{source}|asns|v6'
|
||||
to_delete.update([source_aggregation_key_v4, source_aggregation_key_v6])
|
||||
for asn in self.storage.smembers(f'{day}|{source}'):
|
||||
prefixes_aggregation_key_v4 = f'{day}|{asn}|v4'
|
||||
prefixes_aggregation_key_v6 = f'{day}|{asn}|v6'
|
||||
to_delete.update([prefixes_aggregation_key_v4, prefixes_aggregation_key_v6])
|
||||
if asn == '0':
|
||||
# Default ASN when no matches. Probably spoofed.
|
||||
continue
|
||||
self.logger.debug(f'{day} - Ranking source: {source} / ASN: {asn}')
|
||||
asn_rank_v4 = 0.0
|
||||
asn_rank_v6 = 0.0
|
||||
for prefix in self.storage.smembers(f'{day}|{source}|{asn}'):
|
||||
if prefix == 'None':
|
||||
# This should not happen and requires a DB cleanup.
|
||||
self.logger.critical(f'Fucked up prefix in "{day}|{source}|{asn}"')
|
||||
continue
|
||||
ips = set([ip_ts.split('|')[0]
|
||||
for ip_ts in self.storage.smembers(f'{day}|{source}|{asn}|{prefix}')])
|
||||
py_prefix = ip_network(prefix)
|
||||
prefix_rank = float(len(ips)) / py_prefix.num_addresses
|
||||
r_pipeline.zadd(f'{day}|{source}|{asn}|v{py_prefix.version}|prefixes', {prefix: prefix_rank})
|
||||
if py_prefix.version == 4:
|
||||
asn_rank_v4 += len(ips) * config_files[source]['impact']
|
||||
r_pipeline.zincrby(prefixes_aggregation_key_v4, prefix_rank * config_files[source]['impact'], prefix)
|
||||
else:
|
||||
asn_rank_v6 += len(ips) * config_files[source]['impact']
|
||||
r_pipeline.zincrby(prefixes_aggregation_key_v6, prefix_rank * config_files[source]['impact'], prefix)
|
||||
if asn in cached_meta:
|
||||
v4info = cached_meta[asn]['v4']
|
||||
v6info = cached_meta[asn]['v6']
|
||||
else:
|
||||
retry = 3
|
||||
while retry:
|
||||
try:
|
||||
v4info = self.ipasn.asn_meta(asn=asn, source='caida', address_family='v4', date=day)
|
||||
v6info = self.ipasn.asn_meta(asn=asn, source='caida', address_family='v6', date=day)
|
||||
break
|
||||
except requests.exceptions.ConnectionError:
|
||||
# Sometimes, ipasnhistory is unreachable try again a few times
|
||||
retry -= 1
|
||||
else:
|
||||
# if it keeps failing, the ASN will be ranked on next run.
|
||||
continue
|
||||
|
||||
cached_meta[asn] = {'v4': v4info, 'v6': v6info}
|
||||
ipasnhistory_date_v4 = list(v4info['response'].keys())[0]
|
||||
v4count = v4info['response'][ipasnhistory_date_v4][asn]['ipcount']
|
||||
ipasnhistory_date_v6 = list(v6info['response'].keys())[0]
|
||||
v6count = v6info['response'][ipasnhistory_date_v6][asn]['ipcount']
|
||||
if v4count:
|
||||
asn_rank_v4 /= float(v4count)
|
||||
if asn_rank_v4:
|
||||
r_pipeline.set(f'{day}|{source}|{asn}|v4', asn_rank_v4)
|
||||
r_pipeline.zincrby(asns_aggregation_key_v4, asn_rank_v4, asn)
|
||||
r_pipeline.zadd(source_aggregation_key_v4, {asn: asn_rank_v4})
|
||||
if v6count:
|
||||
asn_rank_v6 /= float(v6count)
|
||||
if asn_rank_v6:
|
||||
r_pipeline.set(f'{day}|{source}|{asn}|v6', asn_rank_v6)
|
||||
r_pipeline.zincrby(asns_aggregation_key_v6, asn_rank_v6, asn)
|
||||
r_pipeline.zadd(source_aggregation_key_v6, {asn: asn_rank_v6})
|
||||
self.ranking.delete(*to_delete)
|
||||
r_pipeline.execute()
|
||||
|
||||
def compute(self):
|
||||
ready, message = sanity_check_ipasn(self.ipasn)
|
||||
if not ready:
|
||||
# Try again later.
|
||||
self.logger.warning(message)
|
||||
return
|
||||
self.logger.debug(message)
|
||||
|
||||
self.logger.info('Start ranking')
|
||||
today = date.today()
|
||||
now = datetime.now()
|
||||
today12am = now.replace(hour=12, minute=0, second=0, microsecond=0)
|
||||
if now < today12am:
|
||||
# Compute yesterday and today's ranking (useful when we have lists generated only once a day)
|
||||
self.rank_a_day((today - timedelta(days=1)).isoformat())
|
||||
self.rank_a_day(today.isoformat())
|
||||
self.logger.info('Ranking done.')
|
||||
|
||||
def _to_run_forever(self):
|
||||
self.ranking.compute()
|
||||
self.compute()
|
||||
|
||||
|
||||
def main():
|
||||
ranking = Ranking()
|
||||
ranking.run(sleep_in_sec=3600)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
ranking = RankingManager()
|
||||
ranking.run(sleep_in_sec=3600)
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,79 +1,120 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from bgpranking.libs.helpers import get_homedir, check_running
|
||||
from subprocess import Popen
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from subprocess import Popen
|
||||
from typing import Optional, Dict
|
||||
|
||||
import argparse
|
||||
from redis import Redis
|
||||
from redis.exceptions import ConnectionError
|
||||
|
||||
from bgpranking.default import get_homedir, get_socket_path, get_config
|
||||
|
||||
|
||||
def launch_cache(storage_directory: Path=None):
|
||||
def check_running(name: str) -> bool:
|
||||
if name == "storage":
|
||||
r = Redis(get_config('generic', 'storage_db_hostname'), get_config('generic', 'storage_db_port'))
|
||||
elif name == "ranking":
|
||||
r = Redis(get_config('generic', 'ranking_db_hostname'), get_config('generic', 'ranking_db_port'))
|
||||
else:
|
||||
socket_path = get_socket_path(name)
|
||||
if not os.path.exists(socket_path):
|
||||
return False
|
||||
r = Redis(unix_socket_path=socket_path)
|
||||
try:
|
||||
return True if r.ping() else False
|
||||
except ConnectionError:
|
||||
return False
|
||||
|
||||
|
||||
def launch_cache(storage_directory: Optional[Path]=None):
|
||||
if not storage_directory:
|
||||
storage_directory = get_homedir()
|
||||
if not check_running('cache'):
|
||||
Popen(["./run_redis.sh"], cwd=(storage_directory / 'cache'))
|
||||
|
||||
|
||||
def shutdown_cache(storage_directory: Path=None):
|
||||
def shutdown_cache(storage_directory: Optional[Path]=None):
|
||||
if not storage_directory:
|
||||
storage_directory = get_homedir()
|
||||
Popen(["./shutdown_redis.sh"], cwd=(storage_directory / 'cache'))
|
||||
r = Redis(unix_socket_path=get_socket_path('cache'))
|
||||
r.shutdown(save=True)
|
||||
print('Redis cache database shutdown.')
|
||||
|
||||
|
||||
def launch_temp(storage_directory: Path=None):
|
||||
def launch_temp(storage_directory: Optional[Path]=None):
|
||||
if not storage_directory:
|
||||
storage_directory = get_homedir()
|
||||
if not check_running('intake') and not check_running('prepare'):
|
||||
Popen(["./run_redis.sh"], cwd=(storage_directory / 'temp'))
|
||||
|
||||
|
||||
def shutdown_temp(storage_directory: Path=None):
|
||||
def shutdown_temp(storage_directory: Optional[Path]=None):
|
||||
if not storage_directory:
|
||||
storage_directory = get_homedir()
|
||||
Popen(["./shutdown_redis.sh"], cwd=(storage_directory / 'temp'))
|
||||
r = Redis(unix_socket_path=get_socket_path('intake'))
|
||||
r.shutdown(save=True)
|
||||
print('Redis intake database shutdown.')
|
||||
r = Redis(unix_socket_path=get_socket_path('prepare'))
|
||||
r.shutdown(save=True)
|
||||
print('Redis prepare database shutdown.')
|
||||
|
||||
|
||||
def launch_storage(storage_directory: Path=None):
|
||||
def launch_storage(storage_directory: Optional[Path]=None):
|
||||
if not storage_directory:
|
||||
storage_directory = get_homedir()
|
||||
if not check_running('storage'):
|
||||
Popen(["./run_ardb.sh"], cwd=(storage_directory / 'storage'))
|
||||
Popen(["./run_kvrocks.sh"], cwd=(storage_directory / 'storage'))
|
||||
|
||||
|
||||
def shutdown_storage(storage_directory: Path=None):
|
||||
def shutdown_storage(storage_directory: Optional[Path]=None):
|
||||
redis = Redis(get_config('generic', 'storage_db_hostname'), get_config('generic', 'storage_db_port'))
|
||||
redis.shutdown()
|
||||
|
||||
|
||||
def launch_ranking(storage_directory: Optional[Path]=None):
|
||||
if not storage_directory:
|
||||
storage_directory = get_homedir()
|
||||
Popen(["./shutdown_ardb.sh"], cwd=(storage_directory / 'storage'))
|
||||
if not check_running('ranking'):
|
||||
Popen(["./run_kvrocks.sh"], cwd=(storage_directory / 'ranking'))
|
||||
|
||||
|
||||
def shutdown_ranking(storage_directory: Optional[Path]=None):
|
||||
redis = Redis(get_config('generic', 'ranking_db_hostname'), get_config('generic', 'ranking_db_port'))
|
||||
redis.shutdown()
|
||||
|
||||
|
||||
def launch_all():
|
||||
launch_cache()
|
||||
launch_temp()
|
||||
launch_storage()
|
||||
launch_ranking()
|
||||
|
||||
|
||||
def check_all(stop=False):
|
||||
backends = [['cache', False], ['storage', False],
|
||||
['intake', False], ['prepare', False]]
|
||||
def check_all(stop: bool=False):
|
||||
backends: Dict[str, bool] = {'cache': False, 'storage': False, 'ranking': False,
|
||||
'intake': False, 'prepare': False}
|
||||
while True:
|
||||
for b in backends:
|
||||
for db_name in backends.keys():
|
||||
print(backends[db_name])
|
||||
try:
|
||||
b[1] = check_running(b[0])
|
||||
backends[db_name] = check_running(db_name)
|
||||
except Exception:
|
||||
b[1] = False
|
||||
backends[db_name] = False
|
||||
if stop:
|
||||
if not any(b[1] for b in backends):
|
||||
if not any(running for running in backends.values()):
|
||||
break
|
||||
else:
|
||||
if all(b[1] for b in backends):
|
||||
if all(running for running in backends.values()):
|
||||
break
|
||||
for b in backends:
|
||||
if not stop and not b[1]:
|
||||
print(f"Waiting on {b[0]}")
|
||||
if stop and b[1]:
|
||||
print(f"Waiting on {b[0]}")
|
||||
for db_name, running in backends.items():
|
||||
if not stop and not running:
|
||||
print(f"Waiting on {db_name} to start")
|
||||
if stop and running:
|
||||
print(f"Waiting on {db_name} to stop")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
|
|
@ -81,9 +122,10 @@ def stop_all():
|
|||
shutdown_cache()
|
||||
shutdown_temp()
|
||||
shutdown_storage()
|
||||
shutdown_ranking()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Manage backend DBs.')
|
||||
parser.add_argument("--start", action='store_true', default=False, help="Start all")
|
||||
parser.add_argument("--stop", action='store_true', default=False, help="Stop all")
|
||||
|
|
@ -96,3 +138,7 @@ if __name__ == '__main__':
|
|||
stop_all()
|
||||
if not args.stop and args.status:
|
||||
check_all()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
100
bin/sanitizer.py
100
bin/sanitizer.py
|
|
@ -1,25 +1,109 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import time
|
||||
|
||||
from bgpranking.abstractmanager import AbstractManager
|
||||
from bgpranking.sanitizer import Sanitizer
|
||||
from datetime import timezone
|
||||
from typing import Optional, List
|
||||
|
||||
from dateutil import parser
|
||||
from redis import Redis
|
||||
import requests
|
||||
|
||||
from bgpranking.default import AbstractManager, get_socket_path
|
||||
from bgpranking.helpers import get_ipasn, sanity_check_ipasn
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.WARNING, datefmt='%I:%M:%S')
|
||||
|
||||
|
||||
class SanitizerManager(AbstractManager):
|
||||
class Sanitizer(AbstractManager):
|
||||
|
||||
def __init__(self, loglevel: int=logging.WARNING):
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
super().__init__(loglevel)
|
||||
self.sanitizer = Sanitizer(loglevel)
|
||||
self.script_name = 'sanitizer'
|
||||
self.redis_intake = Redis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
|
||||
self.redis_sanitized = Redis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
|
||||
self.ipasn = get_ipasn()
|
||||
self.logger.debug('Starting import')
|
||||
|
||||
def sanitize(self):
|
||||
ready, message = sanity_check_ipasn(self.ipasn)
|
||||
if not ready:
|
||||
# Try again later.
|
||||
self.logger.warning(message)
|
||||
return
|
||||
self.logger.debug(message)
|
||||
|
||||
while True:
|
||||
try:
|
||||
if self.shutdown_requested() or not self.ipasn.is_up:
|
||||
break
|
||||
except requests.exceptions.ConnectionError:
|
||||
# Temporary issue with ipasnhistory
|
||||
self.logger.info('Temporary issue with ipasnhistory, trying again later.')
|
||||
time.sleep(10)
|
||||
continue
|
||||
uuids: Optional[List[str]] = self.redis_intake.spop('intake', 100) # type: ignore
|
||||
if not uuids:
|
||||
break
|
||||
for_cache = []
|
||||
pipeline = self.redis_sanitized.pipeline(transaction=False)
|
||||
for uuid in uuids:
|
||||
data = self.redis_intake.hgetall(uuid)
|
||||
if not data:
|
||||
continue
|
||||
try:
|
||||
ip = ipaddress.ip_address(data['ip'])
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
address_family = 'v6'
|
||||
else:
|
||||
address_family = 'v4'
|
||||
except ValueError:
|
||||
self.logger.info(f"Invalid IP address: {data['ip']}")
|
||||
continue
|
||||
except KeyError:
|
||||
self.logger.info(f"Invalid entry {data}")
|
||||
continue
|
||||
|
||||
if not ip.is_global:
|
||||
self.logger.info(f"The IP address {data['ip']} is not global")
|
||||
continue
|
||||
|
||||
datetime = parser.parse(data['datetime'])
|
||||
if datetime.tzinfo:
|
||||
# Make sure the datetime isn't TZ aware, and UTC.
|
||||
datetime = datetime.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
for_cache.append({'ip': str(ip), 'address_family': address_family, 'source': 'caida',
|
||||
'date': datetime.isoformat(), 'precision_delta': {'days': 3}})
|
||||
|
||||
# Add to temporay DB for further processing
|
||||
pipeline.hmset(uuid, {'ip': str(ip), 'source': data['source'], 'address_family': address_family,
|
||||
'date': datetime.date().isoformat(), 'datetime': datetime.isoformat()})
|
||||
pipeline.sadd('to_insert', uuid)
|
||||
pipeline.execute()
|
||||
self.redis_intake.delete(*uuids)
|
||||
|
||||
try:
|
||||
# Just cache everything so the lookup scripts can do their thing.
|
||||
self.ipasn.mass_cache(for_cache)
|
||||
except Exception:
|
||||
self.logger.info('Mass cache in IPASN History failed, trying again later.')
|
||||
# Rollback the spop
|
||||
self.redis_intake.sadd('intake', *uuids)
|
||||
break
|
||||
|
||||
def _to_run_forever(self):
|
||||
self.sanitizer.sanitize()
|
||||
self.sanitize()
|
||||
|
||||
|
||||
def main():
|
||||
sanitizer = Sanitizer()
|
||||
sanitizer.run(sleep_in_sec=120)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sanitizer = SanitizerManager()
|
||||
sanitizer.run(sleep_in_sec=120)
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,16 +1,25 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from bgpranking.libs.helpers import is_running, get_socket_path
|
||||
import time
|
||||
from redis import StrictRedis
|
||||
|
||||
if __name__ == '__main__':
|
||||
r = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
r.set('shutdown', 1)
|
||||
from bgpranking.default import AbstractManager
|
||||
|
||||
|
||||
def main():
|
||||
AbstractManager.force_shutdown()
|
||||
time.sleep(5)
|
||||
while True:
|
||||
running = is_running()
|
||||
print(running)
|
||||
try:
|
||||
running = AbstractManager.is_running()
|
||||
except FileNotFoundError:
|
||||
print('Redis is already down.')
|
||||
break
|
||||
if not running:
|
||||
break
|
||||
time.sleep(10)
|
||||
print(running)
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
209
bin/ssfetcher.py
209
bin/ssfetcher.py
|
|
@ -2,49 +2,202 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
from logging import Logger
|
||||
import json
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import aiohttp
|
||||
|
||||
from bgpranking.abstractmanager import AbstractManager
|
||||
from bgpranking.shadowserverfetcher import ShadowServerFetcher
|
||||
from bgpranking.libs.helpers import get_config_path, get_homedir
|
||||
from typing import Tuple, Dict, List, Optional, TypeVar, Any
|
||||
from datetime import datetime, date
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from bs4 import BeautifulSoup # type: ignore
|
||||
from dateutil.parser import parse
|
||||
|
||||
from bgpranking.default import AbstractManager, get_homedir, safe_create_dir
|
||||
from bgpranking.helpers import get_data_dir, get_modules_dir
|
||||
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO, datefmt='%I:%M:%S')
|
||||
|
||||
|
||||
Dates = TypeVar('Dates', datetime, date, str)
|
||||
|
||||
|
||||
class ShadowServerFetcher():
|
||||
|
||||
def __init__(self, user, password, logger: Logger) -> None:
|
||||
self.logger = logger
|
||||
self.storage_directory = get_data_dir()
|
||||
self.config_path_modules = get_modules_dir()
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.index_page = 'https://dl.shadowserver.org/reports/index.php'
|
||||
self.vendor = 'shadowserver'
|
||||
self.known_list_types = ('blacklist', 'botnet', 'cc', 'cisco', 'cwsandbox', 'drone',
|
||||
'microsoft', 'scan', 'sinkhole6', 'sinkhole', 'outdated',
|
||||
'compromised', 'hp', 'darknet', 'ddos')
|
||||
self.first_available_day: date
|
||||
self.last_available_day: date
|
||||
self.available_entries: Dict[str, List[Tuple[str, str]]] = {}
|
||||
|
||||
async def __get_index(self):
|
||||
auth_details = {'user': self.user, 'password': self.password, 'login': 'Login'}
|
||||
async with aiohttp.ClientSession() as s:
|
||||
self.logger.debug('Fetching the index.')
|
||||
async with s.post(self.index_page, data=auth_details) as r:
|
||||
return await r.text()
|
||||
|
||||
async def __build_daily_dict(self):
|
||||
html_index = await self.__get_index()
|
||||
soup = BeautifulSoup(html_index, 'html.parser')
|
||||
treeview = soup.find(id='treemenu1')
|
||||
for y in treeview.select(':scope > li'):
|
||||
year = y.contents[0]
|
||||
for m in y.contents[1].select(':scope > li'):
|
||||
month = m.contents[0]
|
||||
for d in m.contents[1].select(':scope > li'):
|
||||
day = d.contents[0]
|
||||
date = parse(f'{year} {month} {day}').date()
|
||||
self.available_entries[date.isoformat()] = []
|
||||
for a in d.contents[1].find_all('a', href=True):
|
||||
if not self.first_available_day:
|
||||
self.first_available_day = date
|
||||
self.last_available_day = date
|
||||
self.available_entries[date.isoformat()].append((a['href'], a.string))
|
||||
self.logger.debug('Dictionary created.')
|
||||
|
||||
def __normalize_day(self, day: Optional[Dates]=None) -> str:
|
||||
if not day:
|
||||
if not self.last_available_day:
|
||||
raise Exception('Unable to figure out the last available day. You need to run build_daily_dict first')
|
||||
to_return = self.last_available_day
|
||||
else:
|
||||
if isinstance(day, str):
|
||||
to_return = parse(day).date()
|
||||
elif isinstance(day, datetime):
|
||||
to_return = day.date()
|
||||
return to_return.isoformat()
|
||||
|
||||
def __split_name(self, name):
|
||||
type_content, country, list_type = name.split('-')
|
||||
if '_' in type_content:
|
||||
type_content, details_type = type_content.split('_', maxsplit=1)
|
||||
if '_' in details_type:
|
||||
details_type, sub = details_type.split('_', maxsplit=1)
|
||||
return list_type, country, (type_content, details_type, sub)
|
||||
return list_type, country, (type_content, details_type)
|
||||
return list_type, country, (type_content)
|
||||
|
||||
def __check_config(self, filename: str) -> Optional[Path]:
|
||||
self.logger.debug(f'Working on config for {filename}.')
|
||||
config: Dict[str, Any] = {'vendor': 'shadowserver', 'parser': '.parsers.shadowserver'}
|
||||
type_content, _, type_details = self.__split_name(filename)
|
||||
prefix = type_content.split('.')[0]
|
||||
|
||||
if isinstance(type_details, str):
|
||||
main_type = type_details
|
||||
config['name'] = '{}-{}'.format(prefix, type_details)
|
||||
else:
|
||||
main_type = type_details[0]
|
||||
config['name'] = '{}-{}'.format(prefix, '_'.join(type_details))
|
||||
|
||||
if main_type not in self.known_list_types:
|
||||
self.logger.warning(f'Unknown type: {main_type}. Please update the config creator script.')
|
||||
return None
|
||||
|
||||
if main_type == 'blacklist':
|
||||
config['impact'] = 5
|
||||
elif main_type == 'botnet':
|
||||
config['impact'] = 2
|
||||
elif main_type == 'cc':
|
||||
config['impact'] = 5
|
||||
elif main_type == 'cisco':
|
||||
config['impact'] = 3
|
||||
elif main_type == 'cwsandbox':
|
||||
config['impact'] = 5
|
||||
elif main_type == 'drone':
|
||||
config['impact'] = 2
|
||||
elif main_type == 'microsoft':
|
||||
config['impact'] = 3
|
||||
elif main_type == 'scan':
|
||||
config['impact'] = 1
|
||||
elif main_type == 'sinkhole6':
|
||||
config['impact'] = 2
|
||||
elif main_type == 'sinkhole':
|
||||
config['impact'] = 2
|
||||
else:
|
||||
config['impact'] = 1
|
||||
|
||||
if not (self.config_path_modules / f"{config['vendor']}_{config['name']}.json").exists():
|
||||
self.logger.debug(f'Creating config file for {filename}.')
|
||||
with open(self.config_path_modules / f"{config['vendor']}_{config['name']}.json", 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
else:
|
||||
with open(self.config_path_modules / f"{config['vendor']}_{config['name']}.json", 'r') as f:
|
||||
# Validate new config file with old
|
||||
config_current = json.load(f)
|
||||
if config_current != config:
|
||||
self.logger.warning('The config file created by this script is different from the one on disk: \n{}\n{}'.format(json.dumps(config), json.dumps(config_current)))
|
||||
# Init list directory
|
||||
directory = self.storage_directory / config['vendor'] / config['name']
|
||||
safe_create_dir(directory)
|
||||
meta = directory / 'meta'
|
||||
safe_create_dir(meta)
|
||||
archive_dir = directory / 'archive'
|
||||
safe_create_dir(archive_dir)
|
||||
self.logger.debug(f'Done with config for {filename}.')
|
||||
return directory
|
||||
|
||||
async def download_daily_entries(self, day: Optional[Dates]=None):
|
||||
await self.__build_daily_dict()
|
||||
for url, filename in self.available_entries[self.__normalize_day(day)]:
|
||||
storage_dir = self.__check_config(filename)
|
||||
if not storage_dir:
|
||||
continue
|
||||
# Check if the file we're trying to download has already been downloaded. Skip if True.
|
||||
uuid = url.split('/')[-1]
|
||||
if (storage_dir / 'meta' / 'last_download').exists():
|
||||
with open(storage_dir / 'meta' / 'last_download') as _fr:
|
||||
last_download_uuid = _fr.read()
|
||||
if last_download_uuid == uuid:
|
||||
self.logger.debug(f'Already downloaded: {url}.')
|
||||
continue
|
||||
async with aiohttp.ClientSession() as s:
|
||||
async with s.get(url) as r:
|
||||
self.logger.info(f'Downloading {url}.')
|
||||
content = await r.content.read()
|
||||
with (storage_dir / f'{datetime.now().isoformat()}.txt').open('wb') as _fw:
|
||||
_fw.write(content)
|
||||
with (storage_dir / 'meta' / 'last_download').open('w') as _fwt:
|
||||
_fwt.write(uuid)
|
||||
|
||||
|
||||
class ShadowServerManager(AbstractManager):
|
||||
|
||||
def __init__(self, config_dir: Path=None, storage_directory: Path=None, loglevel: int=logging.INFO):
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
super().__init__(loglevel)
|
||||
self.script_name = 'shadowserver_fetcher'
|
||||
shadow_server_config_file = get_homedir() / 'config' / 'shadowserver.json'
|
||||
self.config = True
|
||||
if not config_dir:
|
||||
config_dir = get_config_path()
|
||||
if not (config_dir / 'shadowserver.json').exists():
|
||||
if not shadow_server_config_file.exists():
|
||||
self.config = False
|
||||
self.logger.warning(f'No config file available, the shadow server module will not be launched.')
|
||||
self.logger.warning(f'No config file available {shadow_server_config_file}, the shadow server module will not be launched.')
|
||||
return
|
||||
with open(config_dir / 'shadowserver.json') as f:
|
||||
with shadow_server_config_file.open() as f:
|
||||
ss_config = json.load(f)
|
||||
if not storage_directory:
|
||||
storage_directory = get_homedir() / 'rawdata'
|
||||
modules_config = config_dir / 'modules'
|
||||
self.fetcher = ShadowServerFetcher(ss_config['user'], ss_config['password'], modules_config, storage_directory, loglevel)
|
||||
self.fetcher = ShadowServerFetcher(ss_config['user'], ss_config['password'], self.logger)
|
||||
|
||||
def _to_run_forever(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop.run_until_complete(self.fetcher.download_daily_entries())
|
||||
except aiohttp.client_exceptions.ClientConnectorError as e:
|
||||
self.logger.critical(f'Exception while fetching Shadow Server lists: {e}')
|
||||
async def _to_run_forever_async(self):
|
||||
await self.fetcher.download_daily_entries()
|
||||
|
||||
|
||||
def main():
|
||||
modules_manager = ShadowServerManager()
|
||||
if modules_manager.config:
|
||||
asyncio.run(modules_manager.run_async(sleep_in_sec=3600))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
modules_manager = ShadowServerManager()
|
||||
if modules_manager.config:
|
||||
modules_manager.run(sleep_in_sec=3600)
|
||||
main()
|
||||
|
|
|
|||
38
bin/start.py
38
bin/start.py
|
|
@ -1,25 +1,29 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from subprocess import Popen
|
||||
from bgpranking.libs.helpers import get_homedir
|
||||
from subprocess import Popen, run
|
||||
|
||||
import redis
|
||||
import sys
|
||||
from bgpranking.default import get_homedir
|
||||
|
||||
if redis.VERSION < (3, ):
|
||||
print('redis-py >= 3 is required.')
|
||||
sys.exit()
|
||||
|
||||
if __name__ == '__main__':
|
||||
def main():
|
||||
# Just fail if the env isn't set.
|
||||
get_homedir()
|
||||
p = Popen(['run_backend.py', '--start'])
|
||||
p.wait()
|
||||
Popen(['fetcher.py'])
|
||||
Popen(['ssfetcher.py'])
|
||||
Popen(['parser.py'])
|
||||
Popen(['sanitizer.py'])
|
||||
Popen(['dbinsert.py'])
|
||||
Popen(['ranking.py'])
|
||||
Popen(['asn_descriptions.py'])
|
||||
print('Start backend (redis)...')
|
||||
p = run(['run_backend', '--start'])
|
||||
p.check_returncode()
|
||||
print('done.')
|
||||
Popen(['fetcher'])
|
||||
# Popen(['ssfetcher'])
|
||||
Popen(['parser'])
|
||||
Popen(['sanitizer'])
|
||||
Popen(['dbinsert'])
|
||||
Popen(['ranking'])
|
||||
Popen(['asn_descriptions'])
|
||||
print('Start website...')
|
||||
# Popen(['start_website'])
|
||||
print('done.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,14 +1,40 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
from subprocess import Popen
|
||||
from bgpranking.libs.helpers import get_homedir
|
||||
|
||||
from bgpranking.default import AbstractManager
|
||||
from bgpranking.default import get_config, get_homedir
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO)
|
||||
|
||||
|
||||
class Website(AbstractManager):
|
||||
|
||||
def __init__(self, loglevel: int=logging.INFO):
|
||||
super().__init__(loglevel)
|
||||
self.script_name = 'website'
|
||||
self.process = self._launch_website()
|
||||
self.set_running()
|
||||
|
||||
def _launch_website(self):
|
||||
website_dir = get_homedir() / 'website'
|
||||
ip = get_config('generic', 'website_listen_ip')
|
||||
port = get_config('generic', 'website_listen_port')
|
||||
return Popen(['gunicorn', '-w', '10',
|
||||
'--graceful-timeout', '2', '--timeout', '300',
|
||||
'-b', f'{ip}:{port}',
|
||||
'--log-level', 'info',
|
||||
'web:app'],
|
||||
cwd=website_dir)
|
||||
|
||||
|
||||
def main():
|
||||
w = Website()
|
||||
w.run(sleep_in_sec=10)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
website_dir = get_homedir() / 'website'
|
||||
Popen([f'{website_dir}/3drparty.sh'], cwd=website_dir)
|
||||
try:
|
||||
Popen(['gunicorn', '--worker-class', 'gevent', '-w', '10', '-b', '0.0.0.0:5005', 'web:app'],
|
||||
cwd=website_dir).communicate()
|
||||
except KeyboardInterrupt:
|
||||
print('Stopping gunicorn.')
|
||||
main()
|
||||
|
|
|
|||
30
bin/stop.py
30
bin/stop.py
|
|
@ -1,11 +1,29 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from subprocess import Popen
|
||||
from bgpranking.libs.helpers import get_homedir
|
||||
from subprocess import Popen, run
|
||||
|
||||
from redis import Redis
|
||||
from redis.exceptions import ConnectionError
|
||||
|
||||
from bgpranking.default import get_homedir, get_socket_path
|
||||
|
||||
|
||||
def main():
|
||||
get_homedir()
|
||||
p = Popen(['shutdown'])
|
||||
p.wait()
|
||||
try:
|
||||
r = Redis(unix_socket_path=get_socket_path('cache'), db=1)
|
||||
r.delete('shutdown')
|
||||
print('Shutting down databases...')
|
||||
p_backend = run(['run_backend', '--stop'])
|
||||
p_backend.check_returncode()
|
||||
print('done.')
|
||||
except ConnectionError:
|
||||
# Already down, skip the stacktrace
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
get_homedir()
|
||||
p = Popen(['shutdown.py'])
|
||||
p.wait()
|
||||
Popen(['run_backend.py', '--stop'])
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import logging
|
||||
import platform
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from bgpranking.default import get_homedir, get_config
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
|
||||
level=logging.INFO)
|
||||
|
||||
|
||||
def compute_hash_self():
|
||||
m = hashlib.sha256()
|
||||
with (get_homedir() / 'bin' / 'update.py').open('rb') as f:
|
||||
m.update(f.read())
|
||||
return m.digest()
|
||||
|
||||
|
||||
def keep_going(ignore=False):
|
||||
if ignore:
|
||||
return
|
||||
keep_going = input('Continue? (y/N) ')
|
||||
if keep_going.lower() != 'y':
|
||||
print('Okay, quitting.')
|
||||
sys.exit()
|
||||
|
||||
|
||||
def run_command(command, expect_fail: bool=False, capture_output: bool=True):
|
||||
args = shlex.split(command)
|
||||
homedir = get_homedir()
|
||||
process = subprocess.run(args, cwd=homedir, capture_output=capture_output)
|
||||
if capture_output:
|
||||
print(process.stdout.decode())
|
||||
if process.returncode and not expect_fail:
|
||||
print(process.stderr.decode())
|
||||
sys.exit()
|
||||
|
||||
|
||||
def check_poetry_version():
|
||||
args = shlex.split("poetry self -V")
|
||||
homedir = get_homedir()
|
||||
process = subprocess.run(args, cwd=homedir, capture_output=True)
|
||||
poetry_version_str = process.stdout.decode()
|
||||
version = poetry_version_str.split()[2]
|
||||
version_details = tuple(int(i) for i in version.split('.'))
|
||||
if version_details < (1, 1, 0):
|
||||
print('The project requires poetry >= 1.1.0, please update.')
|
||||
print('If you installed with "pip install --user poetry", run "pip install --user -U poetry"')
|
||||
print('If you installed via the recommended method, use "poetry self update"')
|
||||
print('More details: https://github.com/python-poetry/poetry#updating-poetry')
|
||||
sys.exit()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Pull latest release, update dependencies, update and validate the config files, update 3rd deps for the website.')
|
||||
parser.add_argument('--yes', default=False, action='store_true', help='Run all commands without asking.')
|
||||
args = parser.parse_args()
|
||||
|
||||
old_hash = compute_hash_self()
|
||||
|
||||
print('* Update repository.')
|
||||
keep_going(args.yes)
|
||||
run_command('git pull')
|
||||
new_hash = compute_hash_self()
|
||||
if old_hash != new_hash:
|
||||
print('Update script changed, please do "poetry run update"')
|
||||
sys.exit()
|
||||
|
||||
check_poetry_version()
|
||||
|
||||
print('* Install/update dependencies.')
|
||||
keep_going(args.yes)
|
||||
run_command('poetry install')
|
||||
|
||||
print('* Validate configuration files.')
|
||||
keep_going(args.yes)
|
||||
run_command(f'poetry run {(Path("tools") / "validate_config_files.py").as_posix()} --check')
|
||||
|
||||
print('* Update configuration files.')
|
||||
keep_going(args.yes)
|
||||
run_command(f'poetry run {(Path("tools") / "validate_config_files.py").as_posix()} --update')
|
||||
|
||||
print('* Restarting')
|
||||
keep_going(args.yes)
|
||||
if platform.system() == 'Windows':
|
||||
print('Restarting with poetry...')
|
||||
run_command('poetry run stop', expect_fail=True)
|
||||
run_command('poetry run start', capture_output=False)
|
||||
print('Started.')
|
||||
else:
|
||||
service = get_config('generic', 'systemd_service_name')
|
||||
p = subprocess.run(["systemctl", "is-active", "--quiet", service])
|
||||
try:
|
||||
p.check_returncode()
|
||||
print('Restarting with systemd...')
|
||||
run_command(f'sudo service {service} restart')
|
||||
print('done.')
|
||||
except subprocess.CalledProcessError:
|
||||
print('Restarting with poetry...')
|
||||
run_command('poetry run stop', expect_fail=True)
|
||||
run_command('poetry run start', capture_output=False)
|
||||
print('Started.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
from urllib.parse import urljoin
|
||||
from pybgpranking import BGPRanking
|
||||
from pyipasnhistory import IPASNHistory
|
||||
from datetime import date, timedelta
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Run a query against BGP Ranking')
|
||||
parser.add_argument('--url', type=str, help='URL of the instance.')
|
||||
parser.add_argument('--date', default=date.today().isoformat(), help='Date of the dataset required')
|
||||
|
||||
sub_parsers = parser.add_subparsers(title='Available commands')
|
||||
|
||||
index_query = sub_parsers.add_parser('index')
|
||||
index_query.add_argument('--limit', default=100, help='Max number of ASN to get')
|
||||
index_query.add_argument('--family', default='v4', help='v4 or v6')
|
||||
index_query.set_defaults(which='index')
|
||||
|
||||
simple_query = sub_parsers.add_parser('simple')
|
||||
group = simple_query.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--asn', help='ASN to lookup')
|
||||
group.add_argument('--ip', help='IP to lookup')
|
||||
simple_query.set_defaults(which='simple')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.url:
|
||||
bgpranking = BGPRanking(args.url)
|
||||
ipasn = IPASNHistory(urljoin(args.url, 'ipasn_history'))
|
||||
else:
|
||||
bgpranking = BGPRanking()
|
||||
ipasn = IPASNHistory()
|
||||
|
||||
if args.which == 'simple':
|
||||
if args.ip:
|
||||
response = ipasn.query(args.ip)
|
||||
print(json.dumps(response, indent=2))
|
||||
if 'response' in response and response['response']:
|
||||
asn = response['response'][list(response['response'].keys())[0]]['asn']
|
||||
else:
|
||||
asn = args.asn
|
||||
|
||||
response = bgpranking.query(asn, date=(date.today() - timedelta(1)).isoformat())
|
||||
elif args.which == 'index':
|
||||
response = bgpranking.asns_global_ranking(address_family=args.family, limit=args.limit, date=args.date)
|
||||
print(json.dumps(response, indent=2))
|
||||
|
|
@ -1 +0,0 @@
|
|||
from .api import BGPRanking # noqa
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
from typing import Union
|
||||
|
||||
import requests
|
||||
from urllib.parse import urljoin
|
||||
from datetime import date
|
||||
|
||||
|
||||
class BGPRanking():
|
||||
|
||||
def __init__(self, root_url: str='https://bgpranking-ng.circl.lu/'):
|
||||
self.root_url = root_url
|
||||
if not self.root_url.endswith('/'):
|
||||
self.root_url += '/'
|
||||
self.session = requests.session()
|
||||
|
||||
@property
|
||||
def is_up(self):
|
||||
r = self.session.head(self.root_url)
|
||||
return r.status_code == 200
|
||||
|
||||
def query(self, asn: str, address_family: str='v4', date: str=None,
|
||||
source: Union[list, str]=''):
|
||||
'''Launch a query.
|
||||
:param asn: ASN to lookup
|
||||
:param address_family: v4 or v6
|
||||
:param date: Exact date to lookup. Fallback to most recent available.
|
||||
:param source: Source to query. Can be a list of sources.
|
||||
'''
|
||||
to_query = {'asn': asn, 'address_family': address_family}
|
||||
if date:
|
||||
to_query['date'] = date
|
||||
if source:
|
||||
to_query['source'] = source
|
||||
r = self.session.post(urljoin(self.root_url, '/json/asn'), data=json.dumps(to_query))
|
||||
return r.json()
|
||||
|
||||
def asns_global_ranking(self, date: str=date.today().isoformat(), address_family: str='v4', limit: int=100):
|
||||
'''Get the top `limit` ASNs, from worse to best'''
|
||||
to_query = {'date': date, 'ipversion': address_family, 'limit': limit}
|
||||
r = self.session.post(urljoin(self.root_url, '/json/asns_global_ranking'), data=json.dumps(to_query))
|
||||
return r.json()
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from setuptools import setup
|
||||
|
||||
|
||||
setup(
|
||||
name='pybgpranking',
|
||||
version='0.1',
|
||||
author='Raphaël Vinot',
|
||||
author_email='raphael.vinot@circl.lu',
|
||||
maintainer='Raphaël Vinot',
|
||||
url='https://github.com/D4-project/BGP-Ranking/client',
|
||||
description='Python client for BGP Ranking',
|
||||
packages=['pybgpranking'],
|
||||
scripts=['bin/bgpranking'],
|
||||
install_requires=['requests'],
|
||||
classifiers=[
|
||||
'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
|
||||
'Development Status :: 3 - Alpha',
|
||||
'Environment :: Console',
|
||||
'Operating System :: POSIX :: Linux',
|
||||
'Intended Audience :: Science/Research',
|
||||
'Intended Audience :: Telecommunications Industry',
|
||||
'Intended Audience :: Information Technology',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Topic :: Security',
|
||||
'Topic :: Internet',
|
||||
]
|
||||
)
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"loglevel": "INFO",
|
||||
"website_listen_ip": "0.0.0.0",
|
||||
"website_listen_port": 5005,
|
||||
"systemd_service_name": "bgpranking",
|
||||
"storage_db_hostname": "127.0.0.1",
|
||||
"storage_db_port": 5188,
|
||||
"ranking_db_hostname": "127.0.0.1",
|
||||
"ranking_db_port": 5189,
|
||||
"ipasnhistory_url": "https://ipasnhistory.circl.lu/",
|
||||
"_notes": {
|
||||
"loglevel": "(lookyloo) Can be one of the value listed here: https://docs.python.org/3/library/logging.html#levels",
|
||||
"website_listen_ip": "IP Flask will listen on. Defaults to 0.0.0.0, meaning all interfaces.",
|
||||
"website_listen_port": "Port Flask will listen on.",
|
||||
"systemd_service_name": "(Optional) Name of the systemd service if your project has one.",
|
||||
"storage_db_hostname": "Hostname of the storage database (kvrocks)",
|
||||
"storage_db_port": "Port of the storage database (kvrocks)",
|
||||
"ranking_db_hostname": "Hostname of the ranking database (kvrocks)",
|
||||
"ranking_db_port": "Port of the ranking database (kvrocks)",
|
||||
"ipasnhistory_url": "URL of the IP ASN History service, defaults to the public one."
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,64 @@
|
|||
[tool.poetry]
|
||||
name = "bgpranking"
|
||||
version = "2.0"
|
||||
description = "BGP Ranking is a software to rank AS numbers based on their malicious activities."
|
||||
authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
|
||||
license = "AGPLv3"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
start = "bin.start:main"
|
||||
stop = "bin.stop:main"
|
||||
update = "bin.update:main"
|
||||
shutdown = "bin.shutdown:main"
|
||||
run_backend = "bin.run_backend:main"
|
||||
start_website = "bin.start_website:main"
|
||||
|
||||
archiver = "bin.archiver:main"
|
||||
asn_descriptions = "bin.asn_descriptions:main"
|
||||
dbinsert = "bin.dbinsert:main"
|
||||
fetcher = "bin.fetcher:main"
|
||||
parser = "bin.parser:main"
|
||||
ranking = "bin.ranking:main"
|
||||
sanitizer = "bin.sanitizer:main"
|
||||
ssfetcher = "bin.ssfetcher:main"
|
||||
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.8"
|
||||
redis = {version = "^4.0.2", extras = ["hiredis"]}
|
||||
flask-restx = "^0.5.1"
|
||||
gunicorn = "^20.1.0"
|
||||
python-dateutil = "^2.8.2"
|
||||
pyipasnhistory = "^2.0"
|
||||
pycountry = "^20.7.3"
|
||||
beautifulsoup4 = "^4.10.0"
|
||||
aiohttp = "^3.8.1"
|
||||
Bootstrap-Flask = "^1.8.0"
|
||||
pid = "^3.0.4"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
ipython = "^7.23.0"
|
||||
mypy = "^0.920"
|
||||
types-setuptools = "^57.4.4"
|
||||
types-redis = "^4.0.3"
|
||||
types-requests = "^2.26.1"
|
||||
types-python-dateutil = "^2.8.3"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.mypy]
|
||||
python_version = 3.8
|
||||
check_untyped_defs = true
|
||||
ignore_errors = false
|
||||
ignore_missing_imports = false
|
||||
strict_optional = true
|
||||
no_implicit_optional = true
|
||||
warn_unused_ignores = true
|
||||
warn_redundant_casts = true
|
||||
warn_unused_configs = true
|
||||
warn_unreachable = true
|
||||
|
||||
show_error_context = true
|
||||
pretty = true
|
||||
|
|
@ -0,0 +1,497 @@
|
|||
################################ GENERAL #####################################
|
||||
|
||||
# By default kvrocks listens for connections from all the network interfaces
|
||||
# available on the server. It is possible to listen to just one or multiple
|
||||
# interfaces using the "bind" configuration directive, followed by one or
|
||||
# more IP addresses.
|
||||
#
|
||||
# Examples:
|
||||
#
|
||||
# bind 192.168.1.100 10.0.0.1
|
||||
# bind 127.0.0.1
|
||||
bind 0.0.0.0
|
||||
|
||||
# Accept connections on the specified port, default is 6666.
|
||||
port 5189
|
||||
|
||||
# Close the connection after a client is idle for N seconds (0 to disable)
|
||||
timeout 0
|
||||
|
||||
# The number of worker's threads, increase or decrease it would effect the performance.
|
||||
workers 8
|
||||
|
||||
# The number of replication worker's threads, increase or decrease it would effect the replication performance.
|
||||
# Default: 1
|
||||
repl-workers 1
|
||||
|
||||
# By default kvrocks does not run as a daemon. Use 'yes' if you need it.
|
||||
# Note that kvrocks will write a pid file in /var/run/kvrocks.pid when daemonized.
|
||||
daemonize no
|
||||
|
||||
# Kvrocks implements cluster solution that is similar with redis cluster sulution.
|
||||
# You can get cluster information by CLUSTER NODES|SLOTS|INFO command, it also is
|
||||
# adapted to redis-cli, redis-benchmark, redis cluster SDK and redis cluster proxy.
|
||||
# But kvrocks doesn't support to communicate with each others, so you must set
|
||||
# cluster topology by CLUSTER SETNODES|SETNODEID commands, more details: #219.
|
||||
#
|
||||
# PLEASE NOTE:
|
||||
# If you enable cluster, kvrocks will encode key with its slot id calculated by
|
||||
# CRC16 and modulo 16384, endoding key with its slot id makes it efficient to
|
||||
# migrate keys based on slot. So if you enabled at first time, cluster mode must
|
||||
# not be disabled after restarting, and vice versa. That is to say, data is not
|
||||
# compatible between standalone mode with cluster mode, you must migrate data
|
||||
# if you want to change mode, otherwise, kvrocks will make data corrupt.
|
||||
#
|
||||
# Default: no
|
||||
cluster-enabled no
|
||||
|
||||
# Set the max number of connected clients at the same time. By default
|
||||
# this limit is set to 10000 clients, however if the server is not
|
||||
# able to configure the process file limit to allow for the specified limit
|
||||
# the max number of allowed clients is set to the current file limit
|
||||
#
|
||||
# Once the limit is reached the server will close all the new connections sending
|
||||
# an error 'max number of clients reached'.
|
||||
#
|
||||
maxclients 10000
|
||||
|
||||
# Require clients to issue AUTH <PASSWORD> before processing any other
|
||||
# commands. This might be useful in environments in which you do not trust
|
||||
# others with access to the host running kvrocks.
|
||||
#
|
||||
# This should stay commented out for backward compatibility and because most
|
||||
# people do not need auth (e.g. they run their own servers).
|
||||
#
|
||||
# Warning: since kvrocks is pretty fast an outside user can try up to
|
||||
# 150k passwords per second against a good box. This means that you should
|
||||
# use a very strong password otherwise it will be very easy to break.
|
||||
#
|
||||
# requirepass foobared
|
||||
|
||||
# If the master is password protected (using the "masterauth" configuration
|
||||
# directive below) it is possible to tell the slave to authenticate before
|
||||
# starting the replication synchronization process, otherwise the master will
|
||||
# refuse the slave request.
|
||||
#
|
||||
# masterauth foobared
|
||||
|
||||
# Master-Salve replication would check db name is matched. if not, the slave should
|
||||
# refuse to sync the db from master. Don't use default value, set the db-name to identify
|
||||
# the cluster.
|
||||
db-name storage.db
|
||||
|
||||
# The working directory
|
||||
#
|
||||
# The DB will be written inside this directory
|
||||
# Note that you must specify a directory here, not a file name.
|
||||
dir ./
|
||||
|
||||
# The logs of server will be stored in this directory. If you don't specify
|
||||
# one directory, by default, we store logs in the working directory that set
|
||||
# by 'dir' above.
|
||||
# log-dir /tmp/kvrocks
|
||||
|
||||
# When running daemonized, kvrocks writes a pid file in ${CONFIG_DIR}/kvrocks.pid by
|
||||
# default. You can specify a custom pid file location here.
|
||||
# pidfile /var/run/kvrocks.pid
|
||||
pidfile storage.pid
|
||||
|
||||
# You can configure a slave instance to accept writes or not. Writing against
|
||||
# a slave instance may be useful to store some ephemeral data (because data
|
||||
# written on a slave will be easily deleted after resync with the master) but
|
||||
# may also cause problems if clients are writing to it because of a
|
||||
# misconfiguration.
|
||||
slave-read-only yes
|
||||
|
||||
# The slave priority is an integer number published by Kvrocks in the INFO output.
|
||||
# It is used by Redis Sentinel in order to select a slave to promote into a
|
||||
# master if the master is no longer working correctly.
|
||||
#
|
||||
# A slave with a low priority number is considered better for promotion, so
|
||||
# for instance if there are three slave with priority 10, 100, 25 Sentinel will
|
||||
# pick the one with priority 10, that is the lowest.
|
||||
#
|
||||
# However a special priority of 0 marks the replica as not able to perform the
|
||||
# role of master, so a slave with priority of 0 will never be selected by
|
||||
# Redis Sentinel for promotion.
|
||||
#
|
||||
# By default the priority is 100.
|
||||
slave-priority 100
|
||||
|
||||
# TCP listen() backlog.
|
||||
#
|
||||
# In high requests-per-second environments you need an high backlog in order
|
||||
# to avoid slow clients connections issues. Note that the Linux kernel
|
||||
# will silently truncate it to the value of /proc/sys/net/core/somaxconn so
|
||||
# make sure to raise both the value of somaxconn and tcp_max_syn_backlog
|
||||
# in order to Get the desired effect.
|
||||
tcp-backlog 511
|
||||
|
||||
# If the master is an old version, it may have specified replication threads
|
||||
# that use 'port + 1' as listening port, but in new versions, we don't use
|
||||
# extra port to implement replication. In order to allow the new replicas to
|
||||
# copy old masters, you should indicate that the master uses replication port
|
||||
# or not.
|
||||
# If yes, that indicates master uses replication port and replicas will connect
|
||||
# to 'master's listening port + 1' when synchronization.
|
||||
# If no, that indicates master doesn't use replication port and replicas will
|
||||
# connect 'master's listening port' when synchronization.
|
||||
master-use-repl-port no
|
||||
|
||||
# Master-Slave replication. Use slaveof to make a kvrocks instance a copy of
|
||||
# another kvrocks server. A few things to understand ASAP about kvrocks replication.
|
||||
#
|
||||
# 1) Kvrocks replication is asynchronous, but you can configure a master to
|
||||
# stop accepting writes if it appears to be not connected with at least
|
||||
# a given number of slaves.
|
||||
# 2) Kvrocks slaves are able to perform a partial resynchronization with the
|
||||
# master if the replication link is lost for a relatively small amount of
|
||||
# time. You may want to configure the replication backlog size (see the next
|
||||
# sections of this file) with a sensible value depending on your needs.
|
||||
# 3) Replication is automatic and does not need user intervention. After a
|
||||
# network partition slaves automatically try to reconnect to masters
|
||||
# and resynchronize with them.
|
||||
#
|
||||
# slaveof <masterip> <masterport>
|
||||
# slaveof 127.0.0.1 6379
|
||||
|
||||
# When a slave loses its connection with the master, or when the replication
|
||||
# is still in progress, the slave can act in two different ways:
|
||||
#
|
||||
# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
|
||||
# still reply to client requests, possibly with out of date data, or the
|
||||
# data set may just be empty if this is the first synchronization.
|
||||
#
|
||||
# 2) if slave-serve-stale-data is set to 'no' the slave will reply with
|
||||
# an error "SYNC with master in progress" to all the kind of commands
|
||||
# but to INFO and SLAVEOF.
|
||||
#
|
||||
slave-serve-stale-data yes
|
||||
|
||||
# To guarantee slave's data safe and serve when it is in full synchronization
|
||||
# state, slave still keep itself data. But this way needs to occupy much disk
|
||||
# space, so we provide a way to reduce disk occupation, slave will delete itself
|
||||
# entire database before fetching files from master during full synchronization.
|
||||
# If you want to enable this way, you can set 'slave-delete-db-before-fullsync'
|
||||
# to yes, but you must know that database will be lost if master is down during
|
||||
# full synchronization, unless you have a backup of database.
|
||||
#
|
||||
# This option is similar redis replicas RDB diskless load option:
|
||||
# repl-diskless-load on-empty-db
|
||||
#
|
||||
# Default: no
|
||||
slave-empty-db-before-fullsync no
|
||||
|
||||
# If replicas need full synchronization with master, master need to create
|
||||
# checkpoint for feeding replicas, and replicas also stage a checkpoint of
|
||||
# the master. If we also keep the backup, it maybe occupy extra disk space.
|
||||
# You can enable 'purge-backup-on-fullsync' if disk is not sufficient, but
|
||||
# that may cause remote backup copy failing.
|
||||
#
|
||||
# Default: no
|
||||
purge-backup-on-fullsync no
|
||||
|
||||
# The maximum allowed rate (in MB/s) that should be used by Replication.
|
||||
# If the rate exceeds max-replication-mb, replication will slow down.
|
||||
# Default: 0 (i.e. no limit)
|
||||
max-replication-mb 0
|
||||
|
||||
# The maximum allowed aggregated write rate of flush and compaction (in MB/s).
|
||||
# If the rate exceeds max-io-mb, io will slow down.
|
||||
# 0 is no limit
|
||||
# Default: 500
|
||||
max-io-mb 500
|
||||
|
||||
# The maximum allowed space (in GB) that should be used by RocksDB.
|
||||
# If the total size of the SST files exceeds max_allowed_space, writes to RocksDB will fail.
|
||||
# Please see: https://github.com/facebook/rocksdb/wiki/Managing-Disk-Space-Utilization
|
||||
# Default: 0 (i.e. no limit)
|
||||
max-db-size 0
|
||||
|
||||
# The maximum backup to keep, server cron would run every minutes to check the num of current
|
||||
# backup, and purge the old backup if exceed the max backup num to keep. If max-backup-to-keep
|
||||
# is 0, no backup would be keep. But now, we only support 0 or 1.
|
||||
max-backup-to-keep 1
|
||||
|
||||
# The maximum hours to keep the backup. If max-backup-keep-hours is 0, wouldn't purge any backup.
|
||||
# default: 1 day
|
||||
max-backup-keep-hours 24
|
||||
|
||||
# Ratio of the samples would be recorded when the profiling was enabled.
|
||||
# we simply use the rand to determine whether to record the sample or not.
|
||||
#
|
||||
# Default: 0
|
||||
profiling-sample-ratio 0
|
||||
|
||||
# There is no limit to this length. Just be aware that it will consume memory.
|
||||
# You can reclaim memory used by the perf log with PERFLOG RESET.
|
||||
#
|
||||
# Default: 256
|
||||
profiling-sample-record-max-len 256
|
||||
|
||||
# profiling-sample-record-threshold-ms use to tell the kvrocks when to record.
|
||||
#
|
||||
# Default: 100 millisecond
|
||||
profiling-sample-record-threshold-ms 100
|
||||
|
||||
################################## SLOW LOG ###################################
|
||||
|
||||
# The Kvrocks Slow Log is a system to log queries that exceeded a specified
|
||||
# execution time. The execution time does not include the I/O operations
|
||||
# like talking with the client, sending the reply and so forth,
|
||||
# but just the time needed to actually execute the command (this is the only
|
||||
# stage of command execution where the thread is blocked and can not serve
|
||||
# other requests in the meantime).
|
||||
#
|
||||
# You can configure the slow log with two parameters: one tells Kvrocks
|
||||
# what is the execution time, in microseconds, to exceed in order for the
|
||||
# command to get logged, and the other parameter is the length of the
|
||||
# slow log. When a new command is logged the oldest one is removed from the
|
||||
# queue of logged commands.
|
||||
|
||||
# The following time is expressed in microseconds, so 1000000 is equivalent
|
||||
# to one second. Note that -1 value disables the slow log, while
|
||||
# a value of zero forces the logging of every command.
|
||||
slowlog-log-slower-than 100000
|
||||
|
||||
# There is no limit to this length. Just be aware that it will consume memory.
|
||||
# You can reclaim memory used by the slow log with SLOWLOG RESET.
|
||||
slowlog-max-len 128
|
||||
|
||||
# If you run kvrocks from upstart or systemd, kvrocks can interact with your
|
||||
# supervision tree. Options:
|
||||
# supervised no - no supervision interaction
|
||||
# supervised upstart - signal upstart by putting kvrocks into SIGSTOP mode
|
||||
# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
|
||||
# supervised auto - detect upstart or systemd method based on
|
||||
# UPSTART_JOB or NOTIFY_SOCKET environment variables
|
||||
# Note: these supervision methods only signal "process is ready."
|
||||
# They do not enable continuous liveness pings back to your supervisor.
|
||||
supervised no
|
||||
|
||||
################################## CRON ###################################
|
||||
|
||||
# Compact Scheduler, auto compact at schedule time
|
||||
# time expression format is the same as crontab(currently only support * and int)
|
||||
# e.g. compact-cron 0 3 * * * 0 4 * * *
|
||||
# would compact the db at 3am and 4am everyday
|
||||
# compact-cron 0 3 * * *
|
||||
|
||||
# The hour range that compaction checker would be active
|
||||
# e.g. compaction-checker-range 0-7 means compaction checker would be worker between
|
||||
# 0-7am every day.
|
||||
compaction-checker-range 0-7
|
||||
|
||||
# Bgsave scheduler, auto bgsave at schedule time
|
||||
# time expression format is the same as crontab(currently only support * and int)
|
||||
# e.g. bgsave-cron 0 3 * * * 0 4 * * *
|
||||
# would bgsave the db at 3am and 4am everyday
|
||||
|
||||
# Command renaming.
|
||||
#
|
||||
# It is possible to change the name of dangerous commands in a shared
|
||||
# environment. For instance the KEYS command may be renamed into something
|
||||
# hard to guess so that it will still be available for internal-use tools
|
||||
# but not available for general clients.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# rename-command KEYS b840fc02d524045429941cc15f59e41cb7be6c52
|
||||
#
|
||||
# It is also possible to completely kill a command by renaming it into
|
||||
# an empty string:
|
||||
#
|
||||
# rename-command KEYS ""
|
||||
|
||||
# The key-value size may so be quite different in many scenes, and use 256MiB as SST file size
|
||||
# may cause data loading(large index/filter block) ineffective when the key-value was too small.
|
||||
# kvrocks supports user-defined SST file in config(rocksdb.target_file_size_base),
|
||||
# but it still too trivial and inconvenient to adjust the different sizes for different instances.
|
||||
# so we want to periodic auto-adjust the SST size in-flight with user avg key-value size.
|
||||
#
|
||||
# If enabled, kvrocks will auto resize rocksdb.target_file_size_base
|
||||
# and rocksdb.write_buffer_size in-flight with user avg key-value size.
|
||||
# Please see #118.
|
||||
#
|
||||
# Default: yes
|
||||
auto-resize-block-and-sst yes
|
||||
|
||||
################################ ROCKSDB #####################################
|
||||
|
||||
# Specify the capacity of metadata column family block cache. Larger block cache
|
||||
# may make request faster while more keys would be cached. Max Size is 200*1024.
|
||||
# Default: 2048MB
|
||||
rocksdb.metadata_block_cache_size 2048
|
||||
|
||||
# Specify the capacity of subkey column family block cache. Larger block cache
|
||||
# may make request faster while more keys would be cached. Max Size is 200*1024.
|
||||
# Default: 2048MB
|
||||
rocksdb.subkey_block_cache_size 2048
|
||||
|
||||
# Metadata column family and subkey column family will share a single block cache
|
||||
# if set 'yes'. The capacity of shared block cache is
|
||||
# metadata_block_cache_size + subkey_block_cache_size
|
||||
#
|
||||
# Default: yes
|
||||
rocksdb.share_metadata_and_subkey_block_cache yes
|
||||
|
||||
# Number of open files that can be used by the DB. You may need to
|
||||
# increase this if your database has a large working set. Value -1 means
|
||||
# files opened are always kept open. You can estimate number of files based
|
||||
# on target_file_size_base and target_file_size_multiplier for level-based
|
||||
# compaction. For universal-style compaction, you can usually set it to -1.
|
||||
# Default: 4096
|
||||
rocksdb.max_open_files 8096
|
||||
|
||||
# Amount of data to build up in memory (backed by an unsorted log
|
||||
# on disk) before converting to a sorted on-disk file.
|
||||
#
|
||||
# Larger values increase performance, especially during bulk loads.
|
||||
# Up to max_write_buffer_number write buffers may be held in memory
|
||||
# at the same time,
|
||||
# so you may wish to adjust this parameter to control memory usage.
|
||||
# Also, a larger write buffer will result in a longer recovery time
|
||||
# the next time the database is opened.
|
||||
#
|
||||
# Note that write_buffer_size is enforced per column family.
|
||||
# See db_write_buffer_size for sharing memory across column families.
|
||||
|
||||
# default is 64MB
|
||||
rocksdb.write_buffer_size 16
|
||||
|
||||
# Target file size for compaction, target file size for Leve N can be caculated
|
||||
# by target_file_size_base * (target_file_size_multiplier ^ (L-1))
|
||||
#
|
||||
# Default: 128MB
|
||||
rocksdb.target_file_size_base 16
|
||||
|
||||
# The maximum number of write buffers that are built up in memory.
|
||||
# The default and the minimum number is 2, so that when 1 write buffer
|
||||
# is being flushed to storage, new writes can continue to the other
|
||||
# write buffer.
|
||||
# If max_write_buffer_number > 3, writing will be slowed down to
|
||||
# options.delayed_write_rate if we are writing to the last write buffer
|
||||
# allowed.
|
||||
rocksdb.max_write_buffer_number 4
|
||||
|
||||
# Maximum number of concurrent background compaction jobs, submitted to
|
||||
# the default LOW priority thread pool.
|
||||
rocksdb.max_background_compactions 4
|
||||
|
||||
# Maximum number of concurrent background memtable flush jobs, submitted by
|
||||
# default to the HIGH priority thread pool. If the HIGH priority thread pool
|
||||
# is configured to have zero threads, flush jobs will share the LOW priority
|
||||
# thread pool with compaction jobs.
|
||||
rocksdb.max_background_flushes 4
|
||||
|
||||
# This value represents the maximum number of threads that will
|
||||
# concurrently perform a compaction job by breaking it into multiple,
|
||||
# smaller ones that are run simultaneously.
|
||||
# Default: 2 (i.e. no subcompactions)
|
||||
rocksdb.max_sub_compactions 2
|
||||
|
||||
# In order to limit the size of WALs, RocksDB uses DBOptions::max_total_wal_size
|
||||
# as the trigger of column family flush. Once WALs exceed this size, RocksDB
|
||||
# will start forcing the flush of column families to allow deletion of some
|
||||
# oldest WALs. This config can be useful when column families are updated at
|
||||
# non-uniform frequencies. If there's no size limit, users may need to keep
|
||||
# really old WALs when the infrequently-updated column families hasn't flushed
|
||||
# for a while.
|
||||
#
|
||||
# In kvrocks, we use multiple column families to store metadata, subkeys, etc.
|
||||
# If users always use string type, but use list, hash and other complex data types
|
||||
# infrequently, there will be a lot of old WALs if we don't set size limit
|
||||
# (0 by default in rocksdb), because rocksdb will dynamically choose the WAL size
|
||||
# limit to be [sum of all write_buffer_size * max_write_buffer_number] * 4 if set to 0.
|
||||
#
|
||||
# Moreover, you should increase this value if you already set rocksdb.write_buffer_size
|
||||
# to a big value, to avoid influencing the effect of rocksdb.write_buffer_size and
|
||||
# rocksdb.max_write_buffer_number.
|
||||
#
|
||||
# default is 512MB
|
||||
rocksdb.max_total_wal_size 512
|
||||
|
||||
# We impl the repliction with rocksdb WAL, it would trigger full sync when the seq was out of range.
|
||||
# wal_ttl_seconds and wal_size_limit_mb would affect how archived logswill be deleted.
|
||||
# If WAL_ttl_seconds is not 0, then WAL files will be checked every WAL_ttl_seconds / 2 and those that
|
||||
# are older than WAL_ttl_seconds will be deleted#
|
||||
#
|
||||
# Default: 3 Hours
|
||||
rocksdb.wal_ttl_seconds 10800
|
||||
|
||||
# If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0,
|
||||
# WAL files will be checked every 10 min and if total size is greater
|
||||
# then WAL_size_limit_MB, they will be deleted starting with the
|
||||
# earliest until size_limit is met. All empty files will be deleted
|
||||
# Default: 16GB
|
||||
rocksdb.wal_size_limit_mb 16384
|
||||
|
||||
# Approximate size of user data packed per block. Note that the
|
||||
# block size specified here corresponds to uncompressed data. The
|
||||
# actual size of the unit read from disk may be smaller if
|
||||
# compression is enabled.
|
||||
#
|
||||
# Default: 4KB
|
||||
rocksdb.block_size 2048
|
||||
|
||||
# Indicating if we'd put index/filter blocks to the block cache
|
||||
#
|
||||
# Default: no
|
||||
rocksdb.cache_index_and_filter_blocks yes
|
||||
|
||||
# Specify the compression to use.
|
||||
# Accept value: "no", "snappy"
|
||||
# default snappy
|
||||
rocksdb.compression snappy
|
||||
|
||||
# If non-zero, we perform bigger reads when doing compaction. If you're
|
||||
# running RocksDB on spinning disks, you should set this to at least 2MB.
|
||||
# That way RocksDB's compaction is doing sequential instead of random reads.
|
||||
# When non-zero, we also force new_table_reader_for_compaction_inputs to
|
||||
# true.
|
||||
#
|
||||
# Default: 2 MB
|
||||
rocksdb.compaction_readahead_size 2097152
|
||||
|
||||
# he limited write rate to DB if soft_pending_compaction_bytes_limit or
|
||||
# level0_slowdown_writes_trigger is triggered.
|
||||
|
||||
# If the value is 0, we will infer a value from `rater_limiter` value
|
||||
# if it is not empty, or 16MB if `rater_limiter` is empty. Note that
|
||||
# if users change the rate in `rate_limiter` after DB is opened,
|
||||
# `delayed_write_rate` won't be adjusted.
|
||||
#
|
||||
rocksdb.delayed_write_rate 0
|
||||
# If enable_pipelined_write is true, separate write thread queue is
|
||||
# maintained for WAL write and memtable write.
|
||||
#
|
||||
# Default: no
|
||||
rocksdb.enable_pipelined_write no
|
||||
|
||||
# Soft limit on number of level-0 files. We start slowing down writes at this
|
||||
# point. A value <0 means that no writing slow down will be triggered by
|
||||
# number of files in level-0.
|
||||
#
|
||||
# Default: 20
|
||||
rocksdb.level0_slowdown_writes_trigger 20
|
||||
|
||||
# Maximum number of level-0 files. We stop writes at this point.
|
||||
#
|
||||
# Default: 40
|
||||
rocksdb.level0_stop_writes_trigger 40
|
||||
|
||||
# if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec
|
||||
#
|
||||
# Default: 0
|
||||
rocksdb.stats_dump_period_sec 0
|
||||
|
||||
# if yes, the auto compaction would be disabled, but the manual compaction remain works
|
||||
#
|
||||
# Default: no
|
||||
rocksdb.disable_auto_compactions no
|
||||
################################ NAMESPACE #####################################
|
||||
# namespace.test change.me
|
||||
|
||||
|
||||
backup-dir .//backup
|
||||
log-dir ./
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
../../kvrocks/src/kvrocks -c kvrocks.conf
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
-i https://pypi.org/simple
|
||||
-e .
|
||||
-e ./client
|
||||
-e git+https://github.com/D4-project/IPASN-History.git/@283539cfbbde4bb54497726634407025f7d685c2#egg=pyipasnhistory&subdirectory=client
|
||||
-e git+https://github.com/MISP/PyMISPGalaxies.git@a59a5c18024aedda0c1306c917e09bdb8596ef48#egg=pymispgalaxies
|
||||
-e git+https://github.com/MISP/PyTaxonomies.git@f28fc11bd682aba35778efbcd8a65e68d1225a3f#egg=pytaxonomies
|
||||
-e git+https://github.com/trbs/pid.git/@240d6e848fcb0ebbf80c88b9d724dcb85978a019#egg=pid
|
||||
aiohttp==3.5.4
|
||||
async-timeout==3.0.1
|
||||
attrs==19.1.0
|
||||
beautifulsoup4==4.8.0
|
||||
certifi==2019.6.16
|
||||
chardet==3.0.4
|
||||
click==7.0
|
||||
dominate==2.4.0
|
||||
flask-bootstrap==3.3.7.1
|
||||
flask==1.1.1
|
||||
gevent==1.4.0
|
||||
greenlet==0.4.15 ; platform_python_implementation == 'CPython'
|
||||
gunicorn[gevent]==19.9.0
|
||||
hiredis==1.0.0
|
||||
idna-ssl==1.1.0 ; python_version < '3.7'
|
||||
idna==2.8
|
||||
itsdangerous==1.1.0
|
||||
jinja2==2.10.1
|
||||
markupsafe==1.1.1
|
||||
multidict==4.5.2
|
||||
pycountry==19.8.18
|
||||
python-dateutil==2.8.0
|
||||
redis==3.3.8
|
||||
requests==2.22.0
|
||||
simplejson==3.16.0
|
||||
six==1.12.0
|
||||
soupsieve==1.9.3
|
||||
typing-extensions==3.7.4 ; python_version < '3.7'
|
||||
urllib3==1.25.3
|
||||
visitor==0.1.3
|
||||
werkzeug==0.15.5
|
||||
yarl==1.3.0
|
||||
|
|
@ -1,468 +0,0 @@
|
|||
# Ardb configuration file example, modified from redis's conf file.
|
||||
|
||||
# Home dir for ardb instance, it can be referenced by ${ARDB_HOME} in this config file
|
||||
home .
|
||||
|
||||
# Note on units: when memory size is needed, it is possible to specify
|
||||
# it in the usual form of 1k 5GB 4M and so forth:
|
||||
#
|
||||
# 1k => 1000 bytes
|
||||
# 1kb => 1024 bytes
|
||||
# 1m => 1000000 bytes
|
||||
# 1mb => 1024*1024 bytes
|
||||
# 1g => 1000000000 bytes
|
||||
# 1gb => 1024*1024*1024 bytes
|
||||
#
|
||||
# units are case insensitive so 1GB 1Gb 1gB are all the same.
|
||||
|
||||
# By default Ardb does not run as a daemon. Use 'yes' if you need it.
|
||||
daemonize yes
|
||||
|
||||
# When running daemonized, Ardb writes a pid file in ${ARDB_HOME}/ardb.pid by
|
||||
# default. You can specify a custom pid file location here.
|
||||
pidfile ${ARDB_HOME}/ardb.pid
|
||||
|
||||
# The thread pool size for the corresponding all listen servers, -1 means current machine's cpu number
|
||||
thread-pool-size -1
|
||||
|
||||
#Accept connections on the specified host&port/unix socket, default is 0.0.0.0:16379.
|
||||
#server[0].listen 0.0.0.0:16579
|
||||
# If current qps exceed the limit, Ardb would return an error.
|
||||
#server[0].qps-limit 1000
|
||||
|
||||
#listen on unix socket
|
||||
server[0].listen storage.sock
|
||||
server[0].unixsocketperm 755
|
||||
#server[1].qps-limit 1000
|
||||
|
||||
# 'qps-limit-per-host' used to limit the request per second from same host
|
||||
# 'qps-limit-per-connection' used to limit the request per second from same connection
|
||||
qps-limit-per-host 0
|
||||
qps-limit-per-connection 0
|
||||
|
||||
# Specify the optimized RocksDB compaction strategies.
|
||||
# If anything other than none is set then the rocksdb.options will not be used.
|
||||
# The property can one of:
|
||||
# OptimizeLevelStyleCompaction
|
||||
# OptimizeUniversalStyleCompaction
|
||||
# none
|
||||
#
|
||||
rocksdb.compaction OptimizeLevelStyleCompaction
|
||||
|
||||
# Enable this to indicate that hsca/sscan/zscan command use total order mode for rocksdb engine
|
||||
rocksdb.scan-total-order false
|
||||
|
||||
# Disable RocksDB WAL may improve the write performance but
|
||||
# data in the un-flushed memtables might be lost in case of a RocksDB shutdown.
|
||||
# Disabling WAL provides similar guarantees as Redis.
|
||||
rocksdb.disableWAL false
|
||||
|
||||
#rocksdb's options
|
||||
rocksdb.options write_buffer_size=1024M;max_write_buffer_number=5;min_write_buffer_number_to_merge=3;compression=kSnappyCompression;\
|
||||
bloom_locality=1;memtable_prefix_bloom_size_ratio=0.1;\
|
||||
block_based_table_factory={block_cache=512M;filter_policy=bloomfilter:10:true};\
|
||||
create_if_missing=true;max_open_files=10000;rate_limiter_bytes_per_sec=50M;\
|
||||
use_direct_io_for_flush_and_compaction=true;use_adaptive_mutex=true
|
||||
|
||||
#leveldb's options
|
||||
leveldb.options block_cache_size=512M,write_buffer_size=128M,max_open_files=5000,block_size=4k,block_restart_interval=16,\
|
||||
bloom_bits=10,compression=snappy,logenable=yes,max_file_size=2M
|
||||
|
||||
#lmdb's options
|
||||
lmdb.options database_maxsize=10G,database_maxdbs=4096,readahead=no,batch_commit_watermark=1024
|
||||
|
||||
#perconaft's options
|
||||
perconaft.options cache_size=128M,compression=snappy
|
||||
|
||||
#wiredtiger's options
|
||||
wiredtiger.options cache_size=512M,session_max=8k,chunk_size=100M,block_size=4k,bloom_bits=10,\
|
||||
mmap=false,compressor=snappy
|
||||
|
||||
#forestdb's options
|
||||
forestdb.options chunksize=8,blocksize=4K
|
||||
|
||||
# Close the connection after a client is idle for N seconds (0 to disable)
|
||||
timeout 0
|
||||
|
||||
# TCP keepalive.
|
||||
#
|
||||
# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence
|
||||
# of communication. This is useful for two reasons:
|
||||
#
|
||||
# 1) Detect dead peers.
|
||||
# 2) Take the connection alive from the point of view of network
|
||||
# equipment in the middle.
|
||||
#
|
||||
# On Linux, the specified value (in seconds) is the period used to send ACKs.
|
||||
# Note that to close the connection the double of the time is needed.
|
||||
# On other kernels the period depends on the kernel configuration.
|
||||
#
|
||||
# A reasonable value for this option is 60 seconds.
|
||||
tcp-keepalive 0
|
||||
|
||||
# Specify the server verbosity level.
|
||||
# This can be one of:
|
||||
# error
|
||||
# warn
|
||||
# info
|
||||
# debug
|
||||
# trace
|
||||
loglevel info
|
||||
|
||||
# Specify the log file name. Also 'stdout' can be used to force
|
||||
# Redis to log on the standard output. Note that if you use standard
|
||||
# output for logging but daemonize, logs will be sent to /dev/null
|
||||
#logfile ${ARDB_HOME}/log/ardb-server.log
|
||||
logfile stdout
|
||||
|
||||
|
||||
# The working data directory.
|
||||
#
|
||||
# The DB will be written inside this directory, with the filename specified
|
||||
# above using the 'dbfilename' configuration directive.
|
||||
#
|
||||
# The Append Only File will also be created inside this directory.
|
||||
#
|
||||
# Note that you must specify a directory here, not a file name.
|
||||
data-dir ${ARDB_HOME}/data
|
||||
|
||||
################################# REPLICATION #################################
|
||||
|
||||
# Master-Slave replication. Use slaveof to make a Ardb instance a copy of
|
||||
# another Ardb server. Note that the configuration is local to the slave
|
||||
# so for example it is possible to configure the slave to save the DB with a
|
||||
# different interval, or to listen to another port, and so on.
|
||||
#
|
||||
# slaveof <masterip>:<masterport>
|
||||
#slaveof 127.0.0.1:6379
|
||||
|
||||
|
||||
# By default, ardb use 2 threads to execute commands synced from master.
|
||||
# -1 means use current CPU number threads instead.
|
||||
slave-workers 2
|
||||
|
||||
# Max synced command queue size in memory.
|
||||
max-slave-worker-queue 1024
|
||||
|
||||
# The directory for replication.
|
||||
repl-dir ${ARDB_HOME}/repl
|
||||
|
||||
|
||||
# When a slave loses its connection with the master, or when the replication
|
||||
# is still in progress, the slave can act in two different ways:
|
||||
#
|
||||
# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
|
||||
# still reply to client requests, possibly with out of date data, or the
|
||||
# data set may just be empty if this is the first synchronization.
|
||||
#
|
||||
# 2) if slave-serve-stale-data is set to 'no' the slave will reply with
|
||||
# an error "SYNC with master in progress" to all the kind of commands
|
||||
# but to INFO and SLAVEOF.
|
||||
#
|
||||
slave-serve-stale-data yes
|
||||
|
||||
# The slave priority is an integer number published by Ardb/Redis in the INFO output.
|
||||
# It is used by Redis Sentinel in order to select a slave to promote into a
|
||||
# master if the master is no longer working correctly.
|
||||
#
|
||||
# A slave with a low priority number is considered better for promotion, so
|
||||
# for instance if there are three slaves with priority 10, 100, 25 Sentinel will
|
||||
# pick the one with priority 10, that is the lowest.
|
||||
#
|
||||
# However a special priority of 0 marks the slave as not able to perform the
|
||||
# role of master, so a slave with priority of 0 will never be selected by
|
||||
# Redis Sentinel for promotion.
|
||||
#
|
||||
# By default the priority is 100.
|
||||
slave-priority 100
|
||||
|
||||
# You can configure a slave instance to accept writes or not. Writing against
|
||||
# a slave instance may be useful to store some ephemeral data (because data
|
||||
# written on a slave will be easily deleted after resync with the master) but
|
||||
# may also cause problems if clients are writing to it because of a
|
||||
# misconfiguration.
|
||||
#
|
||||
# Note: read only slaves are not designed to be exposed to untrusted clients
|
||||
# on the internet. It's just a protection layer against misuse of the instance.
|
||||
# Still a read only slave exports by default all the administrative commands
|
||||
# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve
|
||||
# security of read only slaves using 'rename-command' to shadow all the
|
||||
# administrative / dangerous commands.
|
||||
#
|
||||
# Note: any requests processed by non read only slaves would no write to replication
|
||||
# log and sync to connected slaves.
|
||||
slave-read-only yes
|
||||
|
||||
# The directory for backup.
|
||||
backup-dir ${ARDB_HOME}/backup
|
||||
#
|
||||
# You can configure the backup file format as 'redis' or 'ardb'. The 'ardb' format
|
||||
# can only used by ardb instance, while 'redis' format file can be used by redis
|
||||
# and ardb instance.
|
||||
backup-file-format ardb
|
||||
|
||||
|
||||
# Slaves send PINGs to server in a predefined interval. It's possible to change
|
||||
# this interval with the repl_ping_slave_period option. The default value is 10
|
||||
# seconds.
|
||||
#
|
||||
# repl-ping-slave-period 10
|
||||
|
||||
# The following option sets a timeout for both Bulk transfer I/O timeout and
|
||||
# master data or ping response timeout. The default value is 60 seconds.
|
||||
#
|
||||
# It is important to make sure that this value is greater than the value
|
||||
# specified for repl-ping-slave-period otherwise a timeout will be detected
|
||||
# every time there is low traffic between the master and the slave.
|
||||
#
|
||||
# repl-timeout 60
|
||||
|
||||
# Disable TCP_NODELAY on the slave socket after SYNC?
|
||||
#
|
||||
# If you select "yes" Ardb will use a smaller number of TCP packets and
|
||||
# less bandwidth to send data to slaves. But this can add a delay for
|
||||
# the data to appear on the slave side, up to 40 milliseconds with
|
||||
# Linux kernels using a default configuration.
|
||||
#
|
||||
# If you select "no" the delay for data to appear on the slave side will
|
||||
# be reduced but more bandwidth will be used for replication.
|
||||
#
|
||||
# By default we optimize for low latency, but in very high traffic conditions
|
||||
# or when the master and slaves are many hops away, turning this to "yes" may
|
||||
# be a good idea.
|
||||
repl-disable-tcp-nodelay no
|
||||
|
||||
# Set the replication backlog size. The backlog is a buffer that accumulates
|
||||
# slave data when slaves are disconnected for some time, so that when a slave
|
||||
# wants to reconnect again, often a full resync is not needed, but a partial
|
||||
# resync is enough, just passing the portion of data the slave missed while
|
||||
# disconnected.
|
||||
#
|
||||
# The biggest the replication backlog, the longer the time the slave can be
|
||||
# disconnected and later be able to perform a partial resynchronization.
|
||||
#
|
||||
# If the size is configured by 0, then Ardb instance can NOT serve as a master.
|
||||
#
|
||||
# repl-backlog-size 500m
|
||||
repl-backlog-size 1G
|
||||
repl-backlog-cache-size 100M
|
||||
snapshot-max-lag-offset 500M
|
||||
|
||||
# Set the max number of snapshots. By default this limit is set to 10 snapshot.
|
||||
# Once the limit is reached Ardb would try to remove the oldest snapshots
|
||||
maxsnapshots 10
|
||||
|
||||
# It is possible for a master to stop accepting writes if there are less than
|
||||
# N slaves connected, having a lag less or equal than M seconds.
|
||||
#
|
||||
# The N slaves need to be in "online" state.
|
||||
#
|
||||
# The lag in seconds, that must be <= the specified value, is calculated from
|
||||
# the last ping received from the slave, that is usually sent every second.
|
||||
#
|
||||
# This option does not GUARANTEE that N replicas will accept the write, but
|
||||
# will limit the window of exposure for lost writes in case not enough slaves
|
||||
# are available, to the specified number of seconds.
|
||||
#
|
||||
# For example to require at least 3 slaves with a lag <= 10 seconds use:
|
||||
#
|
||||
# min-slaves-to-write 3
|
||||
# min-slaves-max-lag 10
|
||||
|
||||
# When a slave loses its connection with the master, or when the replication
|
||||
# is still in progress, the slave can act in two different ways:
|
||||
#
|
||||
# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
|
||||
# still reply to client requests, possibly with out of date data, or the
|
||||
# data set may just be empty if this is the first synchronization.
|
||||
#
|
||||
# 2) if slave-serve-stale-data is set to 'no' the slave will reply with
|
||||
# an error "SYNC with master in progress" to all the kind of commands
|
||||
# but to INFO and SLAVEOF.
|
||||
#
|
||||
slave-serve-stale-data yes
|
||||
|
||||
# After a master has no longer connected slaves for some time, the backlog
|
||||
# will be freed. The following option configures the amount of seconds that
|
||||
# need to elapse, starting from the time the last slave disconnected, for
|
||||
# the backlog buffer to be freed.
|
||||
#
|
||||
# A value of 0 means to never release the backlog.
|
||||
#
|
||||
# repl-backlog-ttl 3600
|
||||
|
||||
# Slave clear current data store before full resync to master.
|
||||
# It make sure that slave keep consistent with master's data. But slave may cost a
|
||||
# long time to delete data, it depends on
|
||||
# If set by no, then slave may have different data with master.
|
||||
slave-cleardb-before-fullresync yes
|
||||
|
||||
# Master/Slave instance would persist sync state every 'repl-backlog-sync-period' secs.
|
||||
repl-backlog-sync-period 5
|
||||
|
||||
# Slave would ignore any 'expire' setting from replication command if set by 'yes'.
|
||||
# It could be used if master is redis instance serve hot data with expire setting, slave is
|
||||
# ardb instance which persist all data.
|
||||
# Since master redis instance would generate a 'del' for each expired key, slave should ignore
|
||||
# all 'del' command too by setting 'slave-ignore-del' to 'yes' for this scenario.
|
||||
slave-ignore-expire no
|
||||
slave-ignore-del no
|
||||
|
||||
# After a master has no longer connected slaves for some time, the backlog
|
||||
# will be freed. The following option configures the amount of seconds that
|
||||
# need to elapse, starting from the time the last slave disconnected, for
|
||||
# the backlog buffer to be freed.
|
||||
#
|
||||
# A value of 0 means to never release the backlog.
|
||||
#
|
||||
# repl-backlog-ttl 3600
|
||||
|
||||
|
||||
################################## SECURITY ###################################
|
||||
|
||||
# Require clients to issue AUTH <PASSWORD> before processing any other
|
||||
# commands. This might be useful in environments in which you do not trust
|
||||
# others with access to the host running redis-server.
|
||||
#
|
||||
# This should stay commented out for backward compatibility and because most
|
||||
# people do not need auth (e.g. they run their own servers).
|
||||
#
|
||||
# Warning: since Redis is pretty fast an outside user can try up to
|
||||
# 150k passwords per second against a good box. This means that you should
|
||||
# use a very strong password otherwise it will be very easy to break.
|
||||
#
|
||||
# requirepass foobared
|
||||
|
||||
# Command renaming.
|
||||
#
|
||||
# It is possible to change the name of dangerous commands in a shared
|
||||
# environment. For instance the CONFIG command may be renamed into something
|
||||
# hard to guess so that it will still be available for internal-use tools
|
||||
# but not available for general clients.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52
|
||||
#
|
||||
# It is also possible to completely kill a command by renaming it into
|
||||
# an empty string:
|
||||
#
|
||||
# rename-command CONFIG ""
|
||||
#
|
||||
# Please note that changing the name of commands that are logged into the
|
||||
# AOF file or transmitted to slaves may cause problems.
|
||||
|
||||
################################ CLUSTER ###############################
|
||||
|
||||
# Max execution time of a Lua script in milliseconds.
|
||||
#zookeeper-servers 127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183
|
||||
#zk-recv-timeout 10000
|
||||
#zk-clientid-file ${ARDB_HOME}/ardb.zkclientid
|
||||
cluster-name ardb-cluster
|
||||
|
||||
|
||||
################################### LIMITS ####################################
|
||||
|
||||
# Set the max number of connected clients at the same time. By default
|
||||
# this limit is set to 10000 clients, however if the Redis server is not
|
||||
# able to configure the process file limit to allow for the specified limit
|
||||
# the max number of allowed clients is set to the current file limit
|
||||
# minus 32 (as Redis reserves a few file descriptors for internal uses).
|
||||
#
|
||||
# Once the limit is reached Redis will close all the new connections sending
|
||||
# an error 'max number of clients reached'.
|
||||
#
|
||||
# maxclients 10000
|
||||
|
||||
|
||||
# The client output buffer limits can be used to force disconnection of clients
|
||||
# that are not reading data from the server fast enough for some reason (a
|
||||
# common reason is that a Pub/Sub/Slave client can't consume messages as fast as the
|
||||
# publisher can produce them).
|
||||
slave-client-output-buffer-limit 256mb
|
||||
pubsub-client-output-buffer-limit 32mb
|
||||
|
||||
################################## SLOW LOG ###################################
|
||||
|
||||
# The Redis Slow Log is a system to log queries that exceeded a specified
|
||||
# execution time. The execution time does not include the I/O operations
|
||||
# like talking with the client, sending the reply and so forth,
|
||||
# but just the time needed to actually execute the command (this is the only
|
||||
# stage of command execution where the thread is blocked and can not serve
|
||||
# other requests in the meantime).
|
||||
#
|
||||
# You can configure the slow log with two parameters: one tells Redis
|
||||
# what is the execution time, in microseconds, to exceed in order for the
|
||||
# command to get logged, and the other parameter is the length of the
|
||||
# slow log. When a new command is logged the oldest one is removed from the
|
||||
# queue of logged commands.
|
||||
|
||||
# The following time is expressed in microseconds, so 1000000 is equivalent
|
||||
# to one second. Note that a negative number disables the slow log, while
|
||||
# a value of zero forces the logging of every command.
|
||||
slowlog-log-slower-than 10000
|
||||
|
||||
# There is no limit to this length. Just be aware that it will consume memory.
|
||||
# You can reclaim memory used by the slow log with SLOWLOG RESET.
|
||||
slowlog-max-len 128
|
||||
|
||||
################################ LUA SCRIPTING ###############################
|
||||
|
||||
# Max execution time of a Lua script in milliseconds.
|
||||
#
|
||||
# If the maximum execution time is reached Redis will log that a script is
|
||||
# still in execution after the maximum allowed time and will start to
|
||||
# reply to queries with an error.
|
||||
#
|
||||
# When a long running script exceed the maximum execution time only the
|
||||
# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be
|
||||
# used to stop a script that did not yet called write commands. The second
|
||||
# is the only way to shut down the server in the case a write commands was
|
||||
# already issue by the script but the user don't want to wait for the natural
|
||||
# termination of the script.
|
||||
#
|
||||
# Set it to 0 or a negative value for unlimited execution without warnings.
|
||||
lua-time-limit 5000
|
||||
|
||||
############################### ADVANCED CONFIG ###############################
|
||||
## Since some redis clients would check info command's output, this configuration
|
||||
## would be set in 'misc' section of 'info's output
|
||||
#additional-misc-info redis_version:2.8.9\nredis_trick:yes
|
||||
|
||||
|
||||
# HyperLogLog sparse representation bytes limit. The limit includes the
|
||||
# 16 bytes header. When an HyperLogLog using the sparse representation crosses
|
||||
# this limit, it is convereted into the dense representation.
|
||||
#
|
||||
# A value greater than 16000 is totally useless, since at that point the
|
||||
# dense representation is more memory efficient.
|
||||
#
|
||||
# The suggested value is ~ 3000 in order to have the benefits of
|
||||
# the space efficient encoding without slowing down too much PFADD,
|
||||
# which is O(N) with the sparse encoding. Thev value can be raised to
|
||||
# ~ 10000 when CPU is not a concern, but space is, and the data set is
|
||||
# composed of many HyperLogLogs with cardinality in the 0 - 15000 range.
|
||||
hll-sparse-max-bytes 3000
|
||||
|
||||
#trusted-ip 10.10.10.10
|
||||
#trusted-ip 10.10.10.*
|
||||
|
||||
# By default Ardb would not compact whole db after loading a snapshot, which may happens
|
||||
# when slave syncing from master, processing 'import' command from client.
|
||||
# This configuration only works with rocksdb engine.
|
||||
# If ardb dord not compact data after loading snapshot file, there would be poor read performance before rocksdb
|
||||
# completes the next compaction task internally. While the compaction task would cost very long time for a huge data set.
|
||||
compact-after-snapshot-load false
|
||||
|
||||
# Ardb would store cursor in memory
|
||||
scan-redis-compatible yes
|
||||
scan-cursor-expire-after 60
|
||||
|
||||
redis-compatible-mode yes
|
||||
redis-compatible-version 2.8.0
|
||||
|
||||
statistics-log-period 600
|
||||
|
||||
|
||||
# Range deletion min size trigger
|
||||
range-delete-min-size 100
|
||||
|
|
@ -0,0 +1,497 @@
|
|||
################################ GENERAL #####################################
|
||||
|
||||
# By default kvrocks listens for connections from all the network interfaces
|
||||
# available on the server. It is possible to listen to just one or multiple
|
||||
# interfaces using the "bind" configuration directive, followed by one or
|
||||
# more IP addresses.
|
||||
#
|
||||
# Examples:
|
||||
#
|
||||
# bind 192.168.1.100 10.0.0.1
|
||||
# bind 127.0.0.1
|
||||
bind 0.0.0.0
|
||||
|
||||
# Accept connections on the specified port, default is 6666.
|
||||
port 5188
|
||||
|
||||
# Close the connection after a client is idle for N seconds (0 to disable)
|
||||
timeout 0
|
||||
|
||||
# The number of worker's threads, increase or decrease it would effect the performance.
|
||||
workers 8
|
||||
|
||||
# The number of replication worker's threads, increase or decrease it would effect the replication performance.
|
||||
# Default: 1
|
||||
repl-workers 1
|
||||
|
||||
# By default kvrocks does not run as a daemon. Use 'yes' if you need it.
|
||||
# Note that kvrocks will write a pid file in /var/run/kvrocks.pid when daemonized.
|
||||
daemonize no
|
||||
|
||||
# Kvrocks implements cluster solution that is similar with redis cluster sulution.
|
||||
# You can get cluster information by CLUSTER NODES|SLOTS|INFO command, it also is
|
||||
# adapted to redis-cli, redis-benchmark, redis cluster SDK and redis cluster proxy.
|
||||
# But kvrocks doesn't support to communicate with each others, so you must set
|
||||
# cluster topology by CLUSTER SETNODES|SETNODEID commands, more details: #219.
|
||||
#
|
||||
# PLEASE NOTE:
|
||||
# If you enable cluster, kvrocks will encode key with its slot id calculated by
|
||||
# CRC16 and modulo 16384, endoding key with its slot id makes it efficient to
|
||||
# migrate keys based on slot. So if you enabled at first time, cluster mode must
|
||||
# not be disabled after restarting, and vice versa. That is to say, data is not
|
||||
# compatible between standalone mode with cluster mode, you must migrate data
|
||||
# if you want to change mode, otherwise, kvrocks will make data corrupt.
|
||||
#
|
||||
# Default: no
|
||||
cluster-enabled no
|
||||
|
||||
# Set the max number of connected clients at the same time. By default
|
||||
# this limit is set to 10000 clients, however if the server is not
|
||||
# able to configure the process file limit to allow for the specified limit
|
||||
# the max number of allowed clients is set to the current file limit
|
||||
#
|
||||
# Once the limit is reached the server will close all the new connections sending
|
||||
# an error 'max number of clients reached'.
|
||||
#
|
||||
maxclients 10000
|
||||
|
||||
# Require clients to issue AUTH <PASSWORD> before processing any other
|
||||
# commands. This might be useful in environments in which you do not trust
|
||||
# others with access to the host running kvrocks.
|
||||
#
|
||||
# This should stay commented out for backward compatibility and because most
|
||||
# people do not need auth (e.g. they run their own servers).
|
||||
#
|
||||
# Warning: since kvrocks is pretty fast an outside user can try up to
|
||||
# 150k passwords per second against a good box. This means that you should
|
||||
# use a very strong password otherwise it will be very easy to break.
|
||||
#
|
||||
# requirepass foobared
|
||||
|
||||
# If the master is password protected (using the "masterauth" configuration
|
||||
# directive below) it is possible to tell the slave to authenticate before
|
||||
# starting the replication synchronization process, otherwise the master will
|
||||
# refuse the slave request.
|
||||
#
|
||||
# masterauth foobared
|
||||
|
||||
# Master-Salve replication would check db name is matched. if not, the slave should
|
||||
# refuse to sync the db from master. Don't use default value, set the db-name to identify
|
||||
# the cluster.
|
||||
db-name storage.db
|
||||
|
||||
# The working directory
|
||||
#
|
||||
# The DB will be written inside this directory
|
||||
# Note that you must specify a directory here, not a file name.
|
||||
dir ./
|
||||
|
||||
# The logs of server will be stored in this directory. If you don't specify
|
||||
# one directory, by default, we store logs in the working directory that set
|
||||
# by 'dir' above.
|
||||
# log-dir /tmp/kvrocks
|
||||
|
||||
# When running daemonized, kvrocks writes a pid file in ${CONFIG_DIR}/kvrocks.pid by
|
||||
# default. You can specify a custom pid file location here.
|
||||
# pidfile /var/run/kvrocks.pid
|
||||
pidfile storage.pid
|
||||
|
||||
# You can configure a slave instance to accept writes or not. Writing against
|
||||
# a slave instance may be useful to store some ephemeral data (because data
|
||||
# written on a slave will be easily deleted after resync with the master) but
|
||||
# may also cause problems if clients are writing to it because of a
|
||||
# misconfiguration.
|
||||
slave-read-only yes
|
||||
|
||||
# The slave priority is an integer number published by Kvrocks in the INFO output.
|
||||
# It is used by Redis Sentinel in order to select a slave to promote into a
|
||||
# master if the master is no longer working correctly.
|
||||
#
|
||||
# A slave with a low priority number is considered better for promotion, so
|
||||
# for instance if there are three slave with priority 10, 100, 25 Sentinel will
|
||||
# pick the one with priority 10, that is the lowest.
|
||||
#
|
||||
# However a special priority of 0 marks the replica as not able to perform the
|
||||
# role of master, so a slave with priority of 0 will never be selected by
|
||||
# Redis Sentinel for promotion.
|
||||
#
|
||||
# By default the priority is 100.
|
||||
slave-priority 100
|
||||
|
||||
# TCP listen() backlog.
|
||||
#
|
||||
# In high requests-per-second environments you need an high backlog in order
|
||||
# to avoid slow clients connections issues. Note that the Linux kernel
|
||||
# will silently truncate it to the value of /proc/sys/net/core/somaxconn so
|
||||
# make sure to raise both the value of somaxconn and tcp_max_syn_backlog
|
||||
# in order to Get the desired effect.
|
||||
tcp-backlog 511
|
||||
|
||||
# If the master is an old version, it may have specified replication threads
|
||||
# that use 'port + 1' as listening port, but in new versions, we don't use
|
||||
# extra port to implement replication. In order to allow the new replicas to
|
||||
# copy old masters, you should indicate that the master uses replication port
|
||||
# or not.
|
||||
# If yes, that indicates master uses replication port and replicas will connect
|
||||
# to 'master's listening port + 1' when synchronization.
|
||||
# If no, that indicates master doesn't use replication port and replicas will
|
||||
# connect 'master's listening port' when synchronization.
|
||||
master-use-repl-port no
|
||||
|
||||
# Master-Slave replication. Use slaveof to make a kvrocks instance a copy of
|
||||
# another kvrocks server. A few things to understand ASAP about kvrocks replication.
|
||||
#
|
||||
# 1) Kvrocks replication is asynchronous, but you can configure a master to
|
||||
# stop accepting writes if it appears to be not connected with at least
|
||||
# a given number of slaves.
|
||||
# 2) Kvrocks slaves are able to perform a partial resynchronization with the
|
||||
# master if the replication link is lost for a relatively small amount of
|
||||
# time. You may want to configure the replication backlog size (see the next
|
||||
# sections of this file) with a sensible value depending on your needs.
|
||||
# 3) Replication is automatic and does not need user intervention. After a
|
||||
# network partition slaves automatically try to reconnect to masters
|
||||
# and resynchronize with them.
|
||||
#
|
||||
# slaveof <masterip> <masterport>
|
||||
# slaveof 127.0.0.1 6379
|
||||
|
||||
# When a slave loses its connection with the master, or when the replication
|
||||
# is still in progress, the slave can act in two different ways:
|
||||
#
|
||||
# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
|
||||
# still reply to client requests, possibly with out of date data, or the
|
||||
# data set may just be empty if this is the first synchronization.
|
||||
#
|
||||
# 2) if slave-serve-stale-data is set to 'no' the slave will reply with
|
||||
# an error "SYNC with master in progress" to all the kind of commands
|
||||
# but to INFO and SLAVEOF.
|
||||
#
|
||||
slave-serve-stale-data yes
|
||||
|
||||
# To guarantee slave's data safe and serve when it is in full synchronization
|
||||
# state, slave still keep itself data. But this way needs to occupy much disk
|
||||
# space, so we provide a way to reduce disk occupation, slave will delete itself
|
||||
# entire database before fetching files from master during full synchronization.
|
||||
# If you want to enable this way, you can set 'slave-delete-db-before-fullsync'
|
||||
# to yes, but you must know that database will be lost if master is down during
|
||||
# full synchronization, unless you have a backup of database.
|
||||
#
|
||||
# This option is similar redis replicas RDB diskless load option:
|
||||
# repl-diskless-load on-empty-db
|
||||
#
|
||||
# Default: no
|
||||
slave-empty-db-before-fullsync no
|
||||
|
||||
# If replicas need full synchronization with master, master need to create
|
||||
# checkpoint for feeding replicas, and replicas also stage a checkpoint of
|
||||
# the master. If we also keep the backup, it maybe occupy extra disk space.
|
||||
# You can enable 'purge-backup-on-fullsync' if disk is not sufficient, but
|
||||
# that may cause remote backup copy failing.
|
||||
#
|
||||
# Default: no
|
||||
purge-backup-on-fullsync no
|
||||
|
||||
# The maximum allowed rate (in MB/s) that should be used by Replication.
|
||||
# If the rate exceeds max-replication-mb, replication will slow down.
|
||||
# Default: 0 (i.e. no limit)
|
||||
max-replication-mb 0
|
||||
|
||||
# The maximum allowed aggregated write rate of flush and compaction (in MB/s).
|
||||
# If the rate exceeds max-io-mb, io will slow down.
|
||||
# 0 is no limit
|
||||
# Default: 500
|
||||
max-io-mb 500
|
||||
|
||||
# The maximum allowed space (in GB) that should be used by RocksDB.
|
||||
# If the total size of the SST files exceeds max_allowed_space, writes to RocksDB will fail.
|
||||
# Please see: https://github.com/facebook/rocksdb/wiki/Managing-Disk-Space-Utilization
|
||||
# Default: 0 (i.e. no limit)
|
||||
max-db-size 0
|
||||
|
||||
# The maximum backup to keep, server cron would run every minutes to check the num of current
|
||||
# backup, and purge the old backup if exceed the max backup num to keep. If max-backup-to-keep
|
||||
# is 0, no backup would be keep. But now, we only support 0 or 1.
|
||||
max-backup-to-keep 1
|
||||
|
||||
# The maximum hours to keep the backup. If max-backup-keep-hours is 0, wouldn't purge any backup.
|
||||
# default: 1 day
|
||||
max-backup-keep-hours 24
|
||||
|
||||
# Ratio of the samples would be recorded when the profiling was enabled.
|
||||
# we simply use the rand to determine whether to record the sample or not.
|
||||
#
|
||||
# Default: 0
|
||||
profiling-sample-ratio 0
|
||||
|
||||
# There is no limit to this length. Just be aware that it will consume memory.
|
||||
# You can reclaim memory used by the perf log with PERFLOG RESET.
|
||||
#
|
||||
# Default: 256
|
||||
profiling-sample-record-max-len 256
|
||||
|
||||
# profiling-sample-record-threshold-ms use to tell the kvrocks when to record.
|
||||
#
|
||||
# Default: 100 millisecond
|
||||
profiling-sample-record-threshold-ms 100
|
||||
|
||||
################################## SLOW LOG ###################################
|
||||
|
||||
# The Kvrocks Slow Log is a system to log queries that exceeded a specified
|
||||
# execution time. The execution time does not include the I/O operations
|
||||
# like talking with the client, sending the reply and so forth,
|
||||
# but just the time needed to actually execute the command (this is the only
|
||||
# stage of command execution where the thread is blocked and can not serve
|
||||
# other requests in the meantime).
|
||||
#
|
||||
# You can configure the slow log with two parameters: one tells Kvrocks
|
||||
# what is the execution time, in microseconds, to exceed in order for the
|
||||
# command to get logged, and the other parameter is the length of the
|
||||
# slow log. When a new command is logged the oldest one is removed from the
|
||||
# queue of logged commands.
|
||||
|
||||
# The following time is expressed in microseconds, so 1000000 is equivalent
|
||||
# to one second. Note that -1 value disables the slow log, while
|
||||
# a value of zero forces the logging of every command.
|
||||
slowlog-log-slower-than 100000
|
||||
|
||||
# There is no limit to this length. Just be aware that it will consume memory.
|
||||
# You can reclaim memory used by the slow log with SLOWLOG RESET.
|
||||
slowlog-max-len 128
|
||||
|
||||
# If you run kvrocks from upstart or systemd, kvrocks can interact with your
|
||||
# supervision tree. Options:
|
||||
# supervised no - no supervision interaction
|
||||
# supervised upstart - signal upstart by putting kvrocks into SIGSTOP mode
|
||||
# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
|
||||
# supervised auto - detect upstart or systemd method based on
|
||||
# UPSTART_JOB or NOTIFY_SOCKET environment variables
|
||||
# Note: these supervision methods only signal "process is ready."
|
||||
# They do not enable continuous liveness pings back to your supervisor.
|
||||
supervised no
|
||||
|
||||
################################## CRON ###################################
|
||||
|
||||
# Compact Scheduler, auto compact at schedule time
|
||||
# time expression format is the same as crontab(currently only support * and int)
|
||||
# e.g. compact-cron 0 3 * * * 0 4 * * *
|
||||
# would compact the db at 3am and 4am everyday
|
||||
# compact-cron 0 3 * * *
|
||||
|
||||
# The hour range that compaction checker would be active
|
||||
# e.g. compaction-checker-range 0-7 means compaction checker would be worker between
|
||||
# 0-7am every day.
|
||||
compaction-checker-range 0-7
|
||||
|
||||
# Bgsave scheduler, auto bgsave at schedule time
|
||||
# time expression format is the same as crontab(currently only support * and int)
|
||||
# e.g. bgsave-cron 0 3 * * * 0 4 * * *
|
||||
# would bgsave the db at 3am and 4am everyday
|
||||
|
||||
# Command renaming.
|
||||
#
|
||||
# It is possible to change the name of dangerous commands in a shared
|
||||
# environment. For instance the KEYS command may be renamed into something
|
||||
# hard to guess so that it will still be available for internal-use tools
|
||||
# but not available for general clients.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# rename-command KEYS b840fc02d524045429941cc15f59e41cb7be6c52
|
||||
#
|
||||
# It is also possible to completely kill a command by renaming it into
|
||||
# an empty string:
|
||||
#
|
||||
# rename-command KEYS ""
|
||||
|
||||
# The key-value size may so be quite different in many scenes, and use 256MiB as SST file size
|
||||
# may cause data loading(large index/filter block) ineffective when the key-value was too small.
|
||||
# kvrocks supports user-defined SST file in config(rocksdb.target_file_size_base),
|
||||
# but it still too trivial and inconvenient to adjust the different sizes for different instances.
|
||||
# so we want to periodic auto-adjust the SST size in-flight with user avg key-value size.
|
||||
#
|
||||
# If enabled, kvrocks will auto resize rocksdb.target_file_size_base
|
||||
# and rocksdb.write_buffer_size in-flight with user avg key-value size.
|
||||
# Please see #118.
|
||||
#
|
||||
# Default: yes
|
||||
auto-resize-block-and-sst yes
|
||||
|
||||
################################ ROCKSDB #####################################
|
||||
|
||||
# Specify the capacity of metadata column family block cache. Larger block cache
|
||||
# may make request faster while more keys would be cached. Max Size is 200*1024.
|
||||
# Default: 2048MB
|
||||
rocksdb.metadata_block_cache_size 2048
|
||||
|
||||
# Specify the capacity of subkey column family block cache. Larger block cache
|
||||
# may make request faster while more keys would be cached. Max Size is 200*1024.
|
||||
# Default: 2048MB
|
||||
rocksdb.subkey_block_cache_size 2048
|
||||
|
||||
# Metadata column family and subkey column family will share a single block cache
|
||||
# if set 'yes'. The capacity of shared block cache is
|
||||
# metadata_block_cache_size + subkey_block_cache_size
|
||||
#
|
||||
# Default: yes
|
||||
rocksdb.share_metadata_and_subkey_block_cache yes
|
||||
|
||||
# Number of open files that can be used by the DB. You may need to
|
||||
# increase this if your database has a large working set. Value -1 means
|
||||
# files opened are always kept open. You can estimate number of files based
|
||||
# on target_file_size_base and target_file_size_multiplier for level-based
|
||||
# compaction. For universal-style compaction, you can usually set it to -1.
|
||||
# Default: 4096
|
||||
rocksdb.max_open_files 8096
|
||||
|
||||
# Amount of data to build up in memory (backed by an unsorted log
|
||||
# on disk) before converting to a sorted on-disk file.
|
||||
#
|
||||
# Larger values increase performance, especially during bulk loads.
|
||||
# Up to max_write_buffer_number write buffers may be held in memory
|
||||
# at the same time,
|
||||
# so you may wish to adjust this parameter to control memory usage.
|
||||
# Also, a larger write buffer will result in a longer recovery time
|
||||
# the next time the database is opened.
|
||||
#
|
||||
# Note that write_buffer_size is enforced per column family.
|
||||
# See db_write_buffer_size for sharing memory across column families.
|
||||
|
||||
# default is 64MB
|
||||
rocksdb.write_buffer_size 16
|
||||
|
||||
# Target file size for compaction, target file size for Leve N can be caculated
|
||||
# by target_file_size_base * (target_file_size_multiplier ^ (L-1))
|
||||
#
|
||||
# Default: 128MB
|
||||
rocksdb.target_file_size_base 16
|
||||
|
||||
# The maximum number of write buffers that are built up in memory.
|
||||
# The default and the minimum number is 2, so that when 1 write buffer
|
||||
# is being flushed to storage, new writes can continue to the other
|
||||
# write buffer.
|
||||
# If max_write_buffer_number > 3, writing will be slowed down to
|
||||
# options.delayed_write_rate if we are writing to the last write buffer
|
||||
# allowed.
|
||||
rocksdb.max_write_buffer_number 4
|
||||
|
||||
# Maximum number of concurrent background compaction jobs, submitted to
|
||||
# the default LOW priority thread pool.
|
||||
rocksdb.max_background_compactions 4
|
||||
|
||||
# Maximum number of concurrent background memtable flush jobs, submitted by
|
||||
# default to the HIGH priority thread pool. If the HIGH priority thread pool
|
||||
# is configured to have zero threads, flush jobs will share the LOW priority
|
||||
# thread pool with compaction jobs.
|
||||
rocksdb.max_background_flushes 4
|
||||
|
||||
# This value represents the maximum number of threads that will
|
||||
# concurrently perform a compaction job by breaking it into multiple,
|
||||
# smaller ones that are run simultaneously.
|
||||
# Default: 2 (i.e. no subcompactions)
|
||||
rocksdb.max_sub_compactions 2
|
||||
|
||||
# In order to limit the size of WALs, RocksDB uses DBOptions::max_total_wal_size
|
||||
# as the trigger of column family flush. Once WALs exceed this size, RocksDB
|
||||
# will start forcing the flush of column families to allow deletion of some
|
||||
# oldest WALs. This config can be useful when column families are updated at
|
||||
# non-uniform frequencies. If there's no size limit, users may need to keep
|
||||
# really old WALs when the infrequently-updated column families hasn't flushed
|
||||
# for a while.
|
||||
#
|
||||
# In kvrocks, we use multiple column families to store metadata, subkeys, etc.
|
||||
# If users always use string type, but use list, hash and other complex data types
|
||||
# infrequently, there will be a lot of old WALs if we don't set size limit
|
||||
# (0 by default in rocksdb), because rocksdb will dynamically choose the WAL size
|
||||
# limit to be [sum of all write_buffer_size * max_write_buffer_number] * 4 if set to 0.
|
||||
#
|
||||
# Moreover, you should increase this value if you already set rocksdb.write_buffer_size
|
||||
# to a big value, to avoid influencing the effect of rocksdb.write_buffer_size and
|
||||
# rocksdb.max_write_buffer_number.
|
||||
#
|
||||
# default is 512MB
|
||||
rocksdb.max_total_wal_size 512
|
||||
|
||||
# We impl the repliction with rocksdb WAL, it would trigger full sync when the seq was out of range.
|
||||
# wal_ttl_seconds and wal_size_limit_mb would affect how archived logswill be deleted.
|
||||
# If WAL_ttl_seconds is not 0, then WAL files will be checked every WAL_ttl_seconds / 2 and those that
|
||||
# are older than WAL_ttl_seconds will be deleted#
|
||||
#
|
||||
# Default: 3 Hours
|
||||
rocksdb.wal_ttl_seconds 10800
|
||||
|
||||
# If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0,
|
||||
# WAL files will be checked every 10 min and if total size is greater
|
||||
# then WAL_size_limit_MB, they will be deleted starting with the
|
||||
# earliest until size_limit is met. All empty files will be deleted
|
||||
# Default: 16GB
|
||||
rocksdb.wal_size_limit_mb 16384
|
||||
|
||||
# Approximate size of user data packed per block. Note that the
|
||||
# block size specified here corresponds to uncompressed data. The
|
||||
# actual size of the unit read from disk may be smaller if
|
||||
# compression is enabled.
|
||||
#
|
||||
# Default: 4KB
|
||||
rocksdb.block_size 2048
|
||||
|
||||
# Indicating if we'd put index/filter blocks to the block cache
|
||||
#
|
||||
# Default: no
|
||||
rocksdb.cache_index_and_filter_blocks yes
|
||||
|
||||
# Specify the compression to use.
|
||||
# Accept value: "no", "snappy"
|
||||
# default snappy
|
||||
rocksdb.compression snappy
|
||||
|
||||
# If non-zero, we perform bigger reads when doing compaction. If you're
|
||||
# running RocksDB on spinning disks, you should set this to at least 2MB.
|
||||
# That way RocksDB's compaction is doing sequential instead of random reads.
|
||||
# When non-zero, we also force new_table_reader_for_compaction_inputs to
|
||||
# true.
|
||||
#
|
||||
# Default: 2 MB
|
||||
rocksdb.compaction_readahead_size 2097152
|
||||
|
||||
# he limited write rate to DB if soft_pending_compaction_bytes_limit or
|
||||
# level0_slowdown_writes_trigger is triggered.
|
||||
|
||||
# If the value is 0, we will infer a value from `rater_limiter` value
|
||||
# if it is not empty, or 16MB if `rater_limiter` is empty. Note that
|
||||
# if users change the rate in `rate_limiter` after DB is opened,
|
||||
# `delayed_write_rate` won't be adjusted.
|
||||
#
|
||||
rocksdb.delayed_write_rate 0
|
||||
# If enable_pipelined_write is true, separate write thread queue is
|
||||
# maintained for WAL write and memtable write.
|
||||
#
|
||||
# Default: no
|
||||
rocksdb.enable_pipelined_write no
|
||||
|
||||
# Soft limit on number of level-0 files. We start slowing down writes at this
|
||||
# point. A value <0 means that no writing slow down will be triggered by
|
||||
# number of files in level-0.
|
||||
#
|
||||
# Default: 20
|
||||
rocksdb.level0_slowdown_writes_trigger 20
|
||||
|
||||
# Maximum number of level-0 files. We stop writes at this point.
|
||||
#
|
||||
# Default: 40
|
||||
rocksdb.level0_stop_writes_trigger 40
|
||||
|
||||
# if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec
|
||||
#
|
||||
# Default: 0
|
||||
rocksdb.stats_dump_period_sec 0
|
||||
|
||||
# if yes, the auto compaction would be disabled, but the manual compaction remain works
|
||||
#
|
||||
# Default: no
|
||||
rocksdb.disable_auto_compactions no
|
||||
################################ NAMESPACE #####################################
|
||||
# namespace.test change.me
|
||||
|
||||
|
||||
backup-dir .//backup
|
||||
log-dir ./
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
../../ardb/src/ardb-server ardb.conf
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
../../kvrocks/src/kvrocks -c kvrocks.conf
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
../../redis/src/redis-cli -s ./storage.sock shutdown save
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import requests
|
||||
|
||||
from bgpranking.default import get_homedir
|
||||
|
||||
d3js_version = '7'
|
||||
bootstrap_select_version = "1.13.18"
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dest_dir = get_homedir() / 'website' / 'web' / 'static'
|
||||
|
||||
d3 = requests.get(f'https://d3js.org/d3.v{d3js_version}.min.js')
|
||||
with (dest_dir / f'd3.v{d3js_version}.min.js').open('wb') as f:
|
||||
f.write(d3.content)
|
||||
print(f'Downloaded d3js v{d3js_version}.')
|
||||
|
||||
bootstrap_select_js = requests.get(f'https://cdn.jsdelivr.net/npm/bootstrap-select@{bootstrap_select_version}/dist/js/bootstrap-select.min.js')
|
||||
with (dest_dir / 'bootstrap-select.min.js').open('wb') as f:
|
||||
f.write(bootstrap_select_js.content)
|
||||
print(f'Downloaded bootstrap_select js v{bootstrap_select_version}.')
|
||||
|
||||
bootstrap_select_css = requests.get(f'https://cdn.jsdelivr.net/npm/bootstrap-select@{bootstrap_select_version}/dist/css/bootstrap-select.min.css')
|
||||
with (dest_dir / 'bootstrap-select.min.css').open('wb') as f:
|
||||
f.write(bootstrap_select_css.content)
|
||||
print(f'Downloaded bootstrap_select css v{bootstrap_select_version}.')
|
||||
|
||||
print('All 3rd party modules for the website were downloaded.')
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Set
|
||||
|
||||
from redis import Redis
|
||||
|
||||
redis_src = Redis(unix_socket_path='../storage/storage.sock')
|
||||
redis_dst = Redis('127.0.0.1', 5188)
|
||||
|
||||
chunk_size = 100000
|
||||
|
||||
|
||||
def process_chunk(src: Redis, dst: Redis, keys: Set[str]):
|
||||
src_pipeline = src.pipeline()
|
||||
[src_pipeline.type(key) for key in keys]
|
||||
to_process = {key: key_type for key, key_type in zip(keys, src_pipeline.execute())}
|
||||
|
||||
src_pipeline = src.pipeline()
|
||||
for key, key_type in to_process.items():
|
||||
if key_type == b"string":
|
||||
src_pipeline.get(key)
|
||||
elif key_type == b"list":
|
||||
raise Exception('Lists should not be used.')
|
||||
elif key_type == b"set":
|
||||
src_pipeline.smembers(key)
|
||||
elif key_type == b"zset":
|
||||
src_pipeline.zrangebyscore(key, '-Inf', '+Inf', withscores=True)
|
||||
elif key_type == b"hash":
|
||||
src_pipeline.hgetall(key)
|
||||
else:
|
||||
raise Exception(f'{key_type} not supported {key}.')
|
||||
|
||||
dest_pipeline = dst.pipeline()
|
||||
for key, content in zip(to_process.keys(), src_pipeline.execute()):
|
||||
if to_process[key] == b"string":
|
||||
dest_pipeline.set(key, content)
|
||||
elif to_process[key] == b"set":
|
||||
dest_pipeline.sadd(key, *content)
|
||||
elif to_process[key] == b"zset":
|
||||
dest_pipeline.zadd(key, {value: rank for value, rank in content})
|
||||
elif to_process[key] == b"hash":
|
||||
dest_pipeline.hmset(key, content)
|
||||
|
||||
dest_pipeline.execute()
|
||||
|
||||
|
||||
def migrate(src: Redis, dst: Redis):
|
||||
keys = set()
|
||||
pos = 0
|
||||
for key in src.scan_iter(count=chunk_size, match='2017*'):
|
||||
keys.add(key)
|
||||
|
||||
if len(keys) == chunk_size:
|
||||
process_chunk(src, dst, keys)
|
||||
pos += len(keys)
|
||||
print(f'{datetime.now()} - {pos} keys done.')
|
||||
keys = set()
|
||||
|
||||
# migrate remaining keys
|
||||
process_chunk(src, dst, keys)
|
||||
pos += len(keys)
|
||||
print(f'{datetime.now()} - {pos} keys done.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
migrate(redis_src, redis_dst)
|
||||
|
|
@ -1,20 +1,18 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
import json
|
||||
|
||||
from redis import StrictRedis
|
||||
from .libs.helpers import get_socket_path, get_ipasn
|
||||
from redis import Redis
|
||||
from bgpranking.default import get_socket_path
|
||||
from bgpranking.helpers import get_ipasn
|
||||
|
||||
|
||||
class Monitor():
|
||||
|
||||
def __init__(self):
|
||||
self.intake = StrictRedis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
|
||||
self.sanitize = StrictRedis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
|
||||
self.cache = StrictRedis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
self.intake = Redis(unix_socket_path=get_socket_path('intake'), db=0, decode_responses=True)
|
||||
self.sanitize = Redis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True)
|
||||
self.cache = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True)
|
||||
self.ipasn = get_ipasn()
|
||||
|
||||
def get_values(self):
|
||||
|
|
@ -26,5 +24,11 @@ class Monitor():
|
|||
if len(ipasn_meta['cached_dates']['caida']['v6']['cached']) > 15:
|
||||
ipasn_meta['cached_dates']['caida']['v6']['cached'] = 'Too many entries'
|
||||
return json.dumps({'Non-parsed IPs': ips_in_intake, 'Parsed IPs': ready_to_insert,
|
||||
'running': self.cache.hgetall('running'), 'IPASN History': ipasn_meta},
|
||||
'running': self.cache.zrangebyscore('running', '-inf', '+inf', withscores=True),
|
||||
'IPASN History': ipasn_meta},
|
||||
indent=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
m = Monitor()
|
||||
print(m.get_values())
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import logging
|
||||
import argparse
|
||||
|
||||
from bgpranking.default import get_homedir
|
||||
|
||||
|
||||
def validate_generic_config_file():
|
||||
user_config = get_homedir() / 'config' / 'generic.json'
|
||||
with user_config.open() as f:
|
||||
generic_config = json.load(f)
|
||||
with (get_homedir() / 'config' / 'generic.json.sample').open() as f:
|
||||
generic_config_sample = json.load(f)
|
||||
# Check documentation
|
||||
for key in generic_config_sample.keys():
|
||||
if key == '_notes':
|
||||
continue
|
||||
if key not in generic_config_sample['_notes']:
|
||||
raise Exception(f'###### - Documentation missing for {key}')
|
||||
|
||||
# Check all entries in the sample files are in the user file, and they have the same type
|
||||
for key in generic_config_sample.keys():
|
||||
if key == '_notes':
|
||||
continue
|
||||
if generic_config.get(key) is None:
|
||||
logger.warning(f'Entry missing in user config file: {key}. Will default to: {generic_config_sample[key]}')
|
||||
continue
|
||||
if not isinstance(generic_config[key], type(generic_config_sample[key])):
|
||||
raise Exception(f'Invalid type for {key}. Got: {type(generic_config[key])} ({generic_config[key]}), expected: {type(generic_config_sample[key])} ({generic_config_sample[key]})')
|
||||
|
||||
if isinstance(generic_config[key], dict):
|
||||
# Check entries
|
||||
for sub_key in generic_config_sample[key].keys():
|
||||
if sub_key not in generic_config[key]:
|
||||
raise Exception(f'{sub_key} is missing in generic_config[key]. Default from sample file: {generic_config_sample[key][sub_key]}')
|
||||
if not isinstance(generic_config[key][sub_key], type(generic_config_sample[key][sub_key])):
|
||||
raise Exception(f'Invalid type for {sub_key} in {key}. Got: {type(generic_config[key][sub_key])} ({generic_config[key][sub_key]}), expected: {type(generic_config_sample[key][sub_key])} ({generic_config_sample[key][sub_key]})')
|
||||
|
||||
# Make sure the user config file doesn't have entries missing in the sample config
|
||||
for key in generic_config.keys():
|
||||
if key not in generic_config_sample:
|
||||
raise Exception(f'{key} is missing in the sample config file. You need to compare {user_config} with {user_config}.sample.')
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def update_user_configs():
|
||||
for file_name in ['generic']:
|
||||
with (get_homedir() / 'config' / f'{file_name}.json').open() as f:
|
||||
try:
|
||||
generic_config = json.load(f)
|
||||
except Exception:
|
||||
generic_config = {}
|
||||
with (get_homedir() / 'config' / f'{file_name}.json.sample').open() as f:
|
||||
generic_config_sample = json.load(f)
|
||||
|
||||
has_new_entry = False
|
||||
for key in generic_config_sample.keys():
|
||||
if key == '_notes':
|
||||
continue
|
||||
if generic_config.get(key) is None:
|
||||
print(f'{key} was missing in {file_name}, adding it.')
|
||||
print(f"Description: {generic_config_sample['_notes'][key]}")
|
||||
generic_config[key] = generic_config_sample[key]
|
||||
has_new_entry = True
|
||||
elif isinstance(generic_config[key], dict):
|
||||
for sub_key in generic_config_sample[key].keys():
|
||||
if sub_key not in generic_config[key]:
|
||||
print(f'{sub_key} was missing in {key} from {file_name}, adding it.')
|
||||
generic_config[key][sub_key] = generic_config_sample[key][sub_key]
|
||||
has_new_entry = True
|
||||
if has_new_entry:
|
||||
with (get_homedir() / 'config' / f'{file_name}.json').open('w') as fw:
|
||||
json.dump(generic_config, fw, indent=2, sort_keys=True)
|
||||
return has_new_entry
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logger = logging.getLogger('Config validator')
|
||||
parser = argparse.ArgumentParser(description='Check the config files.')
|
||||
parser.add_argument('--check', default=False, action='store_true', help='Check if the sample config and the user config are in-line')
|
||||
parser.add_argument('--update', default=False, action='store_true', help='Update the user config with the entries from the sample config if entries are missing')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.check:
|
||||
if validate_generic_config_file():
|
||||
print(f"The entries in {get_homedir() / 'config' / 'generic.json'} are valid.")
|
||||
|
||||
if args.update:
|
||||
if not update_user_configs():
|
||||
print(f"No updates needed in {get_homedir() / 'config' / 'generic.json'}.")
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
mkdir -p web/static/
|
||||
|
||||
wget https://d3js.org/d3.v5.js -O web/static/d3.v5.js
|
||||
|
||||
BOOTSTRAP_SELECT="1.13.5"
|
||||
|
||||
wget https://cdnjs.cloudflare.com/ajax/libs/bootstrap-select/${BOOTSTRAP_SELECT}/css/bootstrap-select.min.css -O web/static/bootstrap-select.min.css
|
||||
wget https://cdnjs.cloudflare.com/ajax/libs/bootstrap-select/${BOOTSTRAP_SELECT}/js/bootstrap-select.min.js -O web/static/bootstrap-select.min.js
|
||||
|
|
@ -1,40 +1,40 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import pkg_resources
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import date, timedelta
|
||||
from urllib.parse import urljoin
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
import os
|
||||
from typing import Dict, Any, Tuple, List, Optional, Union
|
||||
|
||||
import pycountry # type: ignore
|
||||
import requests
|
||||
|
||||
from flask import Flask, render_template, request, session, Response, redirect, url_for
|
||||
from flask_bootstrap import Bootstrap
|
||||
from flask_bootstrap import Bootstrap # type: ignore
|
||||
from flask_restx import Api # type: ignore
|
||||
|
||||
from bgpranking.querying import Querying
|
||||
from bgpranking.libs.exceptions import MissingConfigEntry
|
||||
from bgpranking.libs.helpers import load_general_config, get_homedir, get_ipasn
|
||||
from datetime import date, timedelta
|
||||
import pycountry
|
||||
from collections import defaultdict
|
||||
from bgpranking.bgpranking import BGPRanking
|
||||
from bgpranking.default import get_config
|
||||
from bgpranking.helpers import get_ipasn
|
||||
|
||||
from .genericapi import api as generic_api
|
||||
from .helpers import get_secret_key
|
||||
from .proxied import ReverseProxied
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
secret_file_path = get_homedir() / 'website' / 'secret_key'
|
||||
app.wsgi_app = ReverseProxied(app.wsgi_app) # type: ignore
|
||||
|
||||
if not secret_file_path.exists() or secret_file_path.stat().st_size < 64:
|
||||
with open(secret_file_path, 'wb') as f:
|
||||
f.write(os.urandom(64))
|
||||
|
||||
with open(secret_file_path, 'rb') as f:
|
||||
app.config['SECRET_KEY'] = f.read()
|
||||
app.config['SECRET_KEY'] = get_secret_key()
|
||||
|
||||
Bootstrap(app)
|
||||
app.config['BOOTSTRAP_SERVE_LOCAL'] = True
|
||||
|
||||
bgpranking = BGPRanking()
|
||||
|
||||
|
||||
# ############# Helpers #############
|
||||
|
||||
|
|
@ -42,7 +42,7 @@ def load_session():
|
|||
if request.method == 'POST':
|
||||
d = request.form
|
||||
elif request.method == 'GET':
|
||||
d = request.args
|
||||
d = request.args # type: ignore
|
||||
|
||||
for key in d:
|
||||
if '_all' in d.getlist(key):
|
||||
|
|
@ -79,27 +79,25 @@ def index():
|
|||
# Just returns ack if the webserver is running
|
||||
return 'Ack'
|
||||
load_session()
|
||||
q = Querying()
|
||||
sources = q.get_sources(date=session['date'])['response']
|
||||
sources = bgpranking.get_sources(date=session['date'])['response']
|
||||
session.pop('asn', None)
|
||||
session.pop('country', None)
|
||||
ranks = q.asns_global_ranking(limit=100, **session)['response']
|
||||
r = [(asn, rank, q.get_asn_descriptions(int(asn))['response']) for asn, rank in ranks]
|
||||
ranks = bgpranking.asns_global_ranking(limit=100, **session)['response']
|
||||
r = [(asn, rank, bgpranking.get_asn_descriptions(int(asn))['response']) for asn, rank in ranks]
|
||||
return render_template('index.html', ranks=r, sources=sources, countries=get_country_codes(), **session)
|
||||
|
||||
|
||||
@app.route('/asn', methods=['GET', 'POST'])
|
||||
def asn_details():
|
||||
load_session()
|
||||
q = Querying()
|
||||
if 'asn' not in session:
|
||||
return redirect(url_for('/'))
|
||||
asn_descriptions = q.get_asn_descriptions(asn=session['asn'], all_descriptions=True)['response']
|
||||
sources = q.get_sources(date=session['date'])['response']
|
||||
asn_descriptions = bgpranking.get_asn_descriptions(asn=session['asn'], all_descriptions=True)['response']
|
||||
sources = bgpranking.get_sources(date=session['date'])['response']
|
||||
prefix = session.pop('prefix', None)
|
||||
ranks = q.asn_details(**session)['response']
|
||||
ranks = bgpranking.asn_details(**session)['response']
|
||||
if prefix:
|
||||
prefix_ips = q.get_prefix_ips(prefix=prefix, **session)['response']
|
||||
prefix_ips = bgpranking.get_prefix_ips(prefix=prefix, **session)['response']
|
||||
prefix_ips = [(ip, sorted(sources)) for ip, sources in prefix_ips.items()]
|
||||
prefix_ips.sort(key=lambda entry: len(entry[1]), reverse=True)
|
||||
else:
|
||||
|
|
@ -111,20 +109,20 @@ def asn_details():
|
|||
@app.route('/country', methods=['GET', 'POST'])
|
||||
def country():
|
||||
load_session()
|
||||
q = Querying()
|
||||
sources = q.get_sources(date=session['date'])['response']
|
||||
sources = bgpranking.get_sources(date=session['date'])['response']
|
||||
return render_template('country.html', sources=sources, countries=get_country_codes(), **session)
|
||||
|
||||
|
||||
@app.route('/country_history_callback', methods=['GET', 'POST'])
|
||||
def country_history_callback():
|
||||
history_data = request.get_json(force=True)
|
||||
history_data: Dict[str, Tuple[str, str, List[Any]]]
|
||||
history_data = request.get_json(force=True) # type: ignore
|
||||
to_display = []
|
||||
mapping = defaultdict(dict)
|
||||
mapping: Dict[str, Any] = defaultdict(dict)
|
||||
dates = []
|
||||
all_asns = set([])
|
||||
for country, data in history_data.items():
|
||||
for d, r_sum, details in data:
|
||||
for country, foo in history_data.items():
|
||||
for d, r_sum, details in foo:
|
||||
dates.append(d)
|
||||
for detail in details:
|
||||
asn, r = detail
|
||||
|
|
@ -146,7 +144,7 @@ def country_history_callback():
|
|||
|
||||
@app.route('/ipasn', methods=['GET', 'POST'])
|
||||
def ipasn():
|
||||
d = None
|
||||
d: Optional[Dict] = None
|
||||
if request.method == 'POST':
|
||||
d = request.form
|
||||
elif request.method == 'GET':
|
||||
|
|
@ -160,12 +158,11 @@ def ipasn():
|
|||
else:
|
||||
ip = d['ip']
|
||||
ipasn = get_ipasn()
|
||||
q = Querying()
|
||||
response = ipasn.query(first=(date.today() - timedelta(days=60)).isoformat(),
|
||||
aggregate=True, ip=ip)
|
||||
for r in response['response']:
|
||||
r['asn_descriptions'] = []
|
||||
asn_descriptions = q.get_asn_descriptions(asn=r['asn'], all_descriptions=True)['response']
|
||||
asn_descriptions = bgpranking.get_asn_descriptions(asn=r['asn'], all_descriptions=True)['response']
|
||||
for timestamp in sorted(asn_descriptions.keys()):
|
||||
if r['first_seen'] <= timestamp <= r['last_seen']:
|
||||
r['asn_descriptions'].append(asn_descriptions[timestamp])
|
||||
|
|
@ -185,16 +182,11 @@ def ipasn():
|
|||
@app.route('/ipasn_history/', defaults={'path': ''}, methods=['GET', 'POST'])
|
||||
@app.route('/ipasn_history/<path:path>', methods=['GET', 'POST'])
|
||||
def ipasn_history_proxy(path):
|
||||
config, general_config_file = load_general_config()
|
||||
if 'ipasnhistory_url' not in config:
|
||||
raise MissingConfigEntry(f'"ipasnhistory_url" is missing in {general_config_file}.')
|
||||
|
||||
path_for_ipasnhistory = request.full_path.replace('/ipasn_history', '')
|
||||
if '/?' in path_for_ipasnhistory:
|
||||
path_for_ipasnhistory = path_for_ipasnhistory.replace('/?', '/ip?')
|
||||
print(path_for_ipasnhistory)
|
||||
proxied_url = urljoin(config['ipasnhistory_url'], path_for_ipasnhistory)
|
||||
print(proxied_url)
|
||||
proxied_url = urljoin(get_config('generic', 'ipasnhistory_url'), path_for_ipasnhistory)
|
||||
if request.method in ['GET', 'HEAD']:
|
||||
to_return = requests.get(proxied_url).json()
|
||||
elif request.method == 'POST':
|
||||
|
|
@ -206,17 +198,17 @@ def ipasn_history_proxy(path):
|
|||
def json_asn():
|
||||
# TODO
|
||||
# * Filter on date => if only returning one descr, return the desription at that date
|
||||
query = request.get_json(force=True)
|
||||
to_return = {'meta': query, 'response': {}}
|
||||
query: Dict[str, Any] = request.get_json(force=True) # type: ignore
|
||||
to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
|
||||
if 'asn' not in query:
|
||||
to_return['error'] = f'You need to pass an asn - {query}'
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
q = Querying()
|
||||
asn_description_query = {'asn': query['asn']}
|
||||
if 'all_descriptions' in query:
|
||||
asn_description_query['all_descriptions'] = query['all_descriptions']
|
||||
to_return['response']['asn_description'] = q.get_asn_descriptions(**asn_description_query)['response']
|
||||
responses = bgpranking.get_asn_descriptions(**asn_description_query)['response']
|
||||
to_return['response']['asn_description'] = responses # type: ignore
|
||||
|
||||
asn_rank_query = {'asn': query['asn']}
|
||||
if 'date' in query:
|
||||
|
|
@ -228,60 +220,65 @@ def json_asn():
|
|||
if 'ipversion' in query:
|
||||
asn_rank_query['ipversion'] = query['ipversion']
|
||||
|
||||
to_return['response']['ranking'] = q.asn_rank(**asn_rank_query)['response']
|
||||
to_return['response']['ranking'] = bgpranking.asn_rank(**asn_rank_query)['response'] # type: ignore
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
|
||||
@app.route('/json/asn_descriptions', methods=['POST'])
|
||||
def asn_description():
|
||||
query = request.get_json(force=True)
|
||||
to_return = {'meta': query, 'response': {}}
|
||||
query: Dict = request.get_json(force=True) # type: ignore
|
||||
to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
|
||||
if 'asn' not in query:
|
||||
to_return['error'] = f'You need to pass an asn - {query}'
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
q = Querying()
|
||||
to_return['response']['asn_descriptions'] = q.get_asn_descriptions(**query)['response']
|
||||
to_return['response']['asn_descriptions'] = bgpranking.get_asn_descriptions(**query)['response'] # type: ignore
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
|
||||
@app.route('/json/asn_history', methods=['GET', 'POST'])
|
||||
def asn_history():
|
||||
q = Querying()
|
||||
if request.method == 'GET':
|
||||
load_session()
|
||||
if 'asn' in session:
|
||||
return Response(json.dumps(q.get_asn_history(**session)), mimetype='application/json')
|
||||
return Response(json.dumps(bgpranking.get_asn_history(**session)), mimetype='application/json')
|
||||
|
||||
query = request.get_json(force=True)
|
||||
to_return = {'meta': query, 'response': {}}
|
||||
query: Dict = request.get_json(force=True) # type: ignore
|
||||
to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
|
||||
if 'asn' not in query:
|
||||
to_return['error'] = f'You need to pass an asn - {query}'
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
to_return['response']['asn_history'] = q.get_asn_history(**query)['response']
|
||||
to_return['response']['asn_history'] = bgpranking.get_asn_history(**query)['response'] # type: ignore
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
|
||||
@app.route('/json/country_history', methods=['GET', 'POST'])
|
||||
def country_history():
|
||||
q = Querying()
|
||||
if request.method == 'GET':
|
||||
load_session()
|
||||
return Response(json.dumps(q.country_history(**session)), mimetype='application/json')
|
||||
return Response(json.dumps(bgpranking.country_history(**session)), mimetype='application/json')
|
||||
|
||||
query = request.get_json(force=True)
|
||||
to_return = {'meta': query, 'response': {}}
|
||||
to_return['response']['country_history'] = q.country_history(**query)['response']
|
||||
query: Dict = request.get_json(force=True) # type: ignore
|
||||
to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
|
||||
to_return['response']['country_history'] = bgpranking.country_history(**query)['response'] # type: ignore
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
|
||||
@app.route('/json/asns_global_ranking', methods=['POST'])
|
||||
def json_asns_global_ranking():
|
||||
query = request.get_json(force=True)
|
||||
to_return = {'meta': query, 'response': {}}
|
||||
q = Querying()
|
||||
to_return['response'] = q.asns_global_ranking(**query)['response']
|
||||
query: Dict = request.get_json(force=True) # type: ignore
|
||||
to_return: Dict[str, Union[str, Dict[str, Any]]] = {'meta': query, 'response': {}}
|
||||
to_return['response'] = bgpranking.asns_global_ranking(**query)['response']
|
||||
return Response(json.dumps(to_return), mimetype='application/json')
|
||||
|
||||
# ############# Json outputs #############
|
||||
|
||||
|
||||
# Query API
|
||||
|
||||
api = Api(app, title='BGP Ranking API',
|
||||
description='API to query BGP Ranking.',
|
||||
version=pkg_resources.get_distribution('bgpranking').version)
|
||||
|
||||
api.add_namespace(generic_api)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import pkg_resources
|
||||
|
||||
from flask import Flask
|
||||
from flask_restx import Api, Resource # type: ignore
|
||||
|
||||
from bgpranking.bgpranking import BGPRanking
|
||||
|
||||
from .helpers import get_secret_key
|
||||
from .proxied import ReverseProxied
|
||||
|
||||
app: Flask = Flask(__name__)
|
||||
|
||||
app.wsgi_app = ReverseProxied(app.wsgi_app) # type: ignore
|
||||
|
||||
app.config['SECRET_KEY'] = get_secret_key()
|
||||
|
||||
api = Api(app, title='BGP Ranking API',
|
||||
description='API to query BGP Ranking.',
|
||||
version=pkg_resources.get_distribution('bgpranking').version)
|
||||
|
||||
bgpranking: BGPRanking = BGPRanking()
|
||||
|
||||
|
||||
@api.route('/redis_up')
|
||||
@api.doc(description='Check if redis is up and running')
|
||||
class RedisUp(Resource):
|
||||
|
||||
def get(self):
|
||||
return bgpranking.check_redis_up()
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from bgpranking.default import get_homedir
|
||||
|
||||
|
||||
def src_request_ip(request) -> str:
|
||||
# NOTE: X-Real-IP is the IP passed by the reverse proxy in the headers.
|
||||
real_ip = request.headers.get('X-Real-IP')
|
||||
if not real_ip:
|
||||
real_ip = request.remote_addr
|
||||
return real_ip
|
||||
|
||||
|
||||
@lru_cache(64)
|
||||
def get_secret_key() -> bytes:
|
||||
secret_file_path: Path = get_homedir() / 'secret_key'
|
||||
if not secret_file_path.exists() or secret_file_path.stat().st_size < 64:
|
||||
if not secret_file_path.exists() or secret_file_path.stat().st_size < 64:
|
||||
with secret_file_path.open('wb') as f:
|
||||
f.write(os.urandom(64))
|
||||
with secret_file_path.open('rb') as f:
|
||||
return f.read()
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from typing import Any, MutableMapping
|
||||
|
||||
|
||||
class ReverseProxied():
|
||||
def __init__(self, app: Any) -> None:
|
||||
self.app = app
|
||||
|
||||
def __call__(self, environ: MutableMapping[str, Any], start_response: Any) -> Any:
|
||||
scheme = environ.get('HTTP_X_FORWARDED_PROTO')
|
||||
if not scheme:
|
||||
scheme = environ.get('HTTP_X_SCHEME')
|
||||
|
||||
if scheme:
|
||||
environ['wsgi.url_scheme'] = scheme
|
||||
return self.app(environ, start_response)
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
"use strict";
|
||||
|
||||
function linegraph(call_path) {
|
||||
var svg = d3.select("svg"),
|
||||
margin = {top: 20, right: 80, bottom: 30, left: 50},
|
||||
|
|
@ -16,12 +18,11 @@ function linegraph(call_path) {
|
|||
.x(function(d) { return x(d.date); })
|
||||
.y(function(d) { return y(d.rank); });
|
||||
|
||||
d3.json(call_path, {credentials: 'same-origin'}).then(function(data) {
|
||||
|
||||
var country_ranks = d3.entries(data.response).map(function(country_rank) {
|
||||
d3.json(call_path, {credentials: 'same-origin'}).then(data => {
|
||||
var country_ranks = $.map(data.response, function(value, key) {
|
||||
return {
|
||||
country: country_rank.key,
|
||||
values: d3.values(country_rank.value).map(function(d) {
|
||||
country: key,
|
||||
values: $.map(value, function(d) {
|
||||
return {date: parseTime(d[0]), rank: d[1]};
|
||||
})
|
||||
};
|
||||
|
|
@ -72,8 +73,9 @@ function linegraph(call_path) {
|
|||
{credentials: 'same-origin',
|
||||
method: 'POST',
|
||||
body: JSON.stringify(data.response),
|
||||
}).then(function(data) {
|
||||
})
|
||||
.then(function(data) {
|
||||
d3.select('#asn_details').html(data);
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,21 +1,28 @@
|
|||
{% extends "bootstrap/base.html" %}
|
||||
|
||||
{% block scripts %}
|
||||
{{ super() }}
|
||||
<script src='{{ url_for('static', filename='d3.v5.js') }}'></script>
|
||||
<script src='{{ url_for('static', filename='bootstrap-select.min.js') }}'></script>
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$('[data-toggle="popover"]').popover();
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
{% block head %}
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='bootstrap-select.min.css') }}">
|
||||
{{ super() }}
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
{{ super() }}
|
||||
{% endblock %}
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
{% block head %}
|
||||
<!-- Required meta tags -->
|
||||
<meta charset="utf-8">
|
||||
{% block styles %}
|
||||
{{ bootstrap.load_css() }}
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='bootstrap-select.min.css') }}">
|
||||
{% endblock %}
|
||||
{% endblock %}
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
{% block content %}{% endblock%}
|
||||
</div>
|
||||
{% block scripts %}
|
||||
{{ bootstrap.load_js() }}
|
||||
<script src='{{ url_for('static', filename='bootstrap-select.min.js') }}'></script>
|
||||
<script src='{{ url_for('static', filename='d3.v7.min.js') }}'></script>
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$('[data-toggle="popover"]').popover();
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
Loading…
Reference in New Issue