mirror of https://github.com/CIRCL/url-abuse
chg: Move installation process to pipenv
Raphaël Vinot
parent
cb5f6e23b7
commit
aa0b8da23f
|
|
@ -21,3 +21,6 @@ dist
|
|||
*egg-info
|
||||
|
||||
*.rdb
|
||||
|
||||
.env
|
||||
website/secret_key
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
[[source]]
|
||||
name = "pypi"
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[packages]
|
||||
redis = ">=3"
|
||||
pypssl = "*"
|
||||
pypdns = "*"
|
||||
pyeupi = "*"
|
||||
dnspython = "*"
|
||||
beautifulsoup4 = "*"
|
||||
pyipasnhistory = {git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"}
|
||||
pybgpranking = {git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"}
|
||||
flask = "*"
|
||||
flask-bootstrap = "*"
|
||||
flask-mail = "*"
|
||||
flask-wtf = "*"
|
||||
gunicorn = {extras = ["gevent"],version = "*"}
|
||||
pyurlabuse = {editable = true,path = "."}
|
||||
pyfaup = {git = "https://github.com/stricaud/faup.git/",subdirectory = "src/lib/bindings/python/"}
|
||||
|
||||
[requires]
|
||||
python_version = "3.6"
|
||||
|
|
@ -0,0 +1,330 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "4e0f863e88d1c7416abfa49ec8f74d6c51c2d605ca4a504ab7c87af0954c3d10"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.6"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
"url": "https://pypi.org/simple",
|
||||
"verify_ssl": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858",
|
||||
"sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348",
|
||||
"sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.7.1"
|
||||
},
|
||||
"blinker": {
|
||||
"hashes": [
|
||||
"sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6"
|
||||
],
|
||||
"version": "==1.4"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7",
|
||||
"sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033"
|
||||
],
|
||||
"version": "==2018.11.29"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||
],
|
||||
"version": "==3.0.4"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
|
||||
"sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
|
||||
],
|
||||
"version": "==7.0"
|
||||
},
|
||||
"dnspython": {
|
||||
"hashes": [
|
||||
"sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
|
||||
"sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.16.0"
|
||||
},
|
||||
"dominate": {
|
||||
"hashes": [
|
||||
"sha256:4076735c0745fe771e57b2313dbb4bfeec42731816ee23cee509f66e8912aa51",
|
||||
"sha256:4b9fd42d2824b79761799590697db45bf93daad511b130c50513af38da33df9b"
|
||||
],
|
||||
"version": "==2.3.5"
|
||||
},
|
||||
"flask": {
|
||||
"hashes": [
|
||||
"sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48",
|
||||
"sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.0.2"
|
||||
},
|
||||
"flask-bootstrap": {
|
||||
"hashes": [
|
||||
"sha256:cb08ed940183f6343a64e465e83b3a3f13c53e1baabb8d72b5da4545ef123ac8"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.3.7.1"
|
||||
},
|
||||
"flask-mail": {
|
||||
"hashes": [
|
||||
"sha256:22e5eb9a940bf407bcf30410ecc3708f3c56cc44b29c34e1726fe85006935f41"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.9.1"
|
||||
},
|
||||
"flask-wtf": {
|
||||
"hashes": [
|
||||
"sha256:5d14d55cfd35f613d99ee7cba0fc3fbbe63ba02f544d349158c14ca15561cc36",
|
||||
"sha256:d9a9e366b32dcbb98ef17228e76be15702cd2600675668bca23f63a7947fd5ac"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.14.2"
|
||||
},
|
||||
"gevent": {
|
||||
"hashes": [
|
||||
"sha256:0774babec518a24d9a7231d4e689931f31b332c4517a771e532002614e270a64",
|
||||
"sha256:0e1e5b73a445fe82d40907322e1e0eec6a6745ca3cea19291c6f9f50117bb7ea",
|
||||
"sha256:0ff2b70e8e338cf13bedf146b8c29d475e2a544b5d1fe14045aee827c073842c",
|
||||
"sha256:107f4232db2172f7e8429ed7779c10f2ed16616d75ffbe77e0e0c3fcdeb51a51",
|
||||
"sha256:14b4d06d19d39a440e72253f77067d27209c67e7611e352f79fe69e0f618f76e",
|
||||
"sha256:1b7d3a285978b27b469c0ff5fb5a72bcd69f4306dbbf22d7997d83209a8ba917",
|
||||
"sha256:1eb7fa3b9bd9174dfe9c3b59b7a09b768ecd496debfc4976a9530a3e15c990d1",
|
||||
"sha256:2711e69788ddb34c059a30186e05c55a6b611cb9e34ac343e69cf3264d42fe1c",
|
||||
"sha256:28a0c5417b464562ab9842dd1fb0cc1524e60494641d973206ec24d6ec5f6909",
|
||||
"sha256:3249011d13d0c63bea72d91cec23a9cf18c25f91d1f115121e5c9113d753fa12",
|
||||
"sha256:44089ed06a962a3a70e96353c981d628b2d4a2f2a75ea5d90f916a62d22af2e8",
|
||||
"sha256:4bfa291e3c931ff3c99a349d8857605dca029de61d74c6bb82bd46373959c942",
|
||||
"sha256:50024a1ee2cf04645535c5ebaeaa0a60c5ef32e262da981f4be0546b26791950",
|
||||
"sha256:53b72385857e04e7faca13c613c07cab411480822ac658d97fd8a4ddbaf715c8",
|
||||
"sha256:74b7528f901f39c39cdbb50cdf08f1a2351725d9aebaef212a29abfbb06895ee",
|
||||
"sha256:7d0809e2991c9784eceeadef01c27ee6a33ca09ebba6154317a257353e3af922",
|
||||
"sha256:896b2b80931d6b13b5d9feba3d4eebc67d5e6ec54f0cf3339d08487d55d93b0e",
|
||||
"sha256:8d9ec51cc06580f8c21b41fd3f2b3465197ba5b23c00eb7d422b7ae0380510b0",
|
||||
"sha256:9f7a1e96fec45f70ad364e46de32ccacab4d80de238bd3c2edd036867ccd48ad",
|
||||
"sha256:ab4dc33ef0e26dc627559786a4fba0c2227f125db85d970abbf85b77506b3f51",
|
||||
"sha256:d1e6d1f156e999edab069d79d890859806b555ce4e4da5b6418616322f0a3df1",
|
||||
"sha256:d752bcf1b98174780e2317ada12013d612f05116456133a6acf3e17d43b71f05",
|
||||
"sha256:e5bcc4270671936349249d26140c267397b7b4b1381f5ec8b13c53c5b53ab6e1"
|
||||
],
|
||||
"version": "==1.4.0"
|
||||
},
|
||||
"greenlet": {
|
||||
"hashes": [
|
||||
"sha256:000546ad01e6389e98626c1367be58efa613fa82a1be98b0c6fc24b563acc6d0",
|
||||
"sha256:0d48200bc50cbf498716712129eef819b1729339e34c3ae71656964dac907c28",
|
||||
"sha256:23d12eacffa9d0f290c0fe0c4e81ba6d5f3a5b7ac3c30a5eaf0126bf4deda5c8",
|
||||
"sha256:37c9ba82bd82eb6a23c2e5acc03055c0e45697253b2393c9a50cef76a3985304",
|
||||
"sha256:51503524dd6f152ab4ad1fbd168fc6c30b5795e8c70be4410a64940b3abb55c0",
|
||||
"sha256:8041e2de00e745c0e05a502d6e6db310db7faa7c979b3a5877123548a4c0b214",
|
||||
"sha256:81fcd96a275209ef117e9ec91f75c731fa18dcfd9ffaa1c0adbdaa3616a86043",
|
||||
"sha256:853da4f9563d982e4121fed8c92eea1a4594a2299037b3034c3c898cb8e933d6",
|
||||
"sha256:8b4572c334593d449113f9dc8d19b93b7b271bdbe90ba7509eb178923327b625",
|
||||
"sha256:9416443e219356e3c31f1f918a91badf2e37acf297e2fa13d24d1cc2380f8fbc",
|
||||
"sha256:9854f612e1b59ec66804931df5add3b2d5ef0067748ea29dc60f0efdcda9a638",
|
||||
"sha256:99a26afdb82ea83a265137a398f570402aa1f2b5dfb4ac3300c026931817b163",
|
||||
"sha256:a19bf883b3384957e4a4a13e6bd1ae3d85ae87f4beb5957e35b0be287f12f4e4",
|
||||
"sha256:a9f145660588187ff835c55a7d2ddf6abfc570c2651c276d3d4be8a2766db490",
|
||||
"sha256:ac57fcdcfb0b73bb3203b58a14501abb7e5ff9ea5e2edfa06bb03035f0cff248",
|
||||
"sha256:bcb530089ff24f6458a81ac3fa699e8c00194208a724b644ecc68422e1111939",
|
||||
"sha256:beeabe25c3b704f7d56b573f7d2ff88fc99f0138e43480cecdfcaa3b87fe4f87",
|
||||
"sha256:d634a7ea1fc3380ff96f9e44d8d22f38418c1c381d5fac680b272d7d90883720",
|
||||
"sha256:d97b0661e1aead761f0ded3b769044bb00ed5d33e1ec865e891a8b128bf7c656"
|
||||
],
|
||||
"markers": "platform_python_implementation == 'CPython'",
|
||||
"version": "==0.4.15"
|
||||
},
|
||||
"gunicorn": {
|
||||
"extras": [
|
||||
"gevent"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471",
|
||||
"sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==19.9.0"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
|
||||
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
|
||||
],
|
||||
"version": "==2.8"
|
||||
},
|
||||
"itsdangerous": {
|
||||
"hashes": [
|
||||
"sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
|
||||
"sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"
|
||||
],
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"jinja2": {
|
||||
"hashes": [
|
||||
"sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd",
|
||||
"sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"
|
||||
],
|
||||
"version": "==2.10"
|
||||
},
|
||||
"markupsafe": {
|
||||
"hashes": [
|
||||
"sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432",
|
||||
"sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b",
|
||||
"sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9",
|
||||
"sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af",
|
||||
"sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834",
|
||||
"sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd",
|
||||
"sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d",
|
||||
"sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7",
|
||||
"sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b",
|
||||
"sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3",
|
||||
"sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c",
|
||||
"sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2",
|
||||
"sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7",
|
||||
"sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36",
|
||||
"sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1",
|
||||
"sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e",
|
||||
"sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1",
|
||||
"sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c",
|
||||
"sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856",
|
||||
"sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550",
|
||||
"sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492",
|
||||
"sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672",
|
||||
"sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401",
|
||||
"sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6",
|
||||
"sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6",
|
||||
"sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c",
|
||||
"sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd",
|
||||
"sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1"
|
||||
],
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"pybgpranking": {
|
||||
"git": "https://github.com/D4-project/BGP-Ranking.git/",
|
||||
"ref": "5e4264a411b59f6a612c4bfdde307b1d8a61ead8",
|
||||
"subdirectory": "client"
|
||||
},
|
||||
"pyeupi": {
|
||||
"hashes": [
|
||||
"sha256:35b0e6b430f23ecd303f7cc7a8fe5147cf2509a5b2254eaf9695392c0af02901"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.0"
|
||||
},
|
||||
"pyfaup": {
|
||||
"git": "https://github.com/stricaud/faup.git/",
|
||||
"ref": "de31b6965fc4149c2095c7b721f456e428404736",
|
||||
"subdirectory": "src/lib/bindings/python/"
|
||||
},
|
||||
"pyipasnhistory": {
|
||||
"git": "https://github.com/D4-project/IPASN-History.git/",
|
||||
"ref": "54857344c412a903df2abe67a8855f2fcaeef4a8",
|
||||
"subdirectory": "client"
|
||||
},
|
||||
"pypdns": {
|
||||
"hashes": [
|
||||
"sha256:0356360156dd26d2cf27a415a10ff2bd1ff1d2eb3b2dd51b35553d60b87fd328"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.3"
|
||||
},
|
||||
"pypssl": {
|
||||
"hashes": [
|
||||
"sha256:4dbe772aefdf4ab18934d83cde79e2fc5d5ba9d2b4153dc419a63faab3432643"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.1"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93",
|
||||
"sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02"
|
||||
],
|
||||
"version": "==2.7.5"
|
||||
},
|
||||
"pyurlabuse": {
|
||||
"editable": true,
|
||||
"path": "."
|
||||
},
|
||||
"redis": {
|
||||
"hashes": [
|
||||
"sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf",
|
||||
"sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.0.1"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
|
||||
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
|
||||
],
|
||||
"version": "==2.21.0"
|
||||
},
|
||||
"requests-cache": {
|
||||
"hashes": [
|
||||
"sha256:e9270030becc739b0a7f7f834234c73a878b2d794122bf76f40055a22419eb67",
|
||||
"sha256:fe561ca119879bbcfb51f03a35e35b425e18f338248e59fd5cf2166c77f457a2"
|
||||
],
|
||||
"version": "==0.4.13"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||
],
|
||||
"version": "==1.12.0"
|
||||
},
|
||||
"soupsieve": {
|
||||
"hashes": [
|
||||
"sha256:009d8865916766f7f452880d08ff94ed4c5445011a3deaac67543b82bdb0b9ee",
|
||||
"sha256:97599c45a1ddfe9ab0a0cba889b7f214b3e310b703f176a0610c0b54e207cc04"
|
||||
],
|
||||
"version": "==1.7.1"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
|
||||
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
|
||||
],
|
||||
"version": "==1.24.1"
|
||||
},
|
||||
"visitor": {
|
||||
"hashes": [
|
||||
"sha256:2c737903b2b6864ebc6167eef7cf3b997126f1aa94bdf590f90f1436d23e480a"
|
||||
],
|
||||
"version": "==0.1.3"
|
||||
},
|
||||
"werkzeug": {
|
||||
"hashes": [
|
||||
"sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c",
|
||||
"sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"
|
||||
],
|
||||
"version": "==0.14.1"
|
||||
},
|
||||
"wtforms": {
|
||||
"hashes": [
|
||||
"sha256:0cdbac3e7f6878086c334aa25dc5a33869a3954e9d1e015130d65a69309b3b61",
|
||||
"sha256:e3ee092c827582c50877cdbd49e9ce6d2c5c1f6561f849b3b068c1b8029626f1"
|
||||
],
|
||||
"version": "==2.2.1"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
64
README.md
64
README.md
|
|
@ -28,37 +28,47 @@ If you don't want to use the online version or run your own version of URL Abuse
|
|||
|
||||
## Install
|
||||
|
||||
Install the requirements
|
||||
**IMPORTANT**: Use [pipenv](https://pipenv.readthedocs.io/en/latest/)
|
||||
|
||||
~~~
|
||||
pip install -r requirements.txt
|
||||
~~~
|
||||
**NOTE**: Yes, it requires python3.6+. No, it will never support anything older.
|
||||
|
||||
Copy and review the configuration:
|
||||
## Install redis
|
||||
|
||||
~~~
|
||||
```bash
|
||||
git clone https://github.com/antirez/redis.git
|
||||
cd redis
|
||||
git checkout 5.0
|
||||
make
|
||||
make test
|
||||
cd ..
|
||||
```
|
||||
|
||||
# Install Faup
|
||||
|
||||
```bash
|
||||
git clone git://github.com/stricaud/faup.git
|
||||
cd faup
|
||||
mkdir build
|
||||
cd build
|
||||
cmake .. && make
|
||||
sudo make install
|
||||
```
|
||||
|
||||
## Install & run URL Abuse
|
||||
|
||||
```bash
|
||||
git clone https://github.com/CIRCL/url-abuse.git
|
||||
cd url-abuse
|
||||
pipenv install
|
||||
echo URLABUSE_HOME="'`pwd`'" > .env
|
||||
pipenv shell
|
||||
# Copy and review the configuration:
|
||||
cp config.ini.sample config.ini
|
||||
~~~
|
||||
|
||||
Install Redis and update the configuration.
|
||||
|
||||
Start the Redis back-end
|
||||
|
||||
~~~
|
||||
./run_redis.sh
|
||||
~~~
|
||||
|
||||
Start the workers (at least 10)
|
||||
|
||||
~~~
|
||||
seq 10 | parallel -u -j 10 ./worker.py
|
||||
~~~
|
||||
|
||||
Start the web interface
|
||||
|
||||
~~~
|
||||
python runapp.py
|
||||
~~~
|
||||
# Starts all the backend
|
||||
start.py
|
||||
# Start the web interface
|
||||
start_website.py
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ def worker(process_id: int):
|
|||
urlabuse_query = Query()
|
||||
queue = Redis(unix_socket_path=get_socket_path('cache'), db=0,
|
||||
decode_responses=True)
|
||||
print(f'Start Worker {process_id}')
|
||||
while True:
|
||||
jobid = queue.spop('to_process')
|
||||
if not jobid:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from subprocess import Popen
|
||||
from urlabuse.helpers import get_homedir
|
||||
|
||||
if __name__ == '__main__':
|
||||
website_dir = get_homedir() / 'website'
|
||||
Popen([f'{website_dir}/3drparty.sh'], cwd=website_dir)
|
||||
try:
|
||||
Popen(['gunicorn', '--worker-class', 'gevent', '-w', '10', '-b', '0.0.0.0:5100', 'web:app'],
|
||||
cwd=website_dir).communicate()
|
||||
except KeyboardInterrupt:
|
||||
print('Stopping gunicorn.')
|
||||
|
|
@ -1,9 +1,18 @@
|
|||
redis>=3
|
||||
pypssl
|
||||
pypdns
|
||||
pyeupi
|
||||
dnspython
|
||||
beautifulsoup4
|
||||
|
||||
git+https://github.com/D4-project/IPASN-History.git/#egg=pyipasnhistory&subdirectory=client
|
||||
git+https://github.com/D4-project/BGP-Ranking.git/#egg=pybgpranking&subdirectory=client
|
||||
-i https://pypi.org/simple
|
||||
beautifulsoup4==4.7.1
|
||||
certifi==2018.11.29
|
||||
chardet==3.0.4
|
||||
dnspython==1.16.0
|
||||
git+https://github.com/D4-project/BGP-Ranking.git/@5e4264a411b59f6a612c4bfdde307b1d8a61ead8#egg=pybgpranking&subdirectory=client
|
||||
git+https://github.com/D4-project/IPASN-History.git/@54857344c412a903df2abe67a8855f2fcaeef4a8#egg=pyipasnhistory&subdirectory=client
|
||||
idna==2.8
|
||||
pyeupi==1.0
|
||||
pypdns==1.3
|
||||
pypssl==2.1
|
||||
python-dateutil==2.7.5
|
||||
redis==3.0.1
|
||||
requests-cache==0.4.13
|
||||
requests==2.21.0
|
||||
six==1.12.0
|
||||
soupsieve==1.7.1
|
||||
urllib3==1.24.1
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -12,7 +12,7 @@ setup(
|
|||
url='https://github.com/CIRCL/url-abuse/',
|
||||
description='URL Abuse interface',
|
||||
packages=['urlabuse'],
|
||||
scripts=['bin/run_backend.py', 'bin/run_workers.py', 'bin/start.py'],
|
||||
scripts=['bin/run_backend.py', 'bin/run_workers.py', 'bin/start.py', 'bin/start_website.py'],
|
||||
classifiers=[
|
||||
'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
|
||||
'Development Status :: 3 - Alpha',
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
flask
|
||||
flask-bootstrap
|
||||
flask-mail
|
||||
flask-wtf
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*-coding:utf-8 -*
|
||||
|
||||
import os
|
||||
|
||||
from web import create_app
|
||||
|
||||
# create an app instance
|
||||
abspath = os.path.abspath(__file__)
|
||||
dname = os.path.dirname(abspath)
|
||||
os.chdir(dname)
|
||||
app = create_app()
|
||||
app.run(host='0.0.0.0', port = 5100, debug=False, threaded=True)
|
||||
|
|
@ -17,7 +17,7 @@ from logging import Formatter
|
|||
|
||||
from redis import Redis
|
||||
|
||||
from urlabuse.helpers import get_socket_path
|
||||
from urlabuse.helpers import get_socket_path, get_homedir
|
||||
from urlabuse.urlabuse import Query
|
||||
|
||||
import configparser
|
||||
|
|
@ -62,268 +62,298 @@ def prepare_auth():
|
|||
return to_return
|
||||
|
||||
|
||||
def create_app(configfile=None):
|
||||
app = Flask(__name__)
|
||||
handler = RotatingFileHandler('urlabuse.log', maxBytes=10000, backupCount=5)
|
||||
handler.setFormatter(Formatter('%(asctime)s %(message)s'))
|
||||
app.wsgi_app = ReverseProxied(app.wsgi_app)
|
||||
app.logger.addHandler(handler)
|
||||
app.logger.setLevel(logging.INFO)
|
||||
Bootstrap(app)
|
||||
queue = Redis(unix_socket_path=get_socket_path('cache'), db=0,
|
||||
decode_responses=True)
|
||||
urlabuse_query = Query()
|
||||
app = Flask(__name__)
|
||||
handler = RotatingFileHandler('urlabuse.log', maxBytes=10000, backupCount=5)
|
||||
handler.setFormatter(Formatter('%(asctime)s %(message)s'))
|
||||
app.wsgi_app = ReverseProxied(app.wsgi_app)
|
||||
app.logger.addHandler(handler)
|
||||
app.logger.setLevel(logging.INFO)
|
||||
Bootstrap(app)
|
||||
queue = Redis(unix_socket_path=get_socket_path('cache'), db=0,
|
||||
decode_responses=True)
|
||||
urlabuse_query = Query()
|
||||
|
||||
# Mail Config
|
||||
app.config['MAIL_SERVER'] = 'localhost'
|
||||
app.config['MAIL_PORT'] = 25
|
||||
mail = Mail(app)
|
||||
# Mail Config
|
||||
app.config['MAIL_SERVER'] = 'localhost'
|
||||
app.config['MAIL_PORT'] = 25
|
||||
mail = Mail(app)
|
||||
|
||||
app.config['SECRET_KEY'] = 'devkey'
|
||||
app.config['BOOTSTRAP_SERVE_LOCAL'] = True
|
||||
app.config['configfile'] = config_dir / 'config.ini'
|
||||
secret_file_path = get_homedir() / 'website' / 'secret_key'
|
||||
|
||||
parser = configparser.SafeConfigParser()
|
||||
parser.read(app.config['configfile'])
|
||||
if not secret_file_path.exists() or secret_file_path.stat().st_size < 64:
|
||||
with open(secret_file_path, 'wb') as f:
|
||||
f.write(os.urandom(64))
|
||||
|
||||
replacelist = make_dict(parser, 'replacelist')
|
||||
auth_users = prepare_auth()
|
||||
ignorelist = [i.strip()
|
||||
for i in parser.get('abuse', 'ignore').split('\n')
|
||||
if len(i.strip()) > 0]
|
||||
autosend_threshold = 5
|
||||
with open(secret_file_path, 'rb') as f:
|
||||
app.config['SECRET_KEY'] = f.read()
|
||||
|
||||
def _get_user_ip(request):
|
||||
ip = request.headers.get('X-Forwarded-For')
|
||||
if ip is None:
|
||||
ip = request.remote_addr
|
||||
return ip
|
||||
app.config['BOOTSTRAP_SERVE_LOCAL'] = True
|
||||
app.config['configfile'] = config_dir / 'config.ini'
|
||||
|
||||
@app.route('/', methods=['GET', 'POST'])
|
||||
def index():
|
||||
if request.method == 'HEAD':
|
||||
# Just returns ack if the webserver is running
|
||||
return 'Ack'
|
||||
form = URLForm()
|
||||
return render_template('index.html', form=form)
|
||||
parser = configparser.SafeConfigParser()
|
||||
parser.read(app.config['configfile'])
|
||||
|
||||
@app.route('/urlreport', methods=['GET'])
|
||||
def url_report():
|
||||
return render_template('url-report.html')
|
||||
replacelist = make_dict(parser, 'replacelist')
|
||||
auth_users = prepare_auth()
|
||||
ignorelist = [i.strip()
|
||||
for i in parser.get('abuse', 'ignore').split('\n')
|
||||
if len(i.strip()) > 0]
|
||||
autosend_threshold = 5
|
||||
|
||||
@app.errorhandler(404)
|
||||
def page_not_found(e):
|
||||
ip = request.headers.get('X-Forwarded-For')
|
||||
if ip is None:
|
||||
ip = request.remote_addr
|
||||
if request.path != '/_result/':
|
||||
app.logger.info('404 of {} on {}'.format(ip, request.path))
|
||||
return render_template('404.html'), 404
|
||||
|
||||
def authenticate():
|
||||
"""Sends a 401 response that enables basic auth"""
|
||||
return Response('Could not verify your access level for that URL.\n'
|
||||
'You have to login with proper credentials', 401,
|
||||
{'WWW-Authenticate': 'Basic realm="Login Required"'})
|
||||
def _get_user_ip(request):
|
||||
ip = request.headers.get('X-Forwarded-For')
|
||||
if ip is None:
|
||||
ip = request.remote_addr
|
||||
return ip
|
||||
|
||||
def check_auth(username, password):
|
||||
"""This function is called to check if a username /
|
||||
password combination is valid.
|
||||
"""
|
||||
if auth_users is None:
|
||||
return False
|
||||
|
||||
@app.route('/', methods=['GET', 'POST'])
|
||||
def index():
|
||||
if request.method == 'HEAD':
|
||||
# Just returns ack if the webserver is running
|
||||
return 'Ack'
|
||||
form = URLForm()
|
||||
return render_template('index.html', form=form)
|
||||
|
||||
|
||||
@app.route('/urlreport', methods=['GET'])
|
||||
def url_report():
|
||||
return render_template('url-report.html')
|
||||
|
||||
|
||||
@app.errorhandler(404)
|
||||
def page_not_found(e):
|
||||
ip = request.headers.get('X-Forwarded-For')
|
||||
if ip is None:
|
||||
ip = request.remote_addr
|
||||
if request.path != '/_result/':
|
||||
app.logger.info('404 of {} on {}'.format(ip, request.path))
|
||||
return render_template('404.html'), 404
|
||||
|
||||
|
||||
def authenticate():
|
||||
"""Sends a 401 response that enables basic auth"""
|
||||
return Response('Could not verify your access level for that URL.\n'
|
||||
'You have to login with proper credentials', 401,
|
||||
{'WWW-Authenticate': 'Basic realm="Login Required"'})
|
||||
|
||||
|
||||
def check_auth(username, password):
|
||||
"""This function is called to check if a username /
|
||||
password combination is valid.
|
||||
"""
|
||||
if auth_users is None:
|
||||
return False
|
||||
else:
|
||||
db_pass = auth_users.get(username)
|
||||
return db_pass == password
|
||||
|
||||
|
||||
@app.route('/login', methods=['GET', 'POST'])
|
||||
def login():
|
||||
auth = request.authorization
|
||||
if not auth or not check_auth(auth.username, auth.password):
|
||||
return authenticate()
|
||||
return redirect(url_for('index'))
|
||||
|
||||
|
||||
@app.route("/_result/<job_key>", methods=['GET'])
|
||||
def check_valid(job_key):
|
||||
if not job_key or not queue.exists(job_key):
|
||||
return Response(json.dumps(None), mimetype='application/json'), 200
|
||||
if not queue.hexists(job_key, 'result'):
|
||||
return Response(json.dumps('Nay!'), mimetype='application/json'), 202
|
||||
result = queue.hget(job_key, 'result')
|
||||
queue.delete(job_key)
|
||||
return Response(result, mimetype='application/json'), 200
|
||||
|
||||
|
||||
def enqueue(method, data):
|
||||
job_id = str(uuid.uuid4())
|
||||
p = queue.pipeline()
|
||||
p.hmset(job_id, {'method': method, 'data': json.dumps(data)})
|
||||
p.sadd('to_process', job_id)
|
||||
p.execute()
|
||||
return job_id
|
||||
|
||||
|
||||
@app.route('/start', methods=['POST'])
|
||||
def run_query():
|
||||
data = request.get_json(force=True)
|
||||
url = data["url"]
|
||||
ip = _get_user_ip(request)
|
||||
app.logger.info(f'{ip} {url}')
|
||||
if urlabuse_query.get_submissions(url) and urlabuse_query.get_submissions(url) >= autosend_threshold:
|
||||
send(url, '', True)
|
||||
return enqueue('is_valid_url', {'url': url})
|
||||
|
||||
|
||||
@app.route('/urls', methods=['POST'])
|
||||
def urls():
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('url_list', {'url': data["url"]})
|
||||
|
||||
|
||||
@app.route('/resolve', methods=['POST'])
|
||||
def resolve():
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('dns_resolve', {'url': data["url"]})
|
||||
|
||||
|
||||
def read_auth(name):
|
||||
key = config_dir / f'{name}.key'
|
||||
if not key.exists():
|
||||
return ''
|
||||
with open(key) as f:
|
||||
to_return = []
|
||||
for line in f.readlines():
|
||||
to_return.append(line.strip())
|
||||
return to_return
|
||||
|
||||
|
||||
@app.route('/phishtank', methods=['POST'])
|
||||
def phishtank():
|
||||
auth = read_auth('phishtank')
|
||||
if not auth:
|
||||
return ''
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('phish_query', {'url': parser.get("PHISHTANK", "url"),
|
||||
'key': auth[0], 'query': data["query"]})
|
||||
|
||||
|
||||
@app.route('/virustotal_report', methods=['POST'])
|
||||
def vt():
|
||||
auth = read_auth('virustotal')
|
||||
if not auth:
|
||||
return ''
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('vt_query_url', {'url': parser.get("VIRUSTOTAL", "url_report"),
|
||||
'url_up': parser.get("VIRUSTOTAL", "url_upload"),
|
||||
'key': auth[0], 'query': data["query"]})
|
||||
|
||||
|
||||
@app.route('/googlesafebrowsing', methods=['POST'])
|
||||
def gsb():
|
||||
auth = read_auth('googlesafebrowsing')
|
||||
if not auth:
|
||||
return ''
|
||||
key = auth[0]
|
||||
data = request.get_json(force=True)
|
||||
url = parser.get("GOOGLESAFEBROWSING", "url").format(key)
|
||||
return enqueue('gsb_query', {'url': url,
|
||||
'query': data["query"]})
|
||||
|
||||
|
||||
'''
|
||||
@app.route('/urlquery', methods=['POST'])
|
||||
def urlquery():
|
||||
auth = read_auth('urlquery')
|
||||
if not auth:
|
||||
return ''
|
||||
key = auth[0]
|
||||
data = json.loads(request.data.decode())
|
||||
url = parser.get("URLQUERY", "url")
|
||||
query = data["query"]
|
||||
u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500)
|
||||
return u.get_id()
|
||||
|
||||
@app.route('/ticket', methods=['POST'])
|
||||
def ticket():
|
||||
if not request.authorization:
|
||||
return ''
|
||||
data = json.loads(request.data.decode())
|
||||
server = parser.get("SPHINX", "server")
|
||||
port = int(parser.get("SPHINX", "port"))
|
||||
url = parser.get("ITS", "url")
|
||||
query = data["query"]
|
||||
u = q.enqueue_call(func=sphinxsearch, args=(server, port, url, query,),
|
||||
result_ttl=500)
|
||||
return u.get_id()
|
||||
'''
|
||||
|
||||
|
||||
@app.route('/whois', methods=['POST'])
|
||||
def whoismail():
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('whois', {'server': parser.get("WHOIS", "server"),
|
||||
'port': parser.getint("WHOIS", "port"),
|
||||
'domain': data["query"],
|
||||
'ignorelist': ignorelist, 'replacelist': replacelist})
|
||||
|
||||
|
||||
@app.route('/eupi', methods=['POST'])
|
||||
def eu():
|
||||
auth = read_auth('eupi')
|
||||
if not auth:
|
||||
return ''
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('eupi', {'url': parser.get("EUPI", "url"),
|
||||
'key': auth[0], 'q': data["query"]})
|
||||
|
||||
|
||||
@app.route('/pdnscircl', methods=['POST'])
|
||||
def dnscircl():
|
||||
auth = read_auth('pdnscircl')
|
||||
if not auth:
|
||||
return ''
|
||||
user, password = auth
|
||||
url = parser.get("PDNS_CIRCL", "url")
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('pdnscircl', {'url': url, 'user': user.strip(),
|
||||
'passwd': password.strip(), 'q': data["query"]})
|
||||
|
||||
|
||||
@app.route('/bgpranking', methods=['POST'])
|
||||
def bgpr():
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('bgpranking', {'ip': data["query"]})
|
||||
|
||||
|
||||
@app.route('/psslcircl', methods=['POST'])
|
||||
def sslcircl():
|
||||
auth = read_auth('psslcircl')
|
||||
if not auth:
|
||||
return ''
|
||||
user, password = auth
|
||||
url = parser.get("PSSL_CIRCL", "url")
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('psslcircl', {'url': url, 'user': user.strip(),
|
||||
'passwd': password.strip(), 'q': data["query"]})
|
||||
|
||||
|
||||
@app.route('/get_cache', methods=['POST'])
|
||||
def get_cache():
|
||||
data = request.get_json(force=True)
|
||||
url = data["query"]
|
||||
if 'digest' in data:
|
||||
digest = data["digest"]
|
||||
else:
|
||||
digest = False
|
||||
data = urlabuse_query.cached(url, digest)
|
||||
return Response(json.dumps(data), mimetype='application/json')
|
||||
|
||||
|
||||
def send(url, ip='', autosend=False):
|
||||
if not urlabuse_query.get_mail_sent(url):
|
||||
data = urlabuse_query.cached(url, digest=True)
|
||||
if not autosend:
|
||||
subject = 'URL Abuse report from ' + ip
|
||||
else:
|
||||
db_pass = auth_users.get(username)
|
||||
return db_pass == password
|
||||
subject = 'URL Abuse report sent automatically'
|
||||
msg = Message(subject, sender='urlabuse@circl.lu', recipients=["info@circl.lu"])
|
||||
msg.body = data['digest'][0]
|
||||
msg.body += '\n\n'
|
||||
msg.body += json.dumps(data['result'], sort_keys=True, indent=2)
|
||||
mail.send(msg)
|
||||
urlabuse_query.set_mail_sent(url)
|
||||
|
||||
@app.route('/login', methods=['GET', 'POST'])
|
||||
def login():
|
||||
auth = request.authorization
|
||||
if not auth or not check_auth(auth.username, auth.password):
|
||||
return authenticate()
|
||||
return redirect(url_for('index'))
|
||||
|
||||
@app.route("/_result/<job_key>", methods=['GET'])
|
||||
def check_valid(job_key):
|
||||
if not job_key or not queue.exists(job_key):
|
||||
return Response(json.dumps(None), mimetype='application/json'), 200
|
||||
if not queue.hexists(job_key, 'result'):
|
||||
return Response(json.dumps('Nay!'), mimetype='application/json'), 202
|
||||
result = queue.hget(job_key, 'result')
|
||||
queue.delete(job_key)
|
||||
return Response(result, mimetype='application/json'), 200
|
||||
|
||||
def enqueue(method, data):
|
||||
job_id = str(uuid.uuid4())
|
||||
p = queue.pipeline()
|
||||
p.hmset(job_id, {'method': method, 'data': json.dumps(data)})
|
||||
p.sadd('to_process', job_id)
|
||||
p.execute()
|
||||
return job_id
|
||||
|
||||
@app.route('/start', methods=['POST'])
|
||||
def run_query():
|
||||
data = request.get_json(force=True)
|
||||
url = data["url"]
|
||||
@app.route('/submit', methods=['POST'])
|
||||
def send_mail():
|
||||
data = request.get_json(force=True)
|
||||
url = data["url"]
|
||||
if not urlabuse_query.get_mail_sent(url):
|
||||
ip = _get_user_ip(request)
|
||||
app.logger.info(f'{ip} {url}')
|
||||
if urlabuse_query.get_submissions(url) and urlabuse_query.get_submissions(url) >= autosend_threshold:
|
||||
send(url, '', True)
|
||||
return enqueue('is_valid_url', {'url': url})
|
||||
|
||||
@app.route('/urls', methods=['POST'])
|
||||
def urls():
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('url_list', {'url': data["url"]})
|
||||
|
||||
@app.route('/resolve', methods=['POST'])
|
||||
def resolve():
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('dns_resolve', {'url': data["url"]})
|
||||
|
||||
def read_auth(name):
|
||||
key = config_dir / f'{name}.key'
|
||||
if not key.exists():
|
||||
return ''
|
||||
with open(key) as f:
|
||||
to_return = []
|
||||
for line in f.readlines():
|
||||
to_return.append(line.strip())
|
||||
return to_return
|
||||
|
||||
@app.route('/phishtank', methods=['POST'])
|
||||
def phishtank():
|
||||
auth = read_auth('phishtank')
|
||||
if not auth:
|
||||
return ''
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('phish_query', {'url': parser.get("PHISHTANK", "url"),
|
||||
'key': auth[0], 'query': data["query"]})
|
||||
|
||||
@app.route('/virustotal_report', methods=['POST'])
|
||||
def vt():
|
||||
auth = read_auth('virustotal')
|
||||
if not auth:
|
||||
return ''
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('vt_query_url', {'url': parser.get("VIRUSTOTAL", "url_report"),
|
||||
'url_up': parser.get("VIRUSTOTAL", "url_upload"),
|
||||
'key': auth[0], 'query': data["query"]})
|
||||
|
||||
@app.route('/googlesafebrowsing', methods=['POST'])
|
||||
def gsb():
|
||||
auth = read_auth('googlesafebrowsing')
|
||||
if not auth:
|
||||
return ''
|
||||
key = auth[0]
|
||||
data = request.get_json(force=True)
|
||||
url = parser.get("GOOGLESAFEBROWSING", "url").format(key)
|
||||
return enqueue('gsb_query', {'url': url,
|
||||
'query': data["query"]})
|
||||
|
||||
'''
|
||||
@app.route('/urlquery', methods=['POST'])
|
||||
def urlquery():
|
||||
auth = read_auth('urlquery')
|
||||
if not auth:
|
||||
return ''
|
||||
key = auth[0]
|
||||
data = json.loads(request.data.decode())
|
||||
url = parser.get("URLQUERY", "url")
|
||||
query = data["query"]
|
||||
u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500)
|
||||
return u.get_id()
|
||||
|
||||
@app.route('/ticket', methods=['POST'])
|
||||
def ticket():
|
||||
if not request.authorization:
|
||||
return ''
|
||||
data = json.loads(request.data.decode())
|
||||
server = parser.get("SPHINX", "server")
|
||||
port = int(parser.get("SPHINX", "port"))
|
||||
url = parser.get("ITS", "url")
|
||||
query = data["query"]
|
||||
u = q.enqueue_call(func=sphinxsearch, args=(server, port, url, query,),
|
||||
result_ttl=500)
|
||||
return u.get_id()
|
||||
'''
|
||||
|
||||
@app.route('/whois', methods=['POST'])
|
||||
def whoismail():
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('whois', {'server': parser.get("WHOIS", "server"),
|
||||
'port': parser.getint("WHOIS", "port"),
|
||||
'domain': data["query"],
|
||||
'ignorelist': ignorelist, 'replacelist': replacelist})
|
||||
|
||||
@app.route('/eupi', methods=['POST'])
|
||||
def eu():
|
||||
auth = read_auth('eupi')
|
||||
if not auth:
|
||||
return ''
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('eupi', {'url': parser.get("EUPI", "url"),
|
||||
'key': auth[0], 'q': data["query"]})
|
||||
|
||||
@app.route('/pdnscircl', methods=['POST'])
|
||||
def dnscircl():
|
||||
auth = read_auth('pdnscircl')
|
||||
if not auth:
|
||||
return ''
|
||||
user, password = auth
|
||||
url = parser.get("PDNS_CIRCL", "url")
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('pdnscircl', {'url': url, 'user': user.strip(),
|
||||
'passwd': password.strip(), 'q': data["query"]})
|
||||
|
||||
@app.route('/bgpranking', methods=['POST'])
|
||||
def bgpr():
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('bgpranking', {'ip': data["query"]})
|
||||
|
||||
@app.route('/psslcircl', methods=['POST'])
|
||||
def sslcircl():
|
||||
auth = read_auth('psslcircl')
|
||||
if not auth:
|
||||
return ''
|
||||
user, password = auth
|
||||
url = parser.get("PSSL_CIRCL", "url")
|
||||
data = request.get_json(force=True)
|
||||
return enqueue('psslcircl', {'url': url, 'user': user.strip(),
|
||||
'passwd': password.strip(), 'q': data["query"]})
|
||||
|
||||
@app.route('/get_cache', methods=['POST'])
|
||||
def get_cache():
|
||||
data = request.get_json(force=True)
|
||||
url = data["query"]
|
||||
if 'digest' in data:
|
||||
digest = data["digest"]
|
||||
else:
|
||||
digest = False
|
||||
data = urlabuse_query.cached(url, digest)
|
||||
return Response(json.dumps(data), mimetype='application/json')
|
||||
|
||||
def send(url, ip='', autosend=False):
|
||||
if not urlabuse_query.get_mail_sent(url):
|
||||
data = urlabuse_query.cached(url, digest=True)
|
||||
if not autosend:
|
||||
subject = 'URL Abuse report from ' + ip
|
||||
else:
|
||||
subject = 'URL Abuse report sent automatically'
|
||||
msg = Message(subject, sender='urlabuse@circl.lu', recipients=["info@circl.lu"])
|
||||
msg.body = data['digest'][0]
|
||||
msg.body += '\n\n'
|
||||
msg.body += json.dumps(data['result'], sort_keys=True, indent=2)
|
||||
mail.send(msg)
|
||||
urlabuse_query.set_mail_sent(url)
|
||||
|
||||
@app.route('/submit', methods=['POST'])
|
||||
def send_mail():
|
||||
data = request.get_json(force=True)
|
||||
url = data["url"]
|
||||
if not urlabuse_query.get_mail_sent(url):
|
||||
ip = _get_user_ip(request)
|
||||
send(url, ip)
|
||||
return redirect(url_for('index'))
|
||||
|
||||
return app
|
||||
send(url, ip)
|
||||
return redirect(url_for('index'))
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@
|
|||
<uq-phishtank data="ip"></uq-phishtank>
|
||||
<!-- <li><uq-virustotal data="ip"></uq-virustotal></li> -->
|
||||
<uq-bgpranking data="ip"></uq-bgpranking>
|
||||
<uq-urlquery data="ip"></uq-urlquery>
|
||||
<!-- <uq-urlquery data="ip"></uq-urlquery> -->
|
||||
<uq-pdnscircl data="ip"></uq-pdnscircl>
|
||||
<uq-psslcircl data="ip"></uq-psslcircl>
|
||||
<uq-ticket data="ip"></uq-ticket>
|
||||
<!-- <uq-ticket data="ip"></uq-ticket> -->
|
||||
<uq-whois data="ip"></uq-whois>
|
||||
</div>
|
||||
</uib-accordion>
|
||||
|
|
@ -34,9 +34,9 @@
|
|||
<uq-phishtank data="ip"></uq-phishtank>
|
||||
<!-- <li><uq-virustotal data="ip"></uq-virustotal></li> -->
|
||||
<uq-bgpranking data="ip"></uq-bgpranking>
|
||||
<uq-urlquery data="ip"></uq-urlquery>
|
||||
<!-- <uq-urlquery data="ip"></uq-urlquery> -->
|
||||
<uq-pdnscircl data="ip"></uq-pdnscircl>
|
||||
<uq-ticket data="ip"></uq-ticket>
|
||||
<!-- <uq-ticket data="ip"></uq-ticket> -->
|
||||
<uq-whois data="ip"></uq-whois>
|
||||
</div>
|
||||
</uib-accordion>
|
||||
|
|
|
|||
Loading…
Reference in New Issue