diff --git a/.gitignore b/.gitignore index a9ef073..6a84021 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ dist *egg-info *.rdb + +.env +website/secret_key diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..e5b4159 --- /dev/null +++ b/Pipfile @@ -0,0 +1,26 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +redis = ">=3" +pypssl = "*" +pypdns = "*" +pyeupi = "*" +dnspython = "*" +beautifulsoup4 = "*" +pyipasnhistory = {git = "https://github.com/D4-project/IPASN-History.git/",subdirectory = "client"} +pybgpranking = {git = "https://github.com/D4-project/BGP-Ranking.git/",subdirectory = "client"} +flask = "*" +flask-bootstrap = "*" +flask-mail = "*" +flask-wtf = "*" +gunicorn = {extras = ["gevent"],version = "*"} +pyurlabuse = {editable = true,path = "."} +pyfaup = {git = "https://github.com/stricaud/faup.git/",subdirectory = "src/lib/bindings/python/"} + +[requires] +python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..cbc9820 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,330 @@ +{ + "_meta": { + "hash": { + "sha256": "4e0f863e88d1c7416abfa49ec8f74d6c51c2d605ca4a504ab7c87af0954c3d10" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "beautifulsoup4": { + "hashes": [ + "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", + "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", + "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" + ], + "index": "pypi", + "version": "==4.7.1" + }, + "blinker": { + "hashes": [ + "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6" + ], + "version": "==1.4" + }, + "certifi": { + "hashes": [ + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" + ], + "version": "==2018.11.29" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "dnspython": { + "hashes": [ + "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01", + "sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d" + ], + "index": "pypi", + "version": "==1.16.0" + }, + "dominate": { + "hashes": [ + "sha256:4076735c0745fe771e57b2313dbb4bfeec42731816ee23cee509f66e8912aa51", + "sha256:4b9fd42d2824b79761799590697db45bf93daad511b130c50513af38da33df9b" + ], + "version": "==2.3.5" + }, + "flask": { + "hashes": [ + "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", + "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" + ], + "index": "pypi", + "version": "==1.0.2" + }, + "flask-bootstrap": { + "hashes": [ + "sha256:cb08ed940183f6343a64e465e83b3a3f13c53e1baabb8d72b5da4545ef123ac8" + ], + "index": "pypi", + "version": "==3.3.7.1" + }, + "flask-mail": { + "hashes": [ + "sha256:22e5eb9a940bf407bcf30410ecc3708f3c56cc44b29c34e1726fe85006935f41" + ], + "index": "pypi", + "version": "==0.9.1" + }, + "flask-wtf": { + "hashes": [ + "sha256:5d14d55cfd35f613d99ee7cba0fc3fbbe63ba02f544d349158c14ca15561cc36", + "sha256:d9a9e366b32dcbb98ef17228e76be15702cd2600675668bca23f63a7947fd5ac" + ], + "index": "pypi", + "version": "==0.14.2" + }, + "gevent": { + "hashes": [ + "sha256:0774babec518a24d9a7231d4e689931f31b332c4517a771e532002614e270a64", + "sha256:0e1e5b73a445fe82d40907322e1e0eec6a6745ca3cea19291c6f9f50117bb7ea", + "sha256:0ff2b70e8e338cf13bedf146b8c29d475e2a544b5d1fe14045aee827c073842c", + "sha256:107f4232db2172f7e8429ed7779c10f2ed16616d75ffbe77e0e0c3fcdeb51a51", + "sha256:14b4d06d19d39a440e72253f77067d27209c67e7611e352f79fe69e0f618f76e", + "sha256:1b7d3a285978b27b469c0ff5fb5a72bcd69f4306dbbf22d7997d83209a8ba917", + "sha256:1eb7fa3b9bd9174dfe9c3b59b7a09b768ecd496debfc4976a9530a3e15c990d1", + "sha256:2711e69788ddb34c059a30186e05c55a6b611cb9e34ac343e69cf3264d42fe1c", + "sha256:28a0c5417b464562ab9842dd1fb0cc1524e60494641d973206ec24d6ec5f6909", + "sha256:3249011d13d0c63bea72d91cec23a9cf18c25f91d1f115121e5c9113d753fa12", + "sha256:44089ed06a962a3a70e96353c981d628b2d4a2f2a75ea5d90f916a62d22af2e8", + "sha256:4bfa291e3c931ff3c99a349d8857605dca029de61d74c6bb82bd46373959c942", + "sha256:50024a1ee2cf04645535c5ebaeaa0a60c5ef32e262da981f4be0546b26791950", + "sha256:53b72385857e04e7faca13c613c07cab411480822ac658d97fd8a4ddbaf715c8", + "sha256:74b7528f901f39c39cdbb50cdf08f1a2351725d9aebaef212a29abfbb06895ee", + "sha256:7d0809e2991c9784eceeadef01c27ee6a33ca09ebba6154317a257353e3af922", + "sha256:896b2b80931d6b13b5d9feba3d4eebc67d5e6ec54f0cf3339d08487d55d93b0e", + "sha256:8d9ec51cc06580f8c21b41fd3f2b3465197ba5b23c00eb7d422b7ae0380510b0", + "sha256:9f7a1e96fec45f70ad364e46de32ccacab4d80de238bd3c2edd036867ccd48ad", + "sha256:ab4dc33ef0e26dc627559786a4fba0c2227f125db85d970abbf85b77506b3f51", + "sha256:d1e6d1f156e999edab069d79d890859806b555ce4e4da5b6418616322f0a3df1", + "sha256:d752bcf1b98174780e2317ada12013d612f05116456133a6acf3e17d43b71f05", + "sha256:e5bcc4270671936349249d26140c267397b7b4b1381f5ec8b13c53c5b53ab6e1" + ], + "version": "==1.4.0" + }, + "greenlet": { + "hashes": [ + "sha256:000546ad01e6389e98626c1367be58efa613fa82a1be98b0c6fc24b563acc6d0", + "sha256:0d48200bc50cbf498716712129eef819b1729339e34c3ae71656964dac907c28", + "sha256:23d12eacffa9d0f290c0fe0c4e81ba6d5f3a5b7ac3c30a5eaf0126bf4deda5c8", + "sha256:37c9ba82bd82eb6a23c2e5acc03055c0e45697253b2393c9a50cef76a3985304", + "sha256:51503524dd6f152ab4ad1fbd168fc6c30b5795e8c70be4410a64940b3abb55c0", + "sha256:8041e2de00e745c0e05a502d6e6db310db7faa7c979b3a5877123548a4c0b214", + "sha256:81fcd96a275209ef117e9ec91f75c731fa18dcfd9ffaa1c0adbdaa3616a86043", + "sha256:853da4f9563d982e4121fed8c92eea1a4594a2299037b3034c3c898cb8e933d6", + "sha256:8b4572c334593d449113f9dc8d19b93b7b271bdbe90ba7509eb178923327b625", + "sha256:9416443e219356e3c31f1f918a91badf2e37acf297e2fa13d24d1cc2380f8fbc", + "sha256:9854f612e1b59ec66804931df5add3b2d5ef0067748ea29dc60f0efdcda9a638", + "sha256:99a26afdb82ea83a265137a398f570402aa1f2b5dfb4ac3300c026931817b163", + "sha256:a19bf883b3384957e4a4a13e6bd1ae3d85ae87f4beb5957e35b0be287f12f4e4", + "sha256:a9f145660588187ff835c55a7d2ddf6abfc570c2651c276d3d4be8a2766db490", + "sha256:ac57fcdcfb0b73bb3203b58a14501abb7e5ff9ea5e2edfa06bb03035f0cff248", + "sha256:bcb530089ff24f6458a81ac3fa699e8c00194208a724b644ecc68422e1111939", + "sha256:beeabe25c3b704f7d56b573f7d2ff88fc99f0138e43480cecdfcaa3b87fe4f87", + "sha256:d634a7ea1fc3380ff96f9e44d8d22f38418c1c381d5fac680b272d7d90883720", + "sha256:d97b0661e1aead761f0ded3b769044bb00ed5d33e1ec865e891a8b128bf7c656" + ], + "markers": "platform_python_implementation == 'CPython'", + "version": "==0.4.15" + }, + "gunicorn": { + "extras": [ + "gevent" + ], + "hashes": [ + "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", + "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" + ], + "index": "pypi", + "version": "==19.9.0" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "itsdangerous": { + "hashes": [ + "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", + "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + ], + "version": "==1.1.0" + }, + "jinja2": { + "hashes": [ + "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", + "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + ], + "version": "==2.10" + }, + "markupsafe": { + "hashes": [ + "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", + "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", + "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", + "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", + "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", + "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", + "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", + "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", + "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", + "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", + "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", + "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", + "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", + "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", + "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", + "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", + "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", + "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", + "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", + "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", + "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", + "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", + "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", + "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", + "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", + "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", + "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", + "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" + ], + "version": "==1.1.0" + }, + "pybgpranking": { + "git": "https://github.com/D4-project/BGP-Ranking.git/", + "ref": "5e4264a411b59f6a612c4bfdde307b1d8a61ead8", + "subdirectory": "client" + }, + "pyeupi": { + "hashes": [ + "sha256:35b0e6b430f23ecd303f7cc7a8fe5147cf2509a5b2254eaf9695392c0af02901" + ], + "index": "pypi", + "version": "==1.0" + }, + "pyfaup": { + "git": "https://github.com/stricaud/faup.git/", + "ref": "de31b6965fc4149c2095c7b721f456e428404736", + "subdirectory": "src/lib/bindings/python/" + }, + "pyipasnhistory": { + "git": "https://github.com/D4-project/IPASN-History.git/", + "ref": "54857344c412a903df2abe67a8855f2fcaeef4a8", + "subdirectory": "client" + }, + "pypdns": { + "hashes": [ + "sha256:0356360156dd26d2cf27a415a10ff2bd1ff1d2eb3b2dd51b35553d60b87fd328" + ], + "index": "pypi", + "version": "==1.3" + }, + "pypssl": { + "hashes": [ + "sha256:4dbe772aefdf4ab18934d83cde79e2fc5d5ba9d2b4153dc419a63faab3432643" + ], + "index": "pypi", + "version": "==2.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", + "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + ], + "version": "==2.7.5" + }, + "pyurlabuse": { + "editable": true, + "path": "." + }, + "redis": { + "hashes": [ + "sha256:2100750629beff143b6a200a2ea8e719fcf26420adabb81402895e144c5083cf", + "sha256:8e0bdd2de02e829b6225b25646f9fb9daffea99a252610d040409a6738541f0a" + ], + "index": "pypi", + "version": "==3.0.1" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "version": "==2.21.0" + }, + "requests-cache": { + "hashes": [ + "sha256:e9270030becc739b0a7f7f834234c73a878b2d794122bf76f40055a22419eb67", + "sha256:fe561ca119879bbcfb51f03a35e35b425e18f338248e59fd5cf2166c77f457a2" + ], + "version": "==0.4.13" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "soupsieve": { + "hashes": [ + "sha256:009d8865916766f7f452880d08ff94ed4c5445011a3deaac67543b82bdb0b9ee", + "sha256:97599c45a1ddfe9ab0a0cba889b7f214b3e310b703f176a0610c0b54e207cc04" + ], + "version": "==1.7.1" + }, + "urllib3": { + "hashes": [ + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + ], + "version": "==1.24.1" + }, + "visitor": { + "hashes": [ + "sha256:2c737903b2b6864ebc6167eef7cf3b997126f1aa94bdf590f90f1436d23e480a" + ], + "version": "==0.1.3" + }, + "werkzeug": { + "hashes": [ + "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", + "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + ], + "version": "==0.14.1" + }, + "wtforms": { + "hashes": [ + "sha256:0cdbac3e7f6878086c334aa25dc5a33869a3954e9d1e015130d65a69309b3b61", + "sha256:e3ee092c827582c50877cdbd49e9ce6d2c5c1f6561f849b3b068c1b8029626f1" + ], + "version": "==2.2.1" + } + }, + "develop": {} +} diff --git a/README.md b/README.md index 095294f..a45a611 100644 --- a/README.md +++ b/README.md @@ -28,37 +28,47 @@ If you don't want to use the online version or run your own version of URL Abuse ## Install -Install the requirements +**IMPORTANT**: Use [pipenv](https://pipenv.readthedocs.io/en/latest/) -~~~ -pip install -r requirements.txt -~~~ +**NOTE**: Yes, it requires python3.6+. No, it will never support anything older. -Copy and review the configuration: +## Install redis -~~~ +```bash +git clone https://github.com/antirez/redis.git +cd redis +git checkout 5.0 +make +make test +cd .. +``` + +# Install Faup + +```bash +git clone git://github.com/stricaud/faup.git +cd faup +mkdir build +cd build +cmake .. && make +sudo make install +``` + +## Install & run URL Abuse + +```bash +git clone https://github.com/CIRCL/url-abuse.git +cd url-abuse +pipenv install +echo URLABUSE_HOME="'`pwd`'" > .env +pipenv shell +# Copy and review the configuration: cp config.ini.sample config.ini -~~~ - -Install Redis and update the configuration. - -Start the Redis back-end - -~~~ -./run_redis.sh -~~~ - -Start the workers (at least 10) - -~~~ -seq 10 | parallel -u -j 10 ./worker.py -~~~ - -Start the web interface - -~~~ -python runapp.py -~~~ +# Starts all the backend +start.py +# Start the web interface +start_website.py +``` ## Contributing diff --git a/bin/run_workers.py b/bin/run_workers.py index 4111322..d509812 100755 --- a/bin/run_workers.py +++ b/bin/run_workers.py @@ -13,6 +13,7 @@ def worker(process_id: int): urlabuse_query = Query() queue = Redis(unix_socket_path=get_socket_path('cache'), db=0, decode_responses=True) + print(f'Start Worker {process_id}') while True: jobid = queue.spop('to_process') if not jobid: diff --git a/bin/start_website.py b/bin/start_website.py new file mode 100755 index 0000000..0f673ab --- /dev/null +++ b/bin/start_website.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from subprocess import Popen +from urlabuse.helpers import get_homedir + +if __name__ == '__main__': + website_dir = get_homedir() / 'website' + Popen([f'{website_dir}/3drparty.sh'], cwd=website_dir) + try: + Popen(['gunicorn', '--worker-class', 'gevent', '-w', '10', '-b', '0.0.0.0:5100', 'web:app'], + cwd=website_dir).communicate() + except KeyboardInterrupt: + print('Stopping gunicorn.') diff --git a/requirements.txt b/requirements.txt index 114a8d6..8ce2028 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,18 @@ -redis>=3 -pypssl -pypdns -pyeupi -dnspython -beautifulsoup4 - -git+https://github.com/D4-project/IPASN-History.git/#egg=pyipasnhistory&subdirectory=client -git+https://github.com/D4-project/BGP-Ranking.git/#egg=pybgpranking&subdirectory=client +-i https://pypi.org/simple +beautifulsoup4==4.7.1 +certifi==2018.11.29 +chardet==3.0.4 +dnspython==1.16.0 +git+https://github.com/D4-project/BGP-Ranking.git/@5e4264a411b59f6a612c4bfdde307b1d8a61ead8#egg=pybgpranking&subdirectory=client +git+https://github.com/D4-project/IPASN-History.git/@54857344c412a903df2abe67a8855f2fcaeef4a8#egg=pyipasnhistory&subdirectory=client +idna==2.8 +pyeupi==1.0 +pypdns==1.3 +pypssl==2.1 +python-dateutil==2.7.5 +redis==3.0.1 +requests-cache==0.4.13 +requests==2.21.0 +six==1.12.0 +soupsieve==1.7.1 +urllib3==1.24.1 diff --git a/setup.py b/setup.py index 0255b34..f8d92fc 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( url='https://github.com/CIRCL/url-abuse/', description='URL Abuse interface', packages=['urlabuse'], - scripts=['bin/run_backend.py', 'bin/run_workers.py', 'bin/start.py'], + scripts=['bin/run_backend.py', 'bin/run_workers.py', 'bin/start.py', 'bin/start_website.py'], classifiers=[ 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', 'Development Status :: 3 - Alpha', diff --git a/website/requirements.txt b/website/requirements.txt deleted file mode 100644 index 465606a..0000000 --- a/website/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -flask -flask-bootstrap -flask-mail -flask-wtf diff --git a/website/runapp.py b/website/runapp.py deleted file mode 100755 index e50bd8d..0000000 --- a/website/runapp.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*-coding:utf-8 -* - -import os - -from web import create_app - -# create an app instance -abspath = os.path.abspath(__file__) -dname = os.path.dirname(abspath) -os.chdir(dname) -app = create_app() -app.run(host='0.0.0.0', port = 5100, debug=False, threaded=True) diff --git a/website/web/__init__.py b/website/web/__init__.py index 354833c..e37dcd5 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -17,7 +17,7 @@ from logging import Formatter from redis import Redis -from urlabuse.helpers import get_socket_path +from urlabuse.helpers import get_socket_path, get_homedir from urlabuse.urlabuse import Query import configparser @@ -62,268 +62,298 @@ def prepare_auth(): return to_return -def create_app(configfile=None): - app = Flask(__name__) - handler = RotatingFileHandler('urlabuse.log', maxBytes=10000, backupCount=5) - handler.setFormatter(Formatter('%(asctime)s %(message)s')) - app.wsgi_app = ReverseProxied(app.wsgi_app) - app.logger.addHandler(handler) - app.logger.setLevel(logging.INFO) - Bootstrap(app) - queue = Redis(unix_socket_path=get_socket_path('cache'), db=0, - decode_responses=True) - urlabuse_query = Query() +app = Flask(__name__) +handler = RotatingFileHandler('urlabuse.log', maxBytes=10000, backupCount=5) +handler.setFormatter(Formatter('%(asctime)s %(message)s')) +app.wsgi_app = ReverseProxied(app.wsgi_app) +app.logger.addHandler(handler) +app.logger.setLevel(logging.INFO) +Bootstrap(app) +queue = Redis(unix_socket_path=get_socket_path('cache'), db=0, + decode_responses=True) +urlabuse_query = Query() - # Mail Config - app.config['MAIL_SERVER'] = 'localhost' - app.config['MAIL_PORT'] = 25 - mail = Mail(app) +# Mail Config +app.config['MAIL_SERVER'] = 'localhost' +app.config['MAIL_PORT'] = 25 +mail = Mail(app) - app.config['SECRET_KEY'] = 'devkey' - app.config['BOOTSTRAP_SERVE_LOCAL'] = True - app.config['configfile'] = config_dir / 'config.ini' +secret_file_path = get_homedir() / 'website' / 'secret_key' - parser = configparser.SafeConfigParser() - parser.read(app.config['configfile']) +if not secret_file_path.exists() or secret_file_path.stat().st_size < 64: + with open(secret_file_path, 'wb') as f: + f.write(os.urandom(64)) - replacelist = make_dict(parser, 'replacelist') - auth_users = prepare_auth() - ignorelist = [i.strip() - for i in parser.get('abuse', 'ignore').split('\n') - if len(i.strip()) > 0] - autosend_threshold = 5 +with open(secret_file_path, 'rb') as f: + app.config['SECRET_KEY'] = f.read() - def _get_user_ip(request): - ip = request.headers.get('X-Forwarded-For') - if ip is None: - ip = request.remote_addr - return ip +app.config['BOOTSTRAP_SERVE_LOCAL'] = True +app.config['configfile'] = config_dir / 'config.ini' - @app.route('/', methods=['GET', 'POST']) - def index(): - if request.method == 'HEAD': - # Just returns ack if the webserver is running - return 'Ack' - form = URLForm() - return render_template('index.html', form=form) +parser = configparser.SafeConfigParser() +parser.read(app.config['configfile']) - @app.route('/urlreport', methods=['GET']) - def url_report(): - return render_template('url-report.html') +replacelist = make_dict(parser, 'replacelist') +auth_users = prepare_auth() +ignorelist = [i.strip() + for i in parser.get('abuse', 'ignore').split('\n') + if len(i.strip()) > 0] +autosend_threshold = 5 - @app.errorhandler(404) - def page_not_found(e): - ip = request.headers.get('X-Forwarded-For') - if ip is None: - ip = request.remote_addr - if request.path != '/_result/': - app.logger.info('404 of {} on {}'.format(ip, request.path)) - return render_template('404.html'), 404 - def authenticate(): - """Sends a 401 response that enables basic auth""" - return Response('Could not verify your access level for that URL.\n' - 'You have to login with proper credentials', 401, - {'WWW-Authenticate': 'Basic realm="Login Required"'}) +def _get_user_ip(request): + ip = request.headers.get('X-Forwarded-For') + if ip is None: + ip = request.remote_addr + return ip - def check_auth(username, password): - """This function is called to check if a username / - password combination is valid. - """ - if auth_users is None: - return False + +@app.route('/', methods=['GET', 'POST']) +def index(): + if request.method == 'HEAD': + # Just returns ack if the webserver is running + return 'Ack' + form = URLForm() + return render_template('index.html', form=form) + + +@app.route('/urlreport', methods=['GET']) +def url_report(): + return render_template('url-report.html') + + +@app.errorhandler(404) +def page_not_found(e): + ip = request.headers.get('X-Forwarded-For') + if ip is None: + ip = request.remote_addr + if request.path != '/_result/': + app.logger.info('404 of {} on {}'.format(ip, request.path)) + return render_template('404.html'), 404 + + +def authenticate(): + """Sends a 401 response that enables basic auth""" + return Response('Could not verify your access level for that URL.\n' + 'You have to login with proper credentials', 401, + {'WWW-Authenticate': 'Basic realm="Login Required"'}) + + +def check_auth(username, password): + """This function is called to check if a username / + password combination is valid. + """ + if auth_users is None: + return False + else: + db_pass = auth_users.get(username) + return db_pass == password + + +@app.route('/login', methods=['GET', 'POST']) +def login(): + auth = request.authorization + if not auth or not check_auth(auth.username, auth.password): + return authenticate() + return redirect(url_for('index')) + + +@app.route("/_result/", methods=['GET']) +def check_valid(job_key): + if not job_key or not queue.exists(job_key): + return Response(json.dumps(None), mimetype='application/json'), 200 + if not queue.hexists(job_key, 'result'): + return Response(json.dumps('Nay!'), mimetype='application/json'), 202 + result = queue.hget(job_key, 'result') + queue.delete(job_key) + return Response(result, mimetype='application/json'), 200 + + +def enqueue(method, data): + job_id = str(uuid.uuid4()) + p = queue.pipeline() + p.hmset(job_id, {'method': method, 'data': json.dumps(data)}) + p.sadd('to_process', job_id) + p.execute() + return job_id + + +@app.route('/start', methods=['POST']) +def run_query(): + data = request.get_json(force=True) + url = data["url"] + ip = _get_user_ip(request) + app.logger.info(f'{ip} {url}') + if urlabuse_query.get_submissions(url) and urlabuse_query.get_submissions(url) >= autosend_threshold: + send(url, '', True) + return enqueue('is_valid_url', {'url': url}) + + +@app.route('/urls', methods=['POST']) +def urls(): + data = request.get_json(force=True) + return enqueue('url_list', {'url': data["url"]}) + + +@app.route('/resolve', methods=['POST']) +def resolve(): + data = request.get_json(force=True) + return enqueue('dns_resolve', {'url': data["url"]}) + + +def read_auth(name): + key = config_dir / f'{name}.key' + if not key.exists(): + return '' + with open(key) as f: + to_return = [] + for line in f.readlines(): + to_return.append(line.strip()) + return to_return + + +@app.route('/phishtank', methods=['POST']) +def phishtank(): + auth = read_auth('phishtank') + if not auth: + return '' + data = request.get_json(force=True) + return enqueue('phish_query', {'url': parser.get("PHISHTANK", "url"), + 'key': auth[0], 'query': data["query"]}) + + +@app.route('/virustotal_report', methods=['POST']) +def vt(): + auth = read_auth('virustotal') + if not auth: + return '' + data = request.get_json(force=True) + return enqueue('vt_query_url', {'url': parser.get("VIRUSTOTAL", "url_report"), + 'url_up': parser.get("VIRUSTOTAL", "url_upload"), + 'key': auth[0], 'query': data["query"]}) + + +@app.route('/googlesafebrowsing', methods=['POST']) +def gsb(): + auth = read_auth('googlesafebrowsing') + if not auth: + return '' + key = auth[0] + data = request.get_json(force=True) + url = parser.get("GOOGLESAFEBROWSING", "url").format(key) + return enqueue('gsb_query', {'url': url, + 'query': data["query"]}) + + +''' +@app.route('/urlquery', methods=['POST']) +def urlquery(): + auth = read_auth('urlquery') + if not auth: + return '' + key = auth[0] + data = json.loads(request.data.decode()) + url = parser.get("URLQUERY", "url") + query = data["query"] + u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500) + return u.get_id() + +@app.route('/ticket', methods=['POST']) +def ticket(): + if not request.authorization: + return '' + data = json.loads(request.data.decode()) + server = parser.get("SPHINX", "server") + port = int(parser.get("SPHINX", "port")) + url = parser.get("ITS", "url") + query = data["query"] + u = q.enqueue_call(func=sphinxsearch, args=(server, port, url, query,), + result_ttl=500) + return u.get_id() +''' + + +@app.route('/whois', methods=['POST']) +def whoismail(): + data = request.get_json(force=True) + return enqueue('whois', {'server': parser.get("WHOIS", "server"), + 'port': parser.getint("WHOIS", "port"), + 'domain': data["query"], + 'ignorelist': ignorelist, 'replacelist': replacelist}) + + +@app.route('/eupi', methods=['POST']) +def eu(): + auth = read_auth('eupi') + if not auth: + return '' + data = request.get_json(force=True) + return enqueue('eupi', {'url': parser.get("EUPI", "url"), + 'key': auth[0], 'q': data["query"]}) + + +@app.route('/pdnscircl', methods=['POST']) +def dnscircl(): + auth = read_auth('pdnscircl') + if not auth: + return '' + user, password = auth + url = parser.get("PDNS_CIRCL", "url") + data = request.get_json(force=True) + return enqueue('pdnscircl', {'url': url, 'user': user.strip(), + 'passwd': password.strip(), 'q': data["query"]}) + + +@app.route('/bgpranking', methods=['POST']) +def bgpr(): + data = request.get_json(force=True) + return enqueue('bgpranking', {'ip': data["query"]}) + + +@app.route('/psslcircl', methods=['POST']) +def sslcircl(): + auth = read_auth('psslcircl') + if not auth: + return '' + user, password = auth + url = parser.get("PSSL_CIRCL", "url") + data = request.get_json(force=True) + return enqueue('psslcircl', {'url': url, 'user': user.strip(), + 'passwd': password.strip(), 'q': data["query"]}) + + +@app.route('/get_cache', methods=['POST']) +def get_cache(): + data = request.get_json(force=True) + url = data["query"] + if 'digest' in data: + digest = data["digest"] + else: + digest = False + data = urlabuse_query.cached(url, digest) + return Response(json.dumps(data), mimetype='application/json') + + +def send(url, ip='', autosend=False): + if not urlabuse_query.get_mail_sent(url): + data = urlabuse_query.cached(url, digest=True) + if not autosend: + subject = 'URL Abuse report from ' + ip else: - db_pass = auth_users.get(username) - return db_pass == password + subject = 'URL Abuse report sent automatically' + msg = Message(subject, sender='urlabuse@circl.lu', recipients=["info@circl.lu"]) + msg.body = data['digest'][0] + msg.body += '\n\n' + msg.body += json.dumps(data['result'], sort_keys=True, indent=2) + mail.send(msg) + urlabuse_query.set_mail_sent(url) - @app.route('/login', methods=['GET', 'POST']) - def login(): - auth = request.authorization - if not auth or not check_auth(auth.username, auth.password): - return authenticate() - return redirect(url_for('index')) - @app.route("/_result/", methods=['GET']) - def check_valid(job_key): - if not job_key or not queue.exists(job_key): - return Response(json.dumps(None), mimetype='application/json'), 200 - if not queue.hexists(job_key, 'result'): - return Response(json.dumps('Nay!'), mimetype='application/json'), 202 - result = queue.hget(job_key, 'result') - queue.delete(job_key) - return Response(result, mimetype='application/json'), 200 - - def enqueue(method, data): - job_id = str(uuid.uuid4()) - p = queue.pipeline() - p.hmset(job_id, {'method': method, 'data': json.dumps(data)}) - p.sadd('to_process', job_id) - p.execute() - return job_id - - @app.route('/start', methods=['POST']) - def run_query(): - data = request.get_json(force=True) - url = data["url"] +@app.route('/submit', methods=['POST']) +def send_mail(): + data = request.get_json(force=True) + url = data["url"] + if not urlabuse_query.get_mail_sent(url): ip = _get_user_ip(request) - app.logger.info(f'{ip} {url}') - if urlabuse_query.get_submissions(url) and urlabuse_query.get_submissions(url) >= autosend_threshold: - send(url, '', True) - return enqueue('is_valid_url', {'url': url}) - - @app.route('/urls', methods=['POST']) - def urls(): - data = request.get_json(force=True) - return enqueue('url_list', {'url': data["url"]}) - - @app.route('/resolve', methods=['POST']) - def resolve(): - data = request.get_json(force=True) - return enqueue('dns_resolve', {'url': data["url"]}) - - def read_auth(name): - key = config_dir / f'{name}.key' - if not key.exists(): - return '' - with open(key) as f: - to_return = [] - for line in f.readlines(): - to_return.append(line.strip()) - return to_return - - @app.route('/phishtank', methods=['POST']) - def phishtank(): - auth = read_auth('phishtank') - if not auth: - return '' - data = request.get_json(force=True) - return enqueue('phish_query', {'url': parser.get("PHISHTANK", "url"), - 'key': auth[0], 'query': data["query"]}) - - @app.route('/virustotal_report', methods=['POST']) - def vt(): - auth = read_auth('virustotal') - if not auth: - return '' - data = request.get_json(force=True) - return enqueue('vt_query_url', {'url': parser.get("VIRUSTOTAL", "url_report"), - 'url_up': parser.get("VIRUSTOTAL", "url_upload"), - 'key': auth[0], 'query': data["query"]}) - - @app.route('/googlesafebrowsing', methods=['POST']) - def gsb(): - auth = read_auth('googlesafebrowsing') - if not auth: - return '' - key = auth[0] - data = request.get_json(force=True) - url = parser.get("GOOGLESAFEBROWSING", "url").format(key) - return enqueue('gsb_query', {'url': url, - 'query': data["query"]}) - - ''' - @app.route('/urlquery', methods=['POST']) - def urlquery(): - auth = read_auth('urlquery') - if not auth: - return '' - key = auth[0] - data = json.loads(request.data.decode()) - url = parser.get("URLQUERY", "url") - query = data["query"] - u = q.enqueue_call(func=urlquery_query, args=(url, key, query,), result_ttl=500) - return u.get_id() - - @app.route('/ticket', methods=['POST']) - def ticket(): - if not request.authorization: - return '' - data = json.loads(request.data.decode()) - server = parser.get("SPHINX", "server") - port = int(parser.get("SPHINX", "port")) - url = parser.get("ITS", "url") - query = data["query"] - u = q.enqueue_call(func=sphinxsearch, args=(server, port, url, query,), - result_ttl=500) - return u.get_id() - ''' - - @app.route('/whois', methods=['POST']) - def whoismail(): - data = request.get_json(force=True) - return enqueue('whois', {'server': parser.get("WHOIS", "server"), - 'port': parser.getint("WHOIS", "port"), - 'domain': data["query"], - 'ignorelist': ignorelist, 'replacelist': replacelist}) - - @app.route('/eupi', methods=['POST']) - def eu(): - auth = read_auth('eupi') - if not auth: - return '' - data = request.get_json(force=True) - return enqueue('eupi', {'url': parser.get("EUPI", "url"), - 'key': auth[0], 'q': data["query"]}) - - @app.route('/pdnscircl', methods=['POST']) - def dnscircl(): - auth = read_auth('pdnscircl') - if not auth: - return '' - user, password = auth - url = parser.get("PDNS_CIRCL", "url") - data = request.get_json(force=True) - return enqueue('pdnscircl', {'url': url, 'user': user.strip(), - 'passwd': password.strip(), 'q': data["query"]}) - - @app.route('/bgpranking', methods=['POST']) - def bgpr(): - data = request.get_json(force=True) - return enqueue('bgpranking', {'ip': data["query"]}) - - @app.route('/psslcircl', methods=['POST']) - def sslcircl(): - auth = read_auth('psslcircl') - if not auth: - return '' - user, password = auth - url = parser.get("PSSL_CIRCL", "url") - data = request.get_json(force=True) - return enqueue('psslcircl', {'url': url, 'user': user.strip(), - 'passwd': password.strip(), 'q': data["query"]}) - - @app.route('/get_cache', methods=['POST']) - def get_cache(): - data = request.get_json(force=True) - url = data["query"] - if 'digest' in data: - digest = data["digest"] - else: - digest = False - data = urlabuse_query.cached(url, digest) - return Response(json.dumps(data), mimetype='application/json') - - def send(url, ip='', autosend=False): - if not urlabuse_query.get_mail_sent(url): - data = urlabuse_query.cached(url, digest=True) - if not autosend: - subject = 'URL Abuse report from ' + ip - else: - subject = 'URL Abuse report sent automatically' - msg = Message(subject, sender='urlabuse@circl.lu', recipients=["info@circl.lu"]) - msg.body = data['digest'][0] - msg.body += '\n\n' - msg.body += json.dumps(data['result'], sort_keys=True, indent=2) - mail.send(msg) - urlabuse_query.set_mail_sent(url) - - @app.route('/submit', methods=['POST']) - def send_mail(): - data = request.get_json(force=True) - url = data["url"] - if not urlabuse_query.get_mail_sent(url): - ip = _get_user_ip(request) - send(url, ip) - return redirect(url_for('index')) - - return app + send(url, ip) + return redirect(url_for('index')) diff --git a/website/web/templates/url-report.html b/website/web/templates/url-report.html index 968b2b8..d0db091 100644 --- a/website/web/templates/url-report.html +++ b/website/web/templates/url-report.html @@ -18,10 +18,10 @@ - + - + @@ -34,9 +34,9 @@ - + - +