diff --git a/bin/Crawler.py b/bin/Crawler.py index fa0a796d..848d2b67 100755 --- a/bin/Crawler.py +++ b/bin/Crawler.py @@ -341,13 +341,27 @@ if __name__ == '__main__': faup = Faup() + # get HAR files + default_crawler_har = p.config.getboolean("Crawler", "default_crawler_har") + if default_crawler_har: + default_crawler_har = 1 + else: + default_crawler_har = 0 + + # get PNG files + default_crawler_png = p.config.getboolean("Crawler", "default_crawler_png") + if default_crawler_png: + default_crawler_png = 1 + else: + default_crawler_png = 0 + # Default crawler options default_crawler_config = {'html': 1, - 'har': 1, - 'png': 1, + 'har': default_crawler_har, + 'png': default_crawler_png, 'depth_limit': p.config.getint("Crawler", "crawler_depth_limit"), - 'closespider_pagecount': 50, - 'user_agent': 'Mozilla/5.0 (Windows NT 6.1; rv:24.0) Gecko/20100101 Firefox/24.0'} + 'closespider_pagecount': p.config.getint("Crawler", "default_crawler_closespider_pagecount"), + 'user_agent': p.config.get("Crawler", "default_crawler_user_agent")} # Track launched crawler r_cache.sadd('all_crawler', splash_port) diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py new file mode 100755 index 00000000..b769600b --- /dev/null +++ b/bin/packages/Correlation.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +import Flask_config + +r_serv_metadata = Flask_config.r_serv_metadata + + +class Correlation(object): + + def __init__(self, correlation_name): + self.correlation_name = correlation_name + + def _exist_corelation_field(self, correlation_type, field_name): + return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name)) + + + def _get_items(self, correlation_type, field_name): + res = r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name)) + if res: + return list(res) + else: + return [] + + + def _get_metadata(self, correlation_type, field_name): + meta_dict = {} + meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen') + meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen') + return meta_dict + + def _get_correlation_by_date(self, correlation_type, date): + return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date)) + + def verify_correlation_field_request(self, request_dict, correlation_type): + if not request_dict: + return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400) + + field_name = request_dict.get(correlation_type, None) + if not field_name: + return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 ) + if not self._exist_corelation_field(correlation_type, field_name): + return ( {'status': 'error', 'reason': 'Item not found'}, 404 ) + + def get_correlation(self, request_dict, correlation_type, field_name): + dict_resp = {} + + if request_dict.get('items'): + dict_resp['items'] = self._get_items(correlation_type, field_name) + + if request_dict.get('metadata'): + dict_resp['metadata'] = self._get_metadata(correlation_type, field_name) + + dict_resp[correlation_type] = field_name + + return (dict_resp, 200) + + + + +#cryptocurrency_all:cryptocurrency name cryptocurrency address nb seen diff --git a/bin/packages/Cryptocurrency.py b/bin/packages/Cryptocurrency.py new file mode 100755 index 00000000..1b4e4e3f --- /dev/null +++ b/bin/packages/Cryptocurrency.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +from hashlib import sha256 + +import Flask_config +from Correlation import Correlation + +r_serv_metadata = Flask_config.r_serv_metadata + +digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + +cryptocurrency = Correlation('cryptocurrency') + +def decode_base58(bc, length): + n = 0 + for char in bc: + n = n * 58 + digits58.index(char) + return n.to_bytes(length, 'big') + +def check_bitcoin_address(bc): + try: + bcbytes = decode_base58(bc, 25) + return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4] + except Exception: + return False + +def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address): + if cryptocurrency_type == 'bitcoin': + return check_bitcoin_address(cryptocurrency_address) + else: + return True + + +def get_cryptocurrency(request_dict, cryptocurrency_type): + # basic verification + res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type) + if res: + return res + # cerify address + field_name = request_dict.get(cryptocurrency_type) + if not verify_cryptocurrency_address(cryptocurrency_type, field_name): + return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 ) + + return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 4dcdde85..aa2b92c2 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -14,6 +14,7 @@ import Tag PASTES_FOLDER = Flask_config.PASTES_FOLDER r_cache = Flask_config.r_cache +r_serv_metadata = Flask_config.r_serv_metadata def exist_item(item_id): if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)): @@ -93,4 +94,43 @@ def get_item(request_dict): if lines_info: dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None')) + if request_dict.get('pgp'): + dict_item['pgp'] = {} + if request_dict['pgp'].get('key'): + dict_item['pgp']['key'] = get_item_pgp_key(item_id) + if request_dict['pgp'].get('mail'): + dict_item['pgp']['mail'] = get_item_pgp_mail(item_id) + if request_dict['pgp'].get('name'): + dict_item['pgp']['name'] = get_item_pgp_name(item_id) + + if request_dict.get('cryptocurrency'): + dict_item['cryptocurrency'] = {} + if request_dict['cryptocurrency'].get('bitcoin'): + dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id) + return (dict_item, 200) + + +### +### correlation +### + +def _get_item_correlation(correlation_name, correlation_type, item_id): + print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id)) + res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id)) + if res: + return list(res) + else: + return [] + +def get_item_bitcoin(item_id): + return _get_item_correlation('cryptocurrency', 'bitcoin', item_id) + +def get_item_pgp_key(item_id): + return _get_item_correlation('pgpdump', 'key', item_id) + +def get_item_pgp_name(item_id): + return _get_item_correlation('pgpdump', 'name', item_id) + +def get_item_pgp_mail(item_id): + return _get_item_correlation('pgpdump', 'mail', item_id) diff --git a/bin/packages/Pgp.py b/bin/packages/Pgp.py new file mode 100755 index 00000000..9c7b0ec4 --- /dev/null +++ b/bin/packages/Pgp.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +from hashlib import sha256 + +import Flask_config +from Correlation import Correlation + +r_serv_metadata = Flask_config.r_serv_metadata + +pgpdump = Correlation('pgpdump') + + +def get_pgp(request_dict, pgp_type): + # basic verification + res = pgpdump.verify_correlation_field_request(request_dict, pgp_type) + if res: + return res + # cerify address + field_name = request_dict.get(pgp_type) + + return pgpdump.get_correlation(request_dict, pgp_type, field_name) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 52388ed5..0d32293b 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -255,5 +255,9 @@ db = 0 [Crawler] activate_crawler = False crawler_depth_limit = 1 +default_crawler_har = True +default_crawler_png = True +default_crawler_closespider_pagecount = 50 +default_crawler_user_agent = Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0 splash_url = http://127.0.0.1 splash_port = 8050-8052 diff --git a/doc/README.md b/doc/README.md index 03a1aa07..8750c475 100644 --- a/doc/README.md +++ b/doc/README.md @@ -250,6 +250,25 @@ Get item. Filter requested field. - get item lines info - *boolean* - default: `false` +- `cryptocurrency` + - `bitcoin` + - get item bitcoin adress + - *boolean* + - default: `false` +- `pgp` + - `key` + - get item pgp key + - *boolean* + - default: `false` + - `mail` + - get item pgp mail + - *boolean* + - default: `false` + - `name` + - get item pgp name + - *boolean* + - default: `false` + #### JSON response - `content` @@ -276,6 +295,20 @@ Get item. Filter requested field. - `nb` - nb lines item - *int* +- `cryptocurrency` + - `bitcoin` + - item bitcoin adress + - *list* +- `pgp` + - `key` + - item pgp keys + - *list* + - `mail` + - item pgp mails + - *list* + - `name` + - item pgp name + - *list* #### Example @@ -299,12 +332,28 @@ curl https://127.0.0.1:7000/api/v1/get/item --header "Authorization: iHc1_ChZxj1 ```json { "content": "dsvcdsvcdsc vvvv", + "cryptocurrency": { + "bitcoin": [ + "132M1aGTGodHkQNh1augLeMjEXH51wgoCc" + ] + }, "date": "20190726", "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", "lines": { "max_length": 19, "nb": 1 }, + "pgp": { + "key": [ + "0x5180D21F4C20F975" + ], + "mail": [ + "mail@test.test" + ], + "name": [ + "user_test" + ] + }, "size": 0.03, "tags": [ "misp-galaxy:stealer=\"Vidar\"", @@ -546,10 +595,10 @@ Get tag metadata. - *str* - `first_seen` - date: first seen - - *str - YYMMDD* + - *str - YYYYMMDD* - `last_seen` - - date: first seen - - *str - YYMMDD* + - date: last seen + - *str - YYYYMMDD* #### Example ``` curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST @@ -581,6 +630,116 @@ curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc +## Cryptocurrency + + + +### Get bitcoin metadata: `api/v1/get/cryptocurrency/bitcoin/metadata` + +#### Description +Get all metdata from a bitcoin address. + +**Method** : `POST` + +#### Parameters +- `bitcoin` + - bitcoin address + - *str* + - mandatory + +#### JSON response +- `bitcoin` + - bitcoin address + - *str* +- `first_seen` + - date: first seen + - *str - YYYYMMDD* +- `last_seen` + - date: last seen + - *str - YYYYMMDD* +#### Example +``` +curl https://127.0.0.1:7000/api/v1/get/cryptocurrency/bitcoin/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST +``` + +#### input.json Example +```json + { + "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y" + } +``` + +#### Expected Success Response +**HTTP Status Code** : `200` +```json + { + "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y", + "first_seen": "20190605", + "last_seen": "20190726" + } +``` + +#### Expected Fail Response +**HTTP Status Code** : `404` +```json + {"status": "error", "reason": "Item not found"} +``` + + + +### Get bitcoin metadata: `api/v1/get/cryptocurrency/bitcoin/item` + +#### Description +Get all items related to a bitcoin address. + +**Method** : `POST` + +#### Parameters +- `bitcoin` + - bitcoin address + - *str* + - mandatory + +#### JSON response +- `bitcoin` + - bitcoin address + - *str* +- `items` + - list of item id + - *list* +#### Example +``` +curl https://127.0.0.1:7000/api/v1/get/cryptocurrency/bitcoin/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST +``` + +#### input.json Example +```json + { + "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y" + } +``` + +#### Expected Success Response +**HTTP Status Code** : `200` +```json + { + "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y", + "items": [ + "archive/2019/08/26/test_bitcoin001", + "archive/2019/08/26/test_bitcoin002", + "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz" + ] + } +``` + +#### Expected Fail Response +**HTTP Status Code** : `404` +```json + {"status": "error", "reason": "Item not found"} +``` + + + @@ -777,9 +936,6 @@ curl https://127.0.0.1:7000/api/v1/add/tracker/term --header "Authorization: iHc - - - ## Import management diff --git a/ansible/.gitignore b/other_installers/ansible/.gitignore similarity index 100% rename from ansible/.gitignore rename to other_installers/ansible/.gitignore diff --git a/ansible/Dockerfile.testing b/other_installers/ansible/Dockerfile.testing similarity index 100% rename from ansible/Dockerfile.testing rename to other_installers/ansible/Dockerfile.testing diff --git a/ansible/README.md b/other_installers/ansible/README.md similarity index 100% rename from ansible/README.md rename to other_installers/ansible/README.md diff --git a/ansible/deploy.sh b/other_installers/ansible/deploy.sh similarity index 100% rename from ansible/deploy.sh rename to other_installers/ansible/deploy.sh diff --git a/ansible/deploy.yml b/other_installers/ansible/deploy.yml similarity index 100% rename from ansible/deploy.yml rename to other_installers/ansible/deploy.yml diff --git a/ansible/deployLocal.yml b/other_installers/ansible/deployLocal.yml similarity index 100% rename from ansible/deployLocal.yml rename to other_installers/ansible/deployLocal.yml diff --git a/ansible/group_vars/ail.yml b/other_installers/ansible/group_vars/ail.yml similarity index 100% rename from ansible/group_vars/ail.yml rename to other_installers/ansible/group_vars/ail.yml diff --git a/ansible/roles/ail-host/files/ail-flask.service b/other_installers/ansible/roles/ail-host/files/ail-flask.service similarity index 100% rename from ansible/roles/ail-host/files/ail-flask.service rename to other_installers/ansible/roles/ail-host/files/ail-flask.service diff --git a/ansible/roles/ail-host/files/ail-leveldb.service b/other_installers/ansible/roles/ail-host/files/ail-leveldb.service similarity index 100% rename from ansible/roles/ail-host/files/ail-leveldb.service rename to other_installers/ansible/roles/ail-host/files/ail-leveldb.service diff --git a/ansible/roles/ail-host/files/ail-logging.service b/other_installers/ansible/roles/ail-host/files/ail-logging.service similarity index 100% rename from ansible/roles/ail-host/files/ail-logging.service rename to other_installers/ansible/roles/ail-host/files/ail-logging.service diff --git a/ansible/roles/ail-host/files/ail-queues.service b/other_installers/ansible/roles/ail-host/files/ail-queues.service similarity index 100% rename from ansible/roles/ail-host/files/ail-queues.service rename to other_installers/ansible/roles/ail-host/files/ail-queues.service diff --git a/ansible/roles/ail-host/files/ail-redis.service b/other_installers/ansible/roles/ail-host/files/ail-redis.service similarity index 100% rename from ansible/roles/ail-host/files/ail-redis.service rename to other_installers/ansible/roles/ail-host/files/ail-redis.service diff --git a/ansible/roles/ail-host/files/ail-scripts.service b/other_installers/ansible/roles/ail-host/files/ail-scripts.service similarity index 100% rename from ansible/roles/ail-host/files/ail-scripts.service rename to other_installers/ansible/roles/ail-host/files/ail-scripts.service diff --git a/ansible/roles/ail-host/files/startFlask.sh b/other_installers/ansible/roles/ail-host/files/startFlask.sh similarity index 100% rename from ansible/roles/ail-host/files/startFlask.sh rename to other_installers/ansible/roles/ail-host/files/startFlask.sh diff --git a/ansible/roles/ail-host/files/startLogging.sh b/other_installers/ansible/roles/ail-host/files/startLogging.sh similarity index 100% rename from ansible/roles/ail-host/files/startLogging.sh rename to other_installers/ansible/roles/ail-host/files/startLogging.sh diff --git a/ansible/roles/ail-host/files/startQueues.sh b/other_installers/ansible/roles/ail-host/files/startQueues.sh similarity index 100% rename from ansible/roles/ail-host/files/startQueues.sh rename to other_installers/ansible/roles/ail-host/files/startQueues.sh diff --git a/ansible/roles/ail-host/files/startRedis.sh b/other_installers/ansible/roles/ail-host/files/startRedis.sh similarity index 100% rename from ansible/roles/ail-host/files/startRedis.sh rename to other_installers/ansible/roles/ail-host/files/startRedis.sh diff --git a/ansible/roles/ail-host/files/startRedisLevelDB.sh b/other_installers/ansible/roles/ail-host/files/startRedisLevelDB.sh similarity index 100% rename from ansible/roles/ail-host/files/startRedisLevelDB.sh rename to other_installers/ansible/roles/ail-host/files/startRedisLevelDB.sh diff --git a/ansible/roles/ail-host/files/startScripts.sh b/other_installers/ansible/roles/ail-host/files/startScripts.sh similarity index 100% rename from ansible/roles/ail-host/files/startScripts.sh rename to other_installers/ansible/roles/ail-host/files/startScripts.sh diff --git a/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.min.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.min.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.min.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.min.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/bootstrap.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/bootstrap.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/bootstrap.min.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap.min.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/bootstrap.min.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap.min.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/dataTables.bootstrap.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/dataTables.bootstrap.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/dataTables.bootstrap.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/dataTables.bootstrap.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/jquery-ui.min.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/jquery-ui.min.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/jquery-ui.min.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/jquery-ui.min.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/sb-admin-2.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-2.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/sb-admin-2.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-2.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/sb-admin.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/sb-admin.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin.css diff --git a/ansible/roles/ail-host/files/staticBackup/css/timeline.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/timeline.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/css/timeline.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/timeline.css diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.css b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.css diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.min.css b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.min.css similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.min.css rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.min.css diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.svg b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.svg similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.svg rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.svg diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/icons.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/icons.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/icons.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/icons.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/variables.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/variables.less similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/variables.less rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/variables.less diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_icons.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_icons.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_icons.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_icons.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_variables.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_variables.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_variables.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_variables.scss diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.svg b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.svg similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.svg rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.svg diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2 b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2 similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2 rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2 diff --git a/ansible/roles/ail-host/files/staticBackup/image/AIL.png b/other_installers/ansible/roles/ail-host/files/staticBackup/image/AIL.png similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/image/AIL.png rename to other_installers/ansible/roles/ail-host/files/staticBackup/image/AIL.png diff --git a/ansible/roles/ail-host/files/staticBackup/js/bootstrap.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/bootstrap.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/bootstrap.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/bootstrap.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/bootstrap.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/bootstrap.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/bootstrap.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/bootstrap.min.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/dataTables.bootstrap.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/dataTables.bootstrap.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/dataTables.bootstrap.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/dataTables.bootstrap.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/dygraph-combined.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/dygraph-combined.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/dygraph-combined.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/dygraph-combined.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery-ui.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery-ui.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery-ui.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery-ui.min.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.canvasjs.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.canvasjs.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery.canvasjs.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.canvasjs.min.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.dataTables.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.dataTables.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery.dataTables.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.dataTables.min.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery.flot.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.pie.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.pie.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery.flot.pie.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.pie.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.stack.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.stack.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery.flot.stack.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.stack.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.time.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.time.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery.flot.time.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.time.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.sparkline.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.sparkline.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/jquery.sparkline.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.sparkline.min.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/excanvas.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/excanvas.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/excanvas.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/excanvas.min.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/flot-data.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/flot-data.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/flot-data.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/flot-data.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.pie.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.pie.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.pie.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.pie.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.resize.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.resize.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.resize.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.resize.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.tooltip.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.tooltip.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.tooltip.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.tooltip.min.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris-data.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris-data.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris-data.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris-data.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.min.js diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/raphael.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/raphael.min.js similarity index 100% rename from ansible/roles/ail-host/files/staticBackup/js/plugins/morris/raphael.min.js rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/raphael.min.js diff --git a/ansible/roles/ail-host/tasks/main.yml b/other_installers/ansible/roles/ail-host/tasks/main.yml similarity index 100% rename from ansible/roles/ail-host/tasks/main.yml rename to other_installers/ansible/roles/ail-host/tasks/main.yml diff --git a/Dockerfile b/other_installers/docker/Dockerfile similarity index 100% rename from Dockerfile rename to other_installers/docker/Dockerfile diff --git a/doc/X-docker.md b/other_installers/docker/README.md similarity index 100% rename from doc/X-docker.md rename to other_installers/docker/README.md diff --git a/docker-compose.yml b/other_installers/docker/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to other_installers/docker/docker-compose.yml diff --git a/docker_start.sh b/other_installers/docker/docker_start.sh similarity index 100% rename from docker_start.sh rename to other_installers/docker/docker_start.sh diff --git a/pystemon/config.cfg b/other_installers/docker/pystemon/config.cfg similarity index 100% rename from pystemon/config.cfg rename to other_installers/docker/pystemon/config.cfg diff --git a/pystemon/install.sh b/other_installers/docker/pystemon/install.sh similarity index 100% rename from pystemon/install.sh rename to other_installers/docker/pystemon/install.sh diff --git a/pystemon/proxies.txt b/other_installers/docker/pystemon/proxies.txt similarity index 100% rename from pystemon/proxies.txt rename to other_installers/docker/pystemon/proxies.txt diff --git a/pystemon/pystemon.yaml b/other_installers/docker/pystemon/pystemon.yaml similarity index 100% rename from pystemon/pystemon.yaml rename to other_installers/docker/pystemon/pystemon.yaml diff --git a/requirements.txt b/requirements.txt index fdccf9bb..dbebad0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ whoosh beautifulsoup4 ipaddress -pycountry +pycountry==18.12.8 # To fetch Onion urls PySocks diff --git a/reset_AIL.sh b/reset_AIL.sh index e87dfb67..aac94352 100755 --- a/reset_AIL.sh +++ b/reset_AIL.sh @@ -6,52 +6,144 @@ GREEN="\\033[1;32m" [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -# Make sure the reseting is intentional -num=$(( ( RANDOM % 100 ) + 1 )) +function reset_dir { + # Access dirs and delete + cd $AIL_HOME -echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num -read userInput + # Kill all screens + screen -ls | grep Detached | cut -d. -f1 | awk '{print $1}' | xargs kill -if [ $userInput -eq $num ] -then - echo "Reseting AIL..." -else - echo "Wrong number" - exit 1; -fi + # Access dirs and delete + cd $AIL_HOME + if [ -d indexdir/ ]; then + pushd indexdir/ + rm -r * + echo 'cleaned indexdir' + popd + fi -# Kill all screens -screen -ls | grep Detached | cut -d. -f1 | awk '{print $1}' | xargs kill + if [ $userInput -eq $num ] + then + if [ -d DATA_ARDB/ ]; then + pushd DATA_ARDB/ + rm -r * + echo 'cleaned DATA_ARDB' + popd + fi + fi -set -e + if [ -d logs/ ]; then + pushd logs/ + rm * + echo 'cleaned logs' + popd + fi -# Access dirs and delete -cd $AIL_HOME + if [ -d PASTES/ ]; then + pushd PASTES/ + rm -r * + echo 'cleaned PASTES' + popd + fi -pushd dumps/ -rm * -echo 'cleaned dumps' -popd + if [ -d HASHS/ ]; then + pushd HASHS/ + rm -r * + echo 'cleaned HASHS' + popd + fi -pushd indexdir/ -rm -r * -echo 'cleaned indexdir' -popd + if [ -d CRAWLED_SCREESHOT/ ]; then + pushd CRAWLED_SCREESHOT/ + rm -r * + echo 'cleaned CRAWLED_SCREESHOT' + popd + fi -pushd LEVEL_DB_DATA/ -rm -r * -echo 'cleaned LEVEL_DB_DATA' -popd + if [ -d temp/ ]; then + pushd temp/ + rm -r * + echo 'cleaned temp' + popd + fi -pushd logs/ -rm * -echo 'cleaned logs' -popd + if [ -d var/www/submitted/ ]; then + pushd var/www/submitted + rm -r * + echo 'cleaned submitted' + popd + fi -pushd PASTES/ -rm -r * -echo 'cleaned PASTES' -popd + echo -e $GREEN"* AIL has been reset *"$DEFAULT +} -echo -e $GREEN"* AIL has been reset *"$DEFAULT +function flush_DB_keep_user { + bash ${AIL_BIN}LAUNCH.sh -lav & + wait + echo "" + pushd redis/src + ./redis-cli -p 6382 -n 1 FLUSHDB; + ./redis-cli -p 6382 -n 2 FLUSHDB; + ./redis-cli -p 6382 -n 3 FLUSHDB; + ./redis-cli -p 6382 -n 4 FLUSHDB; + ./redis-cli -p 6382 -n 5 FLUSHDB; + ./redis-cli -p 6382 -n 6 FLUSHDB; + ./redis-cli -p 6382 -n 7 FLUSHDB; + ./redis-cli -p 6382 -n 8 FLUSHDB; + ./redis-cli -p 6382 -n 9 FLUSHDB; + echo "ARDB FLUSHED" + popd + bash ${AIL_BIN}LAUNCH.sh -k +} + +function soft_reset { + reset_dir; + flush_DB_keep_user; +} + +#If no params, +[[ $@ ]] || { + # Make sure the reseting is intentional + num=$(( ( RANDOM % 100 ) + 1 )) + + echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num + read userInput + + if [ $userInput -eq $num ] + then + echo "Reseting AIL..." + else + echo "Wrong number" + exit 1; + fi + + num=$(( ( RANDOM % 100 ) + 1 )) + echo -e $RED"If yes you want to delete the DB , enter the following number: "$DEFAULT $num + read userInput + + reset_dir; + + if [ $userInput -eq $num ] + then + if [ -d DATA_ARDB/ ]; then + pushd DATA_ARDB/ + rm -r * + echo 'cleaned DATA_ARDB' + popd + fi + fi + + echo -e $GREEN"* AIL has been reset *"$DEFAULT + + exit +} + +while [ "$1" != "" ]; do + case $1 in + --softReset ) soft_reset; + ;; + * ) exit 1 + esac + shift +done diff --git a/tests/testApi.py b/tests/testApi.py index ceac0185..4cabd2b8 100644 --- a/tests/testApi.py +++ b/tests/testApi.py @@ -23,12 +23,17 @@ def parse_response(obj, ail_response): return res_json def get_api_key(): - with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f: - content = f.read() - content = content.splitlines() - apikey = content[-1] - apikey = apikey.replace('API_Key=', '', 1) - return apikey + api_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD') + if os.path.isfile(api_file): + with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f: + content = f.read() + content = content.splitlines() + apikey = content[-1] + apikey = apikey.replace('API_Key=', '', 1) + # manual tests + else: + apikey = sys.argv[1] + return apikey APIKEY = get_api_key() @@ -162,4 +167,4 @@ class TestApiV1(unittest.TestCase): self.assertTrue(req_json['tags']) if __name__ == "__main__": - unittest.main() + unittest.main(argv=['first-arg-is-ignored'], exit=False) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 3d1b524e..4575ddb9 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -178,7 +178,8 @@ def add_header(response): and also to cache the rendered page for 10 minutes. """ response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1' - response.headers['Cache-Control'] = 'public, max-age=0' + if 'Cache-Control' not in response.headers: + response.headers['Cache-Control'] = 'private, max-age=0' return response # @app.route('/test', methods=['GET']) diff --git a/var/www/modules/Role_Manager.py b/var/www/modules/Role_Manager.py index dc4e7ce1..2fe5f59f 100644 --- a/var/www/modules/Role_Manager.py +++ b/var/www/modules/Role_Manager.py @@ -10,7 +10,7 @@ import configparser from functools import wraps from flask_login import LoginManager, current_user, login_user, logout_user, login_required -from flask import request, current_app +from flask import request, make_response, current_app login_manager = LoginManager() login_manager.login_view = 'role' @@ -36,6 +36,21 @@ default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD') regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,100}$' regex_password = re.compile(regex_password) +############################################################### +############### FLASK CACHE ################## +############################################################### +def no_cache(func): + @wraps(func) + def decorated_view(*args, **kwargs): + resp = make_response(func(*args, **kwargs)) + resp.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' + resp.headers['Pragma'] = 'no-cache' + return resp + return decorated_view +############################################################### +############################################################### +############################################################### + ############################################################### ############### CHECK ROLE ACCESS ################## ############################################################### diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 12fe3177..e3ee2bcb 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -13,7 +13,7 @@ import json from pyfaup.faup import Faup from flask import Flask, render_template, jsonify, request, send_file, Blueprint, redirect, url_for -from Role_Manager import login_admin, login_analyst +from Role_Manager import login_admin, login_analyst, no_cache from flask_login import login_required from Date import Date @@ -819,6 +819,7 @@ def show_domain(): @hiddenServices.route("/crawlers/download_domain", methods=['GET']) @login_required @login_analyst +@no_cache def download_domain(): domain = request.args.get('domain') epoch = request.args.get('epoch') diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index e57157fe..01b08a44 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -14,6 +14,8 @@ import redis import datetime import Import_helper +import Cryptocurrency +import Pgp import Item import Paste import Tag @@ -291,6 +293,7 @@ def get_item_content(): res = Item.get_item(req_data) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -341,6 +344,117 @@ def get_tracker_term_item(): res = Term.parse_get_tracker_term_item(data, user_id) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST']) +@token_required('analyst') +def get_cryptocurrency_bitcoin_metadata(): + data = request.get_json() + crypto_address = data.get('bitcoin', None) + req_data = {'bitcoin': crypto_address, 'metadata': True} + res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST']) +@token_required('analyst') +def get_cryptocurrency_bitcoin_item(): + data = request.get_json() + bitcoin_address = data.get('bitcoin', None) + req_data = {'bitcoin': bitcoin_address, 'items': True} + res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_key_metadata(): + data = request.get_json() + pgp_field = data.get('key', None) + req_data = {'key': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'key') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_mail_metadata(): + data = request.get_json() + pgp_field = data.get('mail', None) + req_data = {'mail': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'mail') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_name_metadata(): + data = request.get_json() + pgp_field = data.get('name', None) + req_data = {'name': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'name') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/key/item", methods=['POST']) +@token_required('analyst') +def get_pgp_key_item(): + data = request.get_json() + pgp_field = data.get('key', None) + req_data = {'key': pgp_field, 'items': True} + res = Pgp.get_pgp(req_data, 'key') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/mail/item", methods=['POST']) +@token_required('analyst') +def get_pgp_mail_item(): + data = request.get_json() + pgp_mail = data.get('mail', None) + req_data = {'mail': pgp_mail, 'items': True} + res = Pgp.get_pgp(req_data, 'mail') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/name/item", methods=['POST']) +@token_required('analyst') +def get_pgp_name_item(): + data = request.get_json() + pgp_name = data.get('name', None) + req_data = {'name': pgp_name, 'items': True} + res = Pgp.get_pgp(req_data, 'name') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +''' + + + +@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/item/pgp/mail", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/item/pgp/name", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] +''' + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index fb990bbf..10519d53 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -10,7 +10,7 @@ import os import flask from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for -from Role_Manager import login_admin, login_analyst +from Role_Manager import login_admin, login_analyst, no_cache from flask_login import login_required import difflib @@ -446,6 +446,7 @@ def showDiff(): @showsavedpastes.route('/screenshot/') @login_required @login_analyst +@no_cache def screenshot(filename): return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True)