diff --git a/bin/Crawler.py b/bin/Crawler.py
index fa0a796d..848d2b67 100755
--- a/bin/Crawler.py
+++ b/bin/Crawler.py
@@ -341,13 +341,27 @@ if __name__ == '__main__':
faup = Faup()
+ # get HAR files
+ default_crawler_har = p.config.getboolean("Crawler", "default_crawler_har")
+ if default_crawler_har:
+ default_crawler_har = 1
+ else:
+ default_crawler_har = 0
+
+ # get PNG files
+ default_crawler_png = p.config.getboolean("Crawler", "default_crawler_png")
+ if default_crawler_png:
+ default_crawler_png = 1
+ else:
+ default_crawler_png = 0
+
# Default crawler options
default_crawler_config = {'html': 1,
- 'har': 1,
- 'png': 1,
+ 'har': default_crawler_har,
+ 'png': default_crawler_png,
'depth_limit': p.config.getint("Crawler", "crawler_depth_limit"),
- 'closespider_pagecount': 50,
- 'user_agent': 'Mozilla/5.0 (Windows NT 6.1; rv:24.0) Gecko/20100101 Firefox/24.0'}
+ 'closespider_pagecount': p.config.getint("Crawler", "default_crawler_closespider_pagecount"),
+ 'user_agent': p.config.get("Crawler", "default_crawler_user_agent")}
# Track launched crawler
r_cache.sadd('all_crawler', splash_port)
diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py
new file mode 100755
index 00000000..b769600b
--- /dev/null
+++ b/bin/packages/Correlation.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+import Flask_config
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+
+class Correlation(object):
+
+ def __init__(self, correlation_name):
+ self.correlation_name = correlation_name
+
+ def _exist_corelation_field(self, correlation_type, field_name):
+ return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
+
+
+ def _get_items(self, correlation_type, field_name):
+ res = r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
+ if res:
+ return list(res)
+ else:
+ return []
+
+
+ def _get_metadata(self, correlation_type, field_name):
+ meta_dict = {}
+ meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
+ meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
+ return meta_dict
+
+ def _get_correlation_by_date(self, correlation_type, date):
+ return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date))
+
+ def verify_correlation_field_request(self, request_dict, correlation_type):
+ if not request_dict:
+ return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
+
+ field_name = request_dict.get(correlation_type, None)
+ if not field_name:
+ return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
+ if not self._exist_corelation_field(correlation_type, field_name):
+ return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
+
+ def get_correlation(self, request_dict, correlation_type, field_name):
+ dict_resp = {}
+
+ if request_dict.get('items'):
+ dict_resp['items'] = self._get_items(correlation_type, field_name)
+
+ if request_dict.get('metadata'):
+ dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
+
+ dict_resp[correlation_type] = field_name
+
+ return (dict_resp, 200)
+
+
+
+
+#cryptocurrency_all:cryptocurrency name cryptocurrency address nb seen
diff --git a/bin/packages/Cryptocurrency.py b/bin/packages/Cryptocurrency.py
new file mode 100755
index 00000000..1b4e4e3f
--- /dev/null
+++ b/bin/packages/Cryptocurrency.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+from hashlib import sha256
+
+import Flask_config
+from Correlation import Correlation
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+
+cryptocurrency = Correlation('cryptocurrency')
+
+def decode_base58(bc, length):
+ n = 0
+ for char in bc:
+ n = n * 58 + digits58.index(char)
+ return n.to_bytes(length, 'big')
+
+def check_bitcoin_address(bc):
+ try:
+ bcbytes = decode_base58(bc, 25)
+ return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
+ except Exception:
+ return False
+
+def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
+ if cryptocurrency_type == 'bitcoin':
+ return check_bitcoin_address(cryptocurrency_address)
+ else:
+ return True
+
+
+def get_cryptocurrency(request_dict, cryptocurrency_type):
+ # basic verification
+ res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type)
+ if res:
+ return res
+ # cerify address
+ field_name = request_dict.get(cryptocurrency_type)
+ if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
+ return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
+
+ return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
diff --git a/bin/packages/Item.py b/bin/packages/Item.py
index 4dcdde85..aa2b92c2 100755
--- a/bin/packages/Item.py
+++ b/bin/packages/Item.py
@@ -14,6 +14,7 @@ import Tag
PASTES_FOLDER = Flask_config.PASTES_FOLDER
r_cache = Flask_config.r_cache
+r_serv_metadata = Flask_config.r_serv_metadata
def exist_item(item_id):
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
@@ -93,4 +94,43 @@ def get_item(request_dict):
if lines_info:
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
+ if request_dict.get('pgp'):
+ dict_item['pgp'] = {}
+ if request_dict['pgp'].get('key'):
+ dict_item['pgp']['key'] = get_item_pgp_key(item_id)
+ if request_dict['pgp'].get('mail'):
+ dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
+ if request_dict['pgp'].get('name'):
+ dict_item['pgp']['name'] = get_item_pgp_name(item_id)
+
+ if request_dict.get('cryptocurrency'):
+ dict_item['cryptocurrency'] = {}
+ if request_dict['cryptocurrency'].get('bitcoin'):
+ dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
+
return (dict_item, 200)
+
+
+###
+### correlation
+###
+
+def _get_item_correlation(correlation_name, correlation_type, item_id):
+ print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
+ res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
+ if res:
+ return list(res)
+ else:
+ return []
+
+def get_item_bitcoin(item_id):
+ return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
+
+def get_item_pgp_key(item_id):
+ return _get_item_correlation('pgpdump', 'key', item_id)
+
+def get_item_pgp_name(item_id):
+ return _get_item_correlation('pgpdump', 'name', item_id)
+
+def get_item_pgp_mail(item_id):
+ return _get_item_correlation('pgpdump', 'mail', item_id)
diff --git a/bin/packages/Pgp.py b/bin/packages/Pgp.py
new file mode 100755
index 00000000..9c7b0ec4
--- /dev/null
+++ b/bin/packages/Pgp.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+from hashlib import sha256
+
+import Flask_config
+from Correlation import Correlation
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+pgpdump = Correlation('pgpdump')
+
+
+def get_pgp(request_dict, pgp_type):
+ # basic verification
+ res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
+ if res:
+ return res
+ # cerify address
+ field_name = request_dict.get(pgp_type)
+
+ return pgpdump.get_correlation(request_dict, pgp_type, field_name)
diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample
index 52388ed5..0d32293b 100644
--- a/bin/packages/config.cfg.sample
+++ b/bin/packages/config.cfg.sample
@@ -255,5 +255,9 @@ db = 0
[Crawler]
activate_crawler = False
crawler_depth_limit = 1
+default_crawler_har = True
+default_crawler_png = True
+default_crawler_closespider_pagecount = 50
+default_crawler_user_agent = Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0
splash_url = http://127.0.0.1
splash_port = 8050-8052
diff --git a/doc/README.md b/doc/README.md
index 03a1aa07..8750c475 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -250,6 +250,25 @@ Get item. Filter requested field.
- get item lines info
- *boolean*
- default: `false`
+- `cryptocurrency`
+ - `bitcoin`
+ - get item bitcoin adress
+ - *boolean*
+ - default: `false`
+- `pgp`
+ - `key`
+ - get item pgp key
+ - *boolean*
+ - default: `false`
+ - `mail`
+ - get item pgp mail
+ - *boolean*
+ - default: `false`
+ - `name`
+ - get item pgp name
+ - *boolean*
+ - default: `false`
+
#### JSON response
- `content`
@@ -276,6 +295,20 @@ Get item. Filter requested field.
- `nb`
- nb lines item
- *int*
+- `cryptocurrency`
+ - `bitcoin`
+ - item bitcoin adress
+ - *list*
+- `pgp`
+ - `key`
+ - item pgp keys
+ - *list*
+ - `mail`
+ - item pgp mails
+ - *list*
+ - `name`
+ - item pgp name
+ - *list*
#### Example
@@ -299,12 +332,28 @@ curl https://127.0.0.1:7000/api/v1/get/item --header "Authorization: iHc1_ChZxj1
```json
{
"content": "dsvcdsvcdsc vvvv",
+ "cryptocurrency": {
+ "bitcoin": [
+ "132M1aGTGodHkQNh1augLeMjEXH51wgoCc"
+ ]
+ },
"date": "20190726",
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
"lines": {
"max_length": 19,
"nb": 1
},
+ "pgp": {
+ "key": [
+ "0x5180D21F4C20F975"
+ ],
+ "mail": [
+ "mail@test.test"
+ ],
+ "name": [
+ "user_test"
+ ]
+ },
"size": 0.03,
"tags": [
"misp-galaxy:stealer=\"Vidar\"",
@@ -546,10 +595,10 @@ Get tag metadata.
- *str*
- `first_seen`
- date: first seen
- - *str - YYMMDD*
+ - *str - YYYYMMDD*
- `last_seen`
- - date: first seen
- - *str - YYMMDD*
+ - date: last seen
+ - *str - YYYYMMDD*
#### Example
```
curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
@@ -581,6 +630,116 @@ curl https://127.0.0.1:7000/api/v1/get/tag/metadata --header "Authorization: iHc
+## Cryptocurrency
+
+
+
+### Get bitcoin metadata: `api/v1/get/cryptocurrency/bitcoin/metadata`
+
+#### Description
+Get all metdata from a bitcoin address.
+
+**Method** : `POST`
+
+#### Parameters
+- `bitcoin`
+ - bitcoin address
+ - *str*
+ - mandatory
+
+#### JSON response
+- `bitcoin`
+ - bitcoin address
+ - *str*
+- `first_seen`
+ - date: first seen
+ - *str - YYYYMMDD*
+- `last_seen`
+ - date: last seen
+ - *str - YYYYMMDD*
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/cryptocurrency/bitcoin/metadata --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+ {
+ "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y"
+ }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+```json
+ {
+ "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y",
+ "first_seen": "20190605",
+ "last_seen": "20190726"
+ }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `404`
+```json
+ {"status": "error", "reason": "Item not found"}
+```
+
+
+
+### Get bitcoin metadata: `api/v1/get/cryptocurrency/bitcoin/item`
+
+#### Description
+Get all items related to a bitcoin address.
+
+**Method** : `POST`
+
+#### Parameters
+- `bitcoin`
+ - bitcoin address
+ - *str*
+ - mandatory
+
+#### JSON response
+- `bitcoin`
+ - bitcoin address
+ - *str*
+- `items`
+ - list of item id
+ - *list*
+#### Example
+```
+curl https://127.0.0.1:7000/api/v1/get/cryptocurrency/bitcoin/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+ {
+ "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y"
+ }
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+```json
+ {
+ "bitcoin": "3DZfm5TQaJKcJm9PsuaWmSz9XmHMLxVv3y",
+ "items": [
+ "archive/2019/08/26/test_bitcoin001",
+ "archive/2019/08/26/test_bitcoin002",
+ "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz"
+ ]
+ }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `404`
+```json
+ {"status": "error", "reason": "Item not found"}
+```
+
+
+
@@ -777,9 +936,6 @@ curl https://127.0.0.1:7000/api/v1/add/tracker/term --header "Authorization: iHc
-
-
-
## Import management
diff --git a/ansible/.gitignore b/other_installers/ansible/.gitignore
similarity index 100%
rename from ansible/.gitignore
rename to other_installers/ansible/.gitignore
diff --git a/ansible/Dockerfile.testing b/other_installers/ansible/Dockerfile.testing
similarity index 100%
rename from ansible/Dockerfile.testing
rename to other_installers/ansible/Dockerfile.testing
diff --git a/ansible/README.md b/other_installers/ansible/README.md
similarity index 100%
rename from ansible/README.md
rename to other_installers/ansible/README.md
diff --git a/ansible/deploy.sh b/other_installers/ansible/deploy.sh
similarity index 100%
rename from ansible/deploy.sh
rename to other_installers/ansible/deploy.sh
diff --git a/ansible/deploy.yml b/other_installers/ansible/deploy.yml
similarity index 100%
rename from ansible/deploy.yml
rename to other_installers/ansible/deploy.yml
diff --git a/ansible/deployLocal.yml b/other_installers/ansible/deployLocal.yml
similarity index 100%
rename from ansible/deployLocal.yml
rename to other_installers/ansible/deployLocal.yml
diff --git a/ansible/group_vars/ail.yml b/other_installers/ansible/group_vars/ail.yml
similarity index 100%
rename from ansible/group_vars/ail.yml
rename to other_installers/ansible/group_vars/ail.yml
diff --git a/ansible/roles/ail-host/files/ail-flask.service b/other_installers/ansible/roles/ail-host/files/ail-flask.service
similarity index 100%
rename from ansible/roles/ail-host/files/ail-flask.service
rename to other_installers/ansible/roles/ail-host/files/ail-flask.service
diff --git a/ansible/roles/ail-host/files/ail-leveldb.service b/other_installers/ansible/roles/ail-host/files/ail-leveldb.service
similarity index 100%
rename from ansible/roles/ail-host/files/ail-leveldb.service
rename to other_installers/ansible/roles/ail-host/files/ail-leveldb.service
diff --git a/ansible/roles/ail-host/files/ail-logging.service b/other_installers/ansible/roles/ail-host/files/ail-logging.service
similarity index 100%
rename from ansible/roles/ail-host/files/ail-logging.service
rename to other_installers/ansible/roles/ail-host/files/ail-logging.service
diff --git a/ansible/roles/ail-host/files/ail-queues.service b/other_installers/ansible/roles/ail-host/files/ail-queues.service
similarity index 100%
rename from ansible/roles/ail-host/files/ail-queues.service
rename to other_installers/ansible/roles/ail-host/files/ail-queues.service
diff --git a/ansible/roles/ail-host/files/ail-redis.service b/other_installers/ansible/roles/ail-host/files/ail-redis.service
similarity index 100%
rename from ansible/roles/ail-host/files/ail-redis.service
rename to other_installers/ansible/roles/ail-host/files/ail-redis.service
diff --git a/ansible/roles/ail-host/files/ail-scripts.service b/other_installers/ansible/roles/ail-host/files/ail-scripts.service
similarity index 100%
rename from ansible/roles/ail-host/files/ail-scripts.service
rename to other_installers/ansible/roles/ail-host/files/ail-scripts.service
diff --git a/ansible/roles/ail-host/files/startFlask.sh b/other_installers/ansible/roles/ail-host/files/startFlask.sh
similarity index 100%
rename from ansible/roles/ail-host/files/startFlask.sh
rename to other_installers/ansible/roles/ail-host/files/startFlask.sh
diff --git a/ansible/roles/ail-host/files/startLogging.sh b/other_installers/ansible/roles/ail-host/files/startLogging.sh
similarity index 100%
rename from ansible/roles/ail-host/files/startLogging.sh
rename to other_installers/ansible/roles/ail-host/files/startLogging.sh
diff --git a/ansible/roles/ail-host/files/startQueues.sh b/other_installers/ansible/roles/ail-host/files/startQueues.sh
similarity index 100%
rename from ansible/roles/ail-host/files/startQueues.sh
rename to other_installers/ansible/roles/ail-host/files/startQueues.sh
diff --git a/ansible/roles/ail-host/files/startRedis.sh b/other_installers/ansible/roles/ail-host/files/startRedis.sh
similarity index 100%
rename from ansible/roles/ail-host/files/startRedis.sh
rename to other_installers/ansible/roles/ail-host/files/startRedis.sh
diff --git a/ansible/roles/ail-host/files/startRedisLevelDB.sh b/other_installers/ansible/roles/ail-host/files/startRedisLevelDB.sh
similarity index 100%
rename from ansible/roles/ail-host/files/startRedisLevelDB.sh
rename to other_installers/ansible/roles/ail-host/files/startRedisLevelDB.sh
diff --git a/ansible/roles/ail-host/files/startScripts.sh b/other_installers/ansible/roles/ail-host/files/startScripts.sh
similarity index 100%
rename from ansible/roles/ail-host/files/startScripts.sh
rename to other_installers/ansible/roles/ail-host/files/startScripts.sh
diff --git a/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.min.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.min.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.min.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap-rtl.min.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/bootstrap.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/bootstrap.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/bootstrap.min.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap.min.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/bootstrap.min.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/bootstrap.min.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/dataTables.bootstrap.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/dataTables.bootstrap.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/dataTables.bootstrap.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/dataTables.bootstrap.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/jquery-ui.min.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/jquery-ui.min.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/jquery-ui.min.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/jquery-ui.min.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/plugins/morris.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/sb-admin-2.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-2.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/sb-admin-2.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-2.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin-rtl.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/sb-admin.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/sb-admin.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/sb-admin.css
diff --git a/ansible/roles/ail-host/files/staticBackup/css/timeline.css b/other_installers/ansible/roles/ail-host/files/staticBackup/css/timeline.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/css/timeline.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/css/timeline.css
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.css b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.css
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.min.css b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.min.css
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.min.css
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/css/font-awesome.min.css
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/FontAwesome.otf
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.eot
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.svg b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.svg
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.svg
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.svg
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.ttf
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/fonts/fontawesome-webfont.woff
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/bordered-pulled.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/core.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/fixed-width.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/font-awesome.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/icons.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/icons.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/icons.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/icons.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/larger.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/list.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/mixins.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/path.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/rotated-flipped.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/spinning.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/stacked.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/less/variables.less b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/variables.less
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/less/variables.less
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/less/variables.less
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_bordered-pulled.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_core.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_fixed-width.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_icons.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_icons.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_icons.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_icons.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_larger.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_list.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_mixins.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_path.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_rotated-flipped.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_spinning.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_stacked.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_variables.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_variables.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_variables.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/_variables.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss b/other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss
rename to other_installers/ansible/roles/ail-host/files/staticBackup/font-awesome/scss/font-awesome.scss
diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot
rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.eot
diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.svg b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.svg
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.svg
rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.svg
diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf
rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.ttf
diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff
rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff
diff --git a/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2 b/other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2
rename to other_installers/ansible/roles/ail-host/files/staticBackup/fonts/glyphicons-halflings-regular.woff2
diff --git a/ansible/roles/ail-host/files/staticBackup/image/AIL.png b/other_installers/ansible/roles/ail-host/files/staticBackup/image/AIL.png
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/image/AIL.png
rename to other_installers/ansible/roles/ail-host/files/staticBackup/image/AIL.png
diff --git a/ansible/roles/ail-host/files/staticBackup/js/bootstrap.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/bootstrap.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/bootstrap.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/bootstrap.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/bootstrap.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/bootstrap.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/bootstrap.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/bootstrap.min.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/dataTables.bootstrap.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/dataTables.bootstrap.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/dataTables.bootstrap.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/dataTables.bootstrap.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/dygraph-combined.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/dygraph-combined.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/dygraph-combined.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/dygraph-combined.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery-ui.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery-ui.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery-ui.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery-ui.min.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.canvasjs.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.canvasjs.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery.canvasjs.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.canvasjs.min.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.dataTables.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.dataTables.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery.dataTables.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.dataTables.min.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery.flot.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.pie.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.pie.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery.flot.pie.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.pie.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.stack.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.stack.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery.flot.stack.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.stack.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.time.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.time.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery.flot.time.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.flot.time.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/jquery.sparkline.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.sparkline.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/jquery.sparkline.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/jquery.sparkline.min.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/excanvas.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/excanvas.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/excanvas.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/excanvas.min.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/flot-data.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/flot-data.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/flot-data.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/flot-data.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.pie.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.pie.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.pie.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.pie.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.resize.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.resize.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.resize.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.resize.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.tooltip.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.tooltip.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.tooltip.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/flot/jquery.flot.tooltip.min.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris-data.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris-data.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris-data.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris-data.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/morris.min.js
diff --git a/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/raphael.min.js b/other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/raphael.min.js
similarity index 100%
rename from ansible/roles/ail-host/files/staticBackup/js/plugins/morris/raphael.min.js
rename to other_installers/ansible/roles/ail-host/files/staticBackup/js/plugins/morris/raphael.min.js
diff --git a/ansible/roles/ail-host/tasks/main.yml b/other_installers/ansible/roles/ail-host/tasks/main.yml
similarity index 100%
rename from ansible/roles/ail-host/tasks/main.yml
rename to other_installers/ansible/roles/ail-host/tasks/main.yml
diff --git a/Dockerfile b/other_installers/docker/Dockerfile
similarity index 100%
rename from Dockerfile
rename to other_installers/docker/Dockerfile
diff --git a/doc/X-docker.md b/other_installers/docker/README.md
similarity index 100%
rename from doc/X-docker.md
rename to other_installers/docker/README.md
diff --git a/docker-compose.yml b/other_installers/docker/docker-compose.yml
similarity index 100%
rename from docker-compose.yml
rename to other_installers/docker/docker-compose.yml
diff --git a/docker_start.sh b/other_installers/docker/docker_start.sh
similarity index 100%
rename from docker_start.sh
rename to other_installers/docker/docker_start.sh
diff --git a/pystemon/config.cfg b/other_installers/docker/pystemon/config.cfg
similarity index 100%
rename from pystemon/config.cfg
rename to other_installers/docker/pystemon/config.cfg
diff --git a/pystemon/install.sh b/other_installers/docker/pystemon/install.sh
similarity index 100%
rename from pystemon/install.sh
rename to other_installers/docker/pystemon/install.sh
diff --git a/pystemon/proxies.txt b/other_installers/docker/pystemon/proxies.txt
similarity index 100%
rename from pystemon/proxies.txt
rename to other_installers/docker/pystemon/proxies.txt
diff --git a/pystemon/pystemon.yaml b/other_installers/docker/pystemon/pystemon.yaml
similarity index 100%
rename from pystemon/pystemon.yaml
rename to other_installers/docker/pystemon/pystemon.yaml
diff --git a/requirements.txt b/requirements.txt
index fdccf9bb..dbebad0f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -57,7 +57,7 @@ whoosh
beautifulsoup4
ipaddress
-pycountry
+pycountry==18.12.8
# To fetch Onion urls
PySocks
diff --git a/reset_AIL.sh b/reset_AIL.sh
index e87dfb67..aac94352 100755
--- a/reset_AIL.sh
+++ b/reset_AIL.sh
@@ -6,52 +6,144 @@ GREEN="\\033[1;32m"
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
-# Make sure the reseting is intentional
-num=$(( ( RANDOM % 100 ) + 1 ))
+function reset_dir {
+ # Access dirs and delete
+ cd $AIL_HOME
-echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num
-read userInput
+ # Kill all screens
+ screen -ls | grep Detached | cut -d. -f1 | awk '{print $1}' | xargs kill
-if [ $userInput -eq $num ]
-then
- echo "Reseting AIL..."
-else
- echo "Wrong number"
- exit 1;
-fi
+ # Access dirs and delete
+ cd $AIL_HOME
+ if [ -d indexdir/ ]; then
+ pushd indexdir/
+ rm -r *
+ echo 'cleaned indexdir'
+ popd
+ fi
-# Kill all screens
-screen -ls | grep Detached | cut -d. -f1 | awk '{print $1}' | xargs kill
+ if [ $userInput -eq $num ]
+ then
+ if [ -d DATA_ARDB/ ]; then
+ pushd DATA_ARDB/
+ rm -r *
+ echo 'cleaned DATA_ARDB'
+ popd
+ fi
+ fi
-set -e
+ if [ -d logs/ ]; then
+ pushd logs/
+ rm *
+ echo 'cleaned logs'
+ popd
+ fi
-# Access dirs and delete
-cd $AIL_HOME
+ if [ -d PASTES/ ]; then
+ pushd PASTES/
+ rm -r *
+ echo 'cleaned PASTES'
+ popd
+ fi
-pushd dumps/
-rm *
-echo 'cleaned dumps'
-popd
+ if [ -d HASHS/ ]; then
+ pushd HASHS/
+ rm -r *
+ echo 'cleaned HASHS'
+ popd
+ fi
-pushd indexdir/
-rm -r *
-echo 'cleaned indexdir'
-popd
+ if [ -d CRAWLED_SCREESHOT/ ]; then
+ pushd CRAWLED_SCREESHOT/
+ rm -r *
+ echo 'cleaned CRAWLED_SCREESHOT'
+ popd
+ fi
-pushd LEVEL_DB_DATA/
-rm -r *
-echo 'cleaned LEVEL_DB_DATA'
-popd
+ if [ -d temp/ ]; then
+ pushd temp/
+ rm -r *
+ echo 'cleaned temp'
+ popd
+ fi
-pushd logs/
-rm *
-echo 'cleaned logs'
-popd
+ if [ -d var/www/submitted/ ]; then
+ pushd var/www/submitted
+ rm -r *
+ echo 'cleaned submitted'
+ popd
+ fi
-pushd PASTES/
-rm -r *
-echo 'cleaned PASTES'
-popd
+ echo -e $GREEN"* AIL has been reset *"$DEFAULT
+}
-echo -e $GREEN"* AIL has been reset *"$DEFAULT
+function flush_DB_keep_user {
+ bash ${AIL_BIN}LAUNCH.sh -lav &
+ wait
+ echo ""
+ pushd redis/src
+ ./redis-cli -p 6382 -n 1 FLUSHDB;
+ ./redis-cli -p 6382 -n 2 FLUSHDB;
+ ./redis-cli -p 6382 -n 3 FLUSHDB;
+ ./redis-cli -p 6382 -n 4 FLUSHDB;
+ ./redis-cli -p 6382 -n 5 FLUSHDB;
+ ./redis-cli -p 6382 -n 6 FLUSHDB;
+ ./redis-cli -p 6382 -n 7 FLUSHDB;
+ ./redis-cli -p 6382 -n 8 FLUSHDB;
+ ./redis-cli -p 6382 -n 9 FLUSHDB;
+ echo "ARDB FLUSHED"
+ popd
+ bash ${AIL_BIN}LAUNCH.sh -k
+}
+
+function soft_reset {
+ reset_dir;
+ flush_DB_keep_user;
+}
+
+#If no params,
+[[ $@ ]] || {
+ # Make sure the reseting is intentional
+ num=$(( ( RANDOM % 100 ) + 1 ))
+
+ echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num
+ read userInput
+
+ if [ $userInput -eq $num ]
+ then
+ echo "Reseting AIL..."
+ else
+ echo "Wrong number"
+ exit 1;
+ fi
+
+ num=$(( ( RANDOM % 100 ) + 1 ))
+ echo -e $RED"If yes you want to delete the DB , enter the following number: "$DEFAULT $num
+ read userInput
+
+ reset_dir;
+
+ if [ $userInput -eq $num ]
+ then
+ if [ -d DATA_ARDB/ ]; then
+ pushd DATA_ARDB/
+ rm -r *
+ echo 'cleaned DATA_ARDB'
+ popd
+ fi
+ fi
+
+ echo -e $GREEN"* AIL has been reset *"$DEFAULT
+
+ exit
+}
+
+while [ "$1" != "" ]; do
+ case $1 in
+ --softReset ) soft_reset;
+ ;;
+ * ) exit 1
+ esac
+ shift
+done
diff --git a/tests/testApi.py b/tests/testApi.py
index ceac0185..4cabd2b8 100644
--- a/tests/testApi.py
+++ b/tests/testApi.py
@@ -23,12 +23,17 @@ def parse_response(obj, ail_response):
return res_json
def get_api_key():
- with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f:
- content = f.read()
- content = content.splitlines()
- apikey = content[-1]
- apikey = apikey.replace('API_Key=', '', 1)
- return apikey
+ api_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
+ if os.path.isfile(api_file):
+ with open(os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD'), 'r') as f:
+ content = f.read()
+ content = content.splitlines()
+ apikey = content[-1]
+ apikey = apikey.replace('API_Key=', '', 1)
+ # manual tests
+ else:
+ apikey = sys.argv[1]
+ return apikey
APIKEY = get_api_key()
@@ -162,4 +167,4 @@ class TestApiV1(unittest.TestCase):
self.assertTrue(req_json['tags'])
if __name__ == "__main__":
- unittest.main()
+ unittest.main(argv=['first-arg-is-ignored'], exit=False)
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 3d1b524e..4575ddb9 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -178,7 +178,8 @@ def add_header(response):
and also to cache the rendered page for 10 minutes.
"""
response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1'
- response.headers['Cache-Control'] = 'public, max-age=0'
+ if 'Cache-Control' not in response.headers:
+ response.headers['Cache-Control'] = 'private, max-age=0'
return response
# @app.route('/test', methods=['GET'])
diff --git a/var/www/modules/Role_Manager.py b/var/www/modules/Role_Manager.py
index dc4e7ce1..2fe5f59f 100644
--- a/var/www/modules/Role_Manager.py
+++ b/var/www/modules/Role_Manager.py
@@ -10,7 +10,7 @@ import configparser
from functools import wraps
from flask_login import LoginManager, current_user, login_user, logout_user, login_required
-from flask import request, current_app
+from flask import request, make_response, current_app
login_manager = LoginManager()
login_manager.login_view = 'role'
@@ -36,6 +36,21 @@ default_passwd_file = os.path.join(os.environ['AIL_HOME'], 'DEFAULT_PASSWORD')
regex_password = r'^(?=(.*\d){2})(?=.*[a-z])(?=.*[A-Z]).{10,100}$'
regex_password = re.compile(regex_password)
+###############################################################
+############### FLASK CACHE ##################
+###############################################################
+def no_cache(func):
+ @wraps(func)
+ def decorated_view(*args, **kwargs):
+ resp = make_response(func(*args, **kwargs))
+ resp.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+ resp.headers['Pragma'] = 'no-cache'
+ return resp
+ return decorated_view
+###############################################################
+###############################################################
+###############################################################
+
###############################################################
############### CHECK ROLE ACCESS ##################
###############################################################
diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py
index 12fe3177..e3ee2bcb 100644
--- a/var/www/modules/hiddenServices/Flask_hiddenServices.py
+++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py
@@ -13,7 +13,7 @@ import json
from pyfaup.faup import Faup
from flask import Flask, render_template, jsonify, request, send_file, Blueprint, redirect, url_for
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, no_cache
from flask_login import login_required
from Date import Date
@@ -819,6 +819,7 @@ def show_domain():
@hiddenServices.route("/crawlers/download_domain", methods=['GET'])
@login_required
@login_analyst
+@no_cache
def download_domain():
domain = request.args.get('domain')
epoch = request.args.get('epoch')
diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py
index e57157fe..01b08a44 100644
--- a/var/www/modules/restApi/Flask_restApi.py
+++ b/var/www/modules/restApi/Flask_restApi.py
@@ -14,6 +14,8 @@ import redis
import datetime
import Import_helper
+import Cryptocurrency
+import Pgp
import Item
import Paste
import Tag
@@ -291,6 +293,7 @@ def get_item_content():
res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@@ -341,6 +344,117 @@ def get_tracker_term_item():
res = Term.parse_get_tracker_term_item(data, user_id)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
+@token_required('analyst')
+def get_cryptocurrency_bitcoin_metadata():
+ data = request.get_json()
+ crypto_address = data.get('bitcoin', None)
+ req_data = {'bitcoin': crypto_address, 'metadata': True}
+ res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
+@token_required('analyst')
+def get_cryptocurrency_bitcoin_item():
+ data = request.get_json()
+ bitcoin_address = data.get('bitcoin', None)
+ req_data = {'bitcoin': bitcoin_address, 'items': True}
+ res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
+@token_required('analyst')
+def get_pgp_key_metadata():
+ data = request.get_json()
+ pgp_field = data.get('key', None)
+ req_data = {'key': pgp_field, 'metadata': True}
+ res = Pgp.get_pgp(req_data, 'key')
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
+@token_required('analyst')
+def get_pgp_mail_metadata():
+ data = request.get_json()
+ pgp_field = data.get('mail', None)
+ req_data = {'mail': pgp_field, 'metadata': True}
+ res = Pgp.get_pgp(req_data, 'mail')
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
+@token_required('analyst')
+def get_pgp_name_metadata():
+ data = request.get_json()
+ pgp_field = data.get('name', None)
+ req_data = {'name': pgp_field, 'metadata': True}
+ res = Pgp.get_pgp(req_data, 'name')
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
+@token_required('analyst')
+def get_pgp_key_item():
+ data = request.get_json()
+ pgp_field = data.get('key', None)
+ req_data = {'key': pgp_field, 'items': True}
+ res = Pgp.get_pgp(req_data, 'key')
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
+@token_required('analyst')
+def get_pgp_mail_item():
+ data = request.get_json()
+ pgp_mail = data.get('mail', None)
+ req_data = {'mail': pgp_mail, 'items': True}
+ res = Pgp.get_pgp(req_data, 'mail')
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
+@token_required('analyst')
+def get_pgp_name_item():
+ data = request.get_json()
+ pgp_name = data.get('name', None)
+ req_data = {'name': pgp_name, 'items': True}
+ res = Pgp.get_pgp(req_data, 'name')
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+'''
+
+
+
+@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST'])
+@token_required('analyst')
+def get_item_cryptocurrency_bitcoin():
+ data = request.get_json()
+ item_id = data.get('id', None)
+ req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}}
+ res = Item.get_item(req_data)
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/item/pgp/mail", methods=['POST'])
+@token_required('analyst')
+def get_item_cryptocurrency_bitcoin():
+ data = request.get_json()
+ item_id = data.get('id', None)
+ req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}}
+ res = Item.get_item(req_data)
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/item/pgp/name", methods=['POST'])
+@token_required('analyst')
+def get_item_cryptocurrency_bitcoin():
+ data = request.get_json()
+ item_id = data.get('id', None)
+ req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}}
+ res = Item.get_item(req_data)
+ return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+'''
+
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py
index fb990bbf..10519d53 100644
--- a/var/www/modules/showpaste/Flask_showpaste.py
+++ b/var/www/modules/showpaste/Flask_showpaste.py
@@ -10,7 +10,7 @@ import os
import flask
from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for
-from Role_Manager import login_admin, login_analyst
+from Role_Manager import login_admin, login_analyst, no_cache
from flask_login import login_required
import difflib
@@ -446,6 +446,7 @@ def showDiff():
@showsavedpastes.route('/screenshot/')
@login_required
@login_analyst
+@no_cache
def screenshot(filename):
return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True)