chg: [config] save config on DB + default values

bin/lib/Config_DB.py

@@ -0,0 +1,155 @@
+#!/usr/bin/python3
+
+"""
+Config save in DB
+===================
+
+
+"""
+
+import os
+import sys
+import redis
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import ConfigLoader
+
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+config_loader = None
+
+#### TO PUT IN CONFIG
+# later => module timeout
+#
+## data retention
+#########################
+
+default_config = {
+    "crawler": {
+        "enable_har_by_default": False,
+        "enable_screenshot_by_default": True,
+        "default_depth_limit": 1,
+        "default_closespider_pagecount": 50,
+        "default_user_agent": "Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0",
+        "default_timeout": 30
+    }
+}
+
+def get_default_config():
+    return default_config
+
+def get_default_config_value(section, field):
+    return default_config[section][field]
+
+config_type = {
+    # crawler config
+    "crawler": {
+        "enable_har_by_default": bool,
+        "enable_screenshot_by_default": bool,
+        "default_depth_limit": int,
+        "default_closespider_pagecount": int,
+        "default_user_agent": str,
+        "default_timeout": int
+    }
+}
+
+def get_config_type(section, field):
+    return config_type[section][field]
+
+# # TODO: add set, dict, list and select_(multiple_)value
+def is_valid_type(obj, section, field, value_type=None):
+    res = isinstance(obj, get_config_type(section, field))
+    return res
+
+def reset_default_config():
+    pass
+
+def set_default_config(section, field):
+    save_config(section, field, get_default_config_value(section, field))
+
+def get_all_config_sections():
+    return list(get_default_config())
+
+def get_all_config_fields_by_section(section):
+    return list(get_default_config()[section])
+
+def get_config(section, field):
+    # config field don't exist
+    if not r_serv_db.hexists(f'config:global:{section}', field):
+        set_default_config(section, field)
+        return get_default_config_value(section, field)
+
+    # load default config section
+    if not r_serv_db.exists('config:global:{}'.format(section)):
+        save_config(section, field, get_default_config_value(section, field))
+        return get_default_config_value(section, field)
+
+    return r_serv_db.hget(f'config:global:{section}', field)
+
+def get_config_dict_by_section(section):
+    config_dict = {}
+    for field in get_all_config_fields_by_section(section):
+        config_dict[field] = get_config(section, field)
+    return config_dict
+
+def save_config(section, field, value, value_type=None): ###########################################
+    if section in default_config:
+        if is_valid_type(value, section, field, value_type=value_type):
+            if value_type in ['list', 'set', 'dict']:
+                pass
+            else:
+                r_serv_db.hset(f'config:global:{section}', field, value)
+                # used by check_integrity
+                r_serv_db.sadd('config:all_global_section', field, value)
+
+# check config value + type
+def check_integrity():
+    pass
+
+
+config_documentation = {
+    "crawler": {
+        "enable_har_by_default": 'Enable HAR by default',
+        "enable_screenshot_by_default": 'Enable screenshot by default',
+        "default_depth_limit": 'Maximum number of url depth',
+        "default_closespider_pagecount": 'Maximum number of pages',
+        "default_user_agent": "User agent used by default",
+        "default_timeout": "Crawler connection timeout"
+    }
+}
+
+def get_config_documentation(section, field):
+    return config_documentation[section][field]
+
+# def conf_view():
+#     class F(MyBaseForm):
+#         pass
+#
+#     F.username = TextField('username')
+#     for name in iterate_some_model_dynamically():
+#         setattr(F, name, TextField(name.title()))
+#
+#     form = F(request.POST, ...)
+
+def get_field_full_config(section, field):
+    dict_config = {}
+    dict_config['value'] = get_config(section, field)
+    dict_config['type'] = get_config_type(section, field)
+    dict_config['info'] = get_config_documentation(section, field)
+    return dict_config
+
+def get_full_config_by_section(section):
+    dict_config = {}
+    for field in get_all_config_fields_by_section(section):
+        dict_config[field] = get_field_full_config(section, field)
+    return dict_config
+
+def get_full_config():
+    dict_config = {}
+    for section in get_all_config_sections():
+        dict_config[section] = get_full_config_by_section(section)
+    return dict_config
+
+if __name__ == '__main__':
+    res = get_full_config()
+    print(res)

var/www/blueprints/crawler_splash.py

@@ -26,6 +26,7 @@ import Tag
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Domain
 import crawlers
+import Config_DB
 
 r_cache = Flask_config.r_cache
 r_serv_db = Flask_config.r_serv_db
@@ -438,10 +439,12 @@ def crawler_splash_setings():
     splash_manager_url = crawlers.get_splash_manager_url()
     api_key = crawlers.get_hidden_splash_api_key()
     is_manager_connected = crawlers.ping_splash_manager()
+    crawler_full_config = Config_DB.get_full_config_by_section('crawler')
 
     return render_template("settings_splash_crawler.html",
                                 is_manager_connected=is_manager_connected,
                                 splash_manager_url=splash_manager_url, api_key=api_key,
-                                all_splash=all_splash, all_proxies=all_proxies)
+                                all_splash=all_splash, all_proxies=all_proxies,
+                                crawler_full_config=crawler_full_config)
 
 ##  - -  ##

var/www/templates/crawler/crawler_splash/settings_splash_crawler.html

@@ -102,6 +102,7 @@
 										<th>
 											Description
 										</th>
+										<th></th>
 									</thead>
 									<tbody>
 										{% for splash_name in all_splash %}
@@ -123,6 +124,13 @@
 												<td>
 													{{all_splash[splash_name]['description']}}
 												</td>
+												<td>
+													<div class="d-flex justify-content-end">
+														<button class="btn btn-outline-dark px-1 py-0">
+															<i class="fas fa-pencil-alt"></i>
+														</button>
+													</div>
+												</td>
 											</tr>
 										{% endfor %}
 									</tbody>
@@ -153,6 +161,7 @@
 										<th>
 											Description
 										</th>
+										<th></th>
 									</thead>
 									<tbody>
 										{% for proxy_name in all_proxies %}
@@ -180,6 +189,13 @@
 												<td>
 													{{all_proxies[proxy_name]['description']}}
 												</td>
+												<td>
+													<div class="d-flex justify-content-end">
+														<button class="btn btn-outline-dark px-1 py-0">
+															<i class="fas fa-pencil-alt"></i>
+														</button>
+													</div>
+												</td>
 											</tr>
 										{% endfor %}
 									</tbody>
@@ -196,6 +212,43 @@
 					</div>
 					<div class="card-body">
 
+						<table class="table table-striped table-hover">
+							<thead class="bg-info text-white">
+								<th>
+									Key
+								</th>
+								<th>
+									Description
+								</th>
+								<th>
+									Value
+								</th>
+								<th></th>
+							</thead>
+							<tbody>
+								{% for config_field in crawler_full_config %}
+									<tr>
+										<td>
+											{{config_field}}
+										</td>
+										<td>
+											{{crawler_full_config[config_field]['info']}}
+										</td>
+										<td>
+											{{crawler_full_config[config_field]['value']}}
+										</td>
+										<td>
+											<div class="d-flex justify-content-end">
+												<button class="btn btn-outline-dark px-1 py-0">
+													<i class="fas fa-pencil-alt"></i>
+												</button>
+											</div>
+										</td>
+									</tr>
+								{% endfor %}
+							</tbody>
+						</table>
+
 					</div>
 				</div>