diff --git a/bin/torcrawler/TorSplashCrawler.py b/bin/torcrawler/TorSplashCrawler.py index 96eb8bf6..cf408d83 100644 --- a/bin/torcrawler/TorSplashCrawler.py +++ b/bin/torcrawler/TorSplashCrawler.py @@ -176,8 +176,8 @@ class TorSplashCrawler(): # detect connection to proxy refused error_log = (json.loads(response.body.decode())) print(error_log) - elif crawlers.is_redirection(self.domains[0], response.data['last_url']): - pass # ignore response + #elif crawlers.is_redirection(self.domains[0], response.data['last_url']): + # pass # ignore response else: item_id = crawlers.create_item_id(self.item_dir, self.domains[0]) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 2f77fb48..3a0fbfdb 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -177,31 +177,10 @@ def domains_explorer_web(): def crawler_cookiejar_add(): return render_template("add_cookiejar.html") -@crawler_splash.route('/crawler/cookiejar/all', methods=['GET']) -#@login_required -#@login_read_only -def crawler_cookies_all(): - user_id = current_user.get_id() - user_cookiejar = crawlers.get_all_user_cookies_metadata(user_id) - global_cookiejar = crawlers.get_all_global_cookies_metadata() - return render_template("all_cookies.html", user_cookies=user_cookies, global_cookies=global_cookies) - - - - - -## - - ## - -@crawler_splash.route('/crawler/cookies/add', methods=['GET']) +@crawler_splash.route('/crawler/cookiejar/add_post', methods=['POST']) #@login_required #@login_analyst -def crawler_cookies_add(): - return render_template("add_cookies.html") - -@crawler_splash.route('/crawler/cookies/add_post', methods=['POST']) -#@login_required -#@login_analyst -def crawler_cookies_add_post(): +def crawler_cookiejar_add_post(): user_id = current_user.get_id() description = request.form.get('description') @@ -213,9 +192,9 @@ def crawler_cookies_add_post(): if 'file' in request.files: file = request.files['file'] - json_file = file.read().decode() + json_cookies = file.read().decode() else: - json_file = '[]' + json_cookies = None # Get cookies to add l_manual_cookie = [] @@ -231,27 +210,38 @@ def crawler_cookies_add_post(): if l_invalid_cookie: return create_json_response({'error': 'invalid cookie', 'invalid fileds': l_invalid_cookie}, 400) - cookies_uuid = crawler_splash.save_cookies(user_id, json_cookies=json_file, l_cookies=l_manual_cookie, level=level, description=description) - return redirect(url_for('crawler_splash.crawler_cookies_all', cookies_uuid=cookies_uuid)) + cookiejar_uuid = crawlers.create_cookiejar(user_id, level=level, description=description) + if json_cookies: + res = crawlers.api_import_cookies_from_json(json_cookies, cookiejar_uuid) + if res: + return create_json_response(res[0], res[1]) + if l_manual_cookie: + crawlers.add_cookies_to_cookiejar(cookiejar_uuid, l_manual_cookie) + return render_template("add_cookiejar.html") -@crawler_splash.route('/crawler/cookies/all', methods=['GET']) +@crawler_splash.route('/crawler/cookiejar/all', methods=['GET']) #@login_required #@login_read_only -def crawler_cookies_all(): +def crawler_cookiejar_all(): user_id = current_user.get_id() - user_cookies = crawlers.get_all_user_cookies_metadata(user_id) - global_cookies = crawlers.get_all_global_cookies_metadata() - return render_template("all_cookies.html", user_cookies=user_cookies, global_cookies=global_cookies) + user_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_user_cookiejar(user_id)) + global_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_global_cookiejar()) + return render_template("all_cookiejar.html", user_cookiejar=user_cookiejar, global_cookiejar=global_cookiejar) -@crawler_splash.route('/crawler/cookies/show', methods=['GET']) +@crawler_splash.route('/crawler/cookiejar/show', methods=['GET']) #@login_required #@login_read_only -def crawler_cookies_show(): +def crawler_cookiejar_show(): user_id = current_user.get_id() - cookies_uuid = request.args.get('cookies_uuid') - res = crawlers.api_get_cookies(cookies_uuid, user_id) + cookiejar_uuid = request.args.get('cookiejar_uuid') + + res = crawlers.api_get_cookiejar_cookies(cookiejar_uuid, user_id)) if res[1] !=200: return create_json_response(res[0], res[1]) - cookies_json = json.dumps(res[0]['json_cookies'], indent=4, sort_keys=True) - cookie_metadata = crawlers.get_cookies_metadata(cookies_uuid) - return render_template("edit_cookies.html", cookie_metadata=cookie_metadata, cookies_json=cookies_json, manual_cookies=res[0]['manual_cookies']) + + cookiejar_metadata = crawlers.get_cookiejar_metadata(cookiejar_uuid, level=False) + + cookies = json.dumps(res[0]['json_cookies'], indent=4, sort_keys=True) + return render_template("show_cookiejar.html", cookiejar_metadata=cookiejar_metadata, l_cookies=res[0]) + +## - - ## diff --git a/var/www/templates/crawler/crawler_splash/add_cookies.html b/var/www/templates/crawler/crawler_splash/add_cookiejar.html similarity index 88% rename from var/www/templates/crawler/crawler_splash/add_cookies.html rename to var/www/templates/crawler/crawler_splash/add_cookiejar.html index 801212a3..ddd2dd2d 100644 --- a/var/www/templates/crawler/crawler_splash/add_cookies.html +++ b/var/www/templates/crawler/crawler_splash/add_cookiejar.html @@ -29,11 +29,11 @@
-
Add Cookies
+
Create Cookijar
-
+
@@ -48,7 +48,7 @@
@@ -56,8 +56,9 @@
+
Import cookies:
- +
@@ -65,7 +66,7 @@
-
Add manual cookies:
+
Create cookies
Cookie Name
@@ -93,7 +94,7 @@
- +
@@ -113,8 +114,8 @@ var chart = {}; $(document).ready(function(){ $("#page-crawler").addClass("active"); - $("#nav_cookies_add").addClass("active"); - $("#nav_title_cookies").removeClass("text-muted"); + $("#nav_cookiejar_add").addClass("active"); + $("#nav_title_cookiejar").removeClass("text-muted"); }); var input_1 = '
+ + + AIL - Cookies + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'crawler/menu_sidebar.html' %} + +
+ +
+
+
Your Cookiejar
+
+
+ {% with all_cookiejar=user_cookiejar, table_id='table_user'%} + {% include 'crawler/crawler_splash/table_cookiejar.html' %} + {% endwith %} +
+
+ +
+
+
Global Cookiejar
+
+
+ {% with all_cookiejar=global_cookiejar, table_id='table_global'%} + {% include 'crawler/crawler_splash/table_cookiejar.html' %} + {% endwith %} +
+
+ + + + Create Cookiejar + + +
+
+
+ + + + + diff --git a/var/www/templates/crawler/crawler_splash/edit_cookies.html b/var/www/templates/crawler/crawler_splash/edit_cookies.html deleted file mode 100644 index e9dbe9b6..00000000 --- a/var/www/templates/crawler/crawler_splash/edit_cookies.html +++ /dev/null @@ -1,148 +0,0 @@ - - - - - - - - AIL - Edit Cookies - - - - - - - - - - - - - - - {% include 'nav_bar.html' %} - -
-
- - {% include 'crawler/menu_sidebar.html' %} - -
- -
-
-
Edit Cookies
-
-
- {% with all_cookies=[cookie_metadata], table_id='table_metadata'%} - {% include 'crawler/crawler_splash/table_cookies.html' %} - {% endwith %} -
-
- -
-
-
JSON Cookies
-
-
-

- 

{{cookies_json}}
-

-
-
- -
-
-
Manual Cookies
-
-
- -
-
Cookie Name
-
Cookie Value
-
- -
-
-
-
-
- - - -
- - {%for manual_cookie in manual_cookies%} -
- - - -
- {%endfor%} - -
-
-
-
- -
-
- -
- -
-
-
- - - - - diff --git a/var/www/templates/crawler/crawler_splash/show_cookiejar.html b/var/www/templates/crawler/crawler_splash/show_cookiejar.html new file mode 100644 index 00000000..4ae8063d --- /dev/null +++ b/var/www/templates/crawler/crawler_splash/show_cookiejar.html @@ -0,0 +1,183 @@ + + + + + AIL - Add Cookies + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'crawler/menu_sidebar.html' %} + +
+ + + + + + + + + + + + + + + + + + +
DescriptionDateUUIDUser
{{dict_cookiejar['description']}} + {%if dict_cookiejar['date']%} + {{dict_cookiejar['date'][0:4]}}/{{dict_cookiejar['date'][4:6]}}/{{dict_cookiejar['date'][6:8]}} + {%endif%} + + {{ dict_cookiejar['cookiejar_uuid']}} + {{dict_cookiejar['user_id']}}
+ +
+
+
Create Cookijar
+
+
+ + +
+ +
+
+
+
+
+
+ +
+
+
+
+ + +
+
+
+ +
+ +
Import cookies:
+
+ + +
+ +
+ +
+ +
Create cookies
+ +
+
Cookie Name
+
Cookie Value
+
+ +
+
+
+
+
+ + + +
+ +
+ + +
+
+
+
+ +
+ +
+ +
+ +
+ +
+
+ + + +
+
+
+ + + + diff --git a/var/www/templates/crawler/crawler_splash/table_cookies.html b/var/www/templates/crawler/crawler_splash/table_cookiejar.html similarity index 58% rename from var/www/templates/crawler/crawler_splash/table_cookies.html rename to var/www/templates/crawler/crawler_splash/table_cookiejar.html index e42c3fa9..10b4fe53 100644 --- a/var/www/templates/crawler/crawler_splash/table_cookies.html +++ b/var/www/templates/crawler/crawler_splash/table_cookiejar.html @@ -8,20 +8,20 @@ - {% for dict_cookies in all_cookies %} + {% for dict_cookiejar in all_cookiejar %} - {{dict_cookies['description']}} + {{dict_cookiejar['description']}} - {%if dict_cookies['date']%} - {{dict_cookies['date'][0:4]}}/{{dict_cookies['date'][4:6]}}/{{dict_cookies['date'][6:8]}} + {%if dict_cookiejar['date']%} + {{dict_cookiejar['date'][0:4]}}/{{dict_cookiejar['date'][4:6]}}/{{dict_cookiejar['date'][6:8]}} {%endif%} - - {{ dict_cookies['cookies_uuid']}} + + {{ dict_cookiejar['cookiejar_uuid']}} - {{dict_cookies['user_id']}} + {{dict_cookiejar['user_id']}} {% endfor %} diff --git a/var/www/templates/crawler/menu_sidebar.html b/var/www/templates/crawler/menu_sidebar.html index b575b68a..c14abbbe 100644 --- a/var/www/templates/crawler/menu_sidebar.html +++ b/var/www/templates/crawler/menu_sidebar.html @@ -63,46 +63,25 @@ -
- Cookies - - - -
-