chg: [cookiejar UI] add cookiejar + show all

pull/486/head
Terrtia 2020-03-27 17:06:26 +01:00
parent 1ce7b7c2a0
commit 169c4a8ec7
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
8 changed files with 348 additions and 244 deletions

View File

@ -176,8 +176,8 @@ class TorSplashCrawler():
# detect connection to proxy refused
error_log = (json.loads(response.body.decode()))
print(error_log)
elif crawlers.is_redirection(self.domains[0], response.data['last_url']):
pass # ignore response
#elif crawlers.is_redirection(self.domains[0], response.data['last_url']):
# pass # ignore response
else:
item_id = crawlers.create_item_id(self.item_dir, self.domains[0])

View File

@ -177,31 +177,10 @@ def domains_explorer_web():
def crawler_cookiejar_add():
return render_template("add_cookiejar.html")
@crawler_splash.route('/crawler/cookiejar/all', methods=['GET'])
#@login_required
#@login_read_only
def crawler_cookies_all():
user_id = current_user.get_id()
user_cookiejar = crawlers.get_all_user_cookies_metadata(user_id)
global_cookiejar = crawlers.get_all_global_cookies_metadata()
return render_template("all_cookies.html", user_cookies=user_cookies, global_cookies=global_cookies)
## - - ##
@crawler_splash.route('/crawler/cookies/add', methods=['GET'])
@crawler_splash.route('/crawler/cookiejar/add_post', methods=['POST'])
#@login_required
#@login_analyst
def crawler_cookies_add():
return render_template("add_cookies.html")
@crawler_splash.route('/crawler/cookies/add_post', methods=['POST'])
#@login_required
#@login_analyst
def crawler_cookies_add_post():
def crawler_cookiejar_add_post():
user_id = current_user.get_id()
description = request.form.get('description')
@ -213,9 +192,9 @@ def crawler_cookies_add_post():
if 'file' in request.files:
file = request.files['file']
json_file = file.read().decode()
json_cookies = file.read().decode()
else:
json_file = '[]'
json_cookies = None
# Get cookies to add
l_manual_cookie = []
@ -231,27 +210,38 @@ def crawler_cookies_add_post():
if l_invalid_cookie:
return create_json_response({'error': 'invalid cookie', 'invalid fileds': l_invalid_cookie}, 400)
cookies_uuid = crawler_splash.save_cookies(user_id, json_cookies=json_file, l_cookies=l_manual_cookie, level=level, description=description)
return redirect(url_for('crawler_splash.crawler_cookies_all', cookies_uuid=cookies_uuid))
cookiejar_uuid = crawlers.create_cookiejar(user_id, level=level, description=description)
if json_cookies:
res = crawlers.api_import_cookies_from_json(json_cookies, cookiejar_uuid)
if res:
return create_json_response(res[0], res[1])
if l_manual_cookie:
crawlers.add_cookies_to_cookiejar(cookiejar_uuid, l_manual_cookie)
return render_template("add_cookiejar.html")
@crawler_splash.route('/crawler/cookies/all', methods=['GET'])
@crawler_splash.route('/crawler/cookiejar/all', methods=['GET'])
#@login_required
#@login_read_only
def crawler_cookies_all():
def crawler_cookiejar_all():
user_id = current_user.get_id()
user_cookies = crawlers.get_all_user_cookies_metadata(user_id)
global_cookies = crawlers.get_all_global_cookies_metadata()
return render_template("all_cookies.html", user_cookies=user_cookies, global_cookies=global_cookies)
user_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_user_cookiejar(user_id))
global_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_global_cookiejar())
return render_template("all_cookiejar.html", user_cookiejar=user_cookiejar, global_cookiejar=global_cookiejar)
@crawler_splash.route('/crawler/cookies/show', methods=['GET'])
@crawler_splash.route('/crawler/cookiejar/show', methods=['GET'])
#@login_required
#@login_read_only
def crawler_cookies_show():
def crawler_cookiejar_show():
user_id = current_user.get_id()
cookies_uuid = request.args.get('cookies_uuid')
res = crawlers.api_get_cookies(cookies_uuid, user_id)
cookiejar_uuid = request.args.get('cookiejar_uuid')
res = crawlers.api_get_cookiejar_cookies(cookiejar_uuid, user_id))
if res[1] !=200:
return create_json_response(res[0], res[1])
cookies_json = json.dumps(res[0]['json_cookies'], indent=4, sort_keys=True)
cookie_metadata = crawlers.get_cookies_metadata(cookies_uuid)
return render_template("edit_cookies.html", cookie_metadata=cookie_metadata, cookies_json=cookies_json, manual_cookies=res[0]['manual_cookies'])
cookiejar_metadata = crawlers.get_cookiejar_metadata(cookiejar_uuid, level=False)
cookies = json.dumps(res[0]['json_cookies'], indent=4, sort_keys=True)
return render_template("show_cookiejar.html", cookiejar_metadata=cookiejar_metadata, l_cookies=res[0])
## - - ##

View File

@ -29,11 +29,11 @@
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title"><i class="fas fa-cookie"></i> Add Cookies</h5>
<h5 class="card-title"><i class="fas fa-box"></i> Create Cookijar <i class="fas fa-cookie"></i></h5>
</div>
<div class="card-body">
<form action="{{ url_for('crawler_splash.crawler_cookies_add_post') }}" method="post" enctype="multipart/form-data">
<form action="{{ url_for('crawler_splash.crawler_cookiejar_add_post') }}" method="post" enctype="multipart/form-data">
<div class="row">
<div class="col-12 col-md-9">
@ -48,7 +48,7 @@
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="level" id="id_level" checked="">
<label class="custom-control-label" for="id_level">
<i class="fas fa-users"></i>&nbsp;Show cookies to all Users
<i class="fas fa-users"></i>&nbsp;Show cookiejar to all Users
</label>
</div>
</div>
@ -56,8 +56,9 @@
<hr>
<h5>Import cookies:</h5>
<div class="form-group">
<label for="file"><b>JSON File</b> Cookies to import:</label>
<label for="file"><b>JSON File</b></label>
<input type="file" class="form-control-file btn btn-outline-secondary" id="file" name="file">
</div>
@ -65,7 +66,7 @@
<div>
<h5>Add manual cookies:</h5>
<h5>Create cookies</h5>
<div class="row">
<div class="col-5" for="obj_input_cookie_name"><b>Cookie Name</b></div>
@ -93,7 +94,7 @@
</div>
<div class="form-group">
<button class="btn btn-info" type="submit" value=Upload><i class="fas fa-cookie-bite"></i> Add Cookies</button>
<button class="btn btn-info" type="submit" value=Upload><i class="fas fa-cookie-bite"></i> Create Cookiejar</button>
</div>
</form>
@ -113,8 +114,8 @@
var chart = {};
$(document).ready(function(){
$("#page-crawler").addClass("active");
$("#nav_cookies_add").addClass("active");
$("#nav_title_cookies").removeClass("text-muted");
$("#nav_cookiejar_add").addClass("active");
$("#nav_title_cookiejar").removeClass("text-muted");
});
var input_1 = '<div class="input-group mb-1"><input type="text" class="form-control col-5" name="'

View File

@ -0,0 +1,99 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AIL - Cookies</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'crawler/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title"><i class="fas fa-cookie-bite"></i> Your Cookiejar</h5>
</div>
<div class="card-body">
{% with all_cookiejar=user_cookiejar, table_id='table_user'%}
{% include 'crawler/crawler_splash/table_cookiejar.html' %}
{% endwith %}
</div>
</div>
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title"><i class="fas fa-cookie-bite"></i> Global Cookiejar</h5>
</div>
<div class="card-body">
{% with all_cookiejar=global_cookiejar, table_id='table_global'%}
{% include 'crawler/crawler_splash/table_cookiejar.html' %}
{% endwith %}
</div>
</div>
<a class="btn btn-info my-4" href="{{url_for('crawler_splash.crawler_cookiejar_add')}}">
<i class="fas fa-plus-circle ml-auto"></i>
Create Cookiejar
</a>
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-crawler").addClass("active");
$("#nav_cookiejar_all").addClass("active");
$("#nav_title_cookiejar").removeClass("text-muted");
$('#table_user').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "desc" ]]
});
$('#table_global').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</body>
</html>

View File

@ -1,148 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AIL - Edit Cookies</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'crawler/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title"><i class="fas fa-cookie-bite"></i> Edit Cookies</h5>
</div>
<div class="card-body">
{% with all_cookies=[cookie_metadata], table_id='table_metadata'%}
{% include 'crawler/crawler_splash/table_cookies.html' %}
{% endwith %}
</div>
</div>
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title"><i class="fas fa-cookie-bite"></i> JSON Cookies</h5>
</div>
<div class="card-body">
<p>
<pre class="border">{{cookies_json}}</pre>
</p>
</div>
</div>
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title"><i class="fas fa-cookie-bite"></i> Manual Cookies</h5>
</div>
<div class="card-body">
<div class="row">
<div class="col-5" for="obj_input_cookie_name"><b>Cookie Name</b></div>
<div class="col-6" for="obj_input_cookie_value"><b>Cookie Value</b></div>
</div>
<div class="form-horizontal">
<div class="form-body">
<div class="form-group">
<div class="fields">
<div class="input-group mb-1">
<input type="text" class="form-control col-5" name="first_cookie" id="obj_input_cookie_name">
<input type="text" class="form-control col-6" name="first_cookie" id="obj_input_cookie_value">
<span class="btn btn-info input-group-addon add-field col-1"><i class="fas fa-plus"></i></span>
</div>
{%for manual_cookie in manual_cookies%}
<div class="input-group mb-1">
<input type="text" class="form-control col-5" name="{{manual_cookie}}" value="{{manual_cookie[0]}}">
<input type="text" class="form-control col-6" name="{{manual_cookie}}" value="{{manual_cookie[1]}}">
<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span>
</div>
{%endfor%}
<span class="help-block" hidden>Manual Cookies></span>
</div>
</div>
</div>
</div>
</div>
</div>
<br>
</div>
</div>
</div>
<script>
$(document).ready(function(){
$("#page-crawler").addClass("active");
$("#nav_title_cookies").removeClass("text-muted");
$('#table_user').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "desc" ]]
});
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
var input_1 = '<div class="input-group mb-1"><input type="text" class="form-control col-5" name="'
var input_2 = '"><input type="text" class="form-control col-6" name="'
var input_3 = '">';
var minusButton = '<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span></div>';
$('.add-field').click(function() {
var new_uuid = uuidv4();
var template = input_1 + new_uuid + input_2 + new_uuid + input_3;
var temp = $(template).insertBefore('.help-block');
temp.append(minusButton);
});
$('.fields').on('click', '.delete-field', function(){
$(this).parent().remove();
});
function uuidv4() {
return ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, c =>
(c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
);
}
</script>
</body>
</html>

View File

@ -0,0 +1,183 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL - Add Cookies</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'crawler/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<table id="{{table_id}}" class="table table-striped table-bordered">
<thead class="bg-dark text-white">
<tr>
<th class="bg-info text-white">Description</th>
<th class="bg-info text-white">Date</th>
<th class="bg-info text-white">UUID</th>
<th class="bg-info text-white">User</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
<tr>
<td>{{dict_cookiejar['description']}}</td>
<td>
{%if dict_cookiejar['date']%}
{{dict_cookiejar['date'][0:4]}}/{{dict_cookiejar['date'][4:6]}}/{{dict_cookiejar['date'][6:8]}}
{%endif%}
</td>
<td>
{{ dict_cookiejar['cookiejar_uuid']}}
</td>
<td>{{dict_cookiejar['user_id']}}</td>
</tr>
</tbody>
</table>
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title"><i class="fas fa-box"></i> Create Cookijar <i class="fas fa-cookie"></i></h5>
</div>
<div class="card-body">
<form action="{{ url_for('crawler_splash.crawler_cookiejar_add_post') }}" method="post" enctype="multipart/form-data">
<div class="row">
<div class="col-12 col-md-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text"><i class="fas fa-tag"></i></div>
</div>
<input id="description" name="description" class="form-control" placeholder="cookies description - (optional)" type="text">
</div>
</div>
<div class="col-12 col-md-3">
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="level" id="id_level" checked="">
<label class="custom-control-label" for="id_level">
<i class="fas fa-users"></i>&nbsp;Show cookiejar to all Users
</label>
</div>
</div>
</div>
<hr>
<h5>Import cookies:</h5>
<div class="form-group">
<label for="file"><b>JSON File</b></label>
<input type="file" class="form-control-file btn btn-outline-secondary" id="file" name="file">
</div>
<hr>
<div>
<h5>Create cookies</h5>
<div class="row">
<div class="col-5" for="obj_input_cookie_name"><b>Cookie Name</b></div>
<div class="col-6" for="obj_input_cookie_value"><b>Cookie Value</b></div>
</div>
<div class="form-horizontal">
<div class="form-body">
<div class="form-group">
<div class="fields">
<div class="input-group mb-1">
<input type="text" class="form-control col-5" name="first_cookie" id="obj_input_cookie_name">
<input type="text" class="form-control col-6" name="first_cookie" id="obj_input_cookie_value">
<span class="btn btn-info input-group-addon add-field col-1"><i class="fas fa-plus"></i></span>
</div>
<br>
<span class="help-block" hidden>Manual Cookies></span>
</div>
</div>
</div>
</div>
</div>
<div class="form-group">
<button class="btn btn-info" type="submit" value=Upload><i class="fas fa-cookie-bite"></i> Create Cookiejar</button>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
var chart = {};
$(document).ready(function(){
$("#page-crawler").addClass("active");
$("#nav_cookiejar_add").addClass("active");
$("#nav_title_cookiejar").removeClass("text-muted");
});
var input_1 = '<div class="input-group mb-1"><input type="text" class="form-control col-5" name="'
var input_2 = '"><input type="text" class="form-control col-6" name="'
var input_3 = '">';
var minusButton = '<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span></div>';
$('.add-field').click(function() {
var new_uuid = uuidv4();
var template = input_1 + new_uuid + input_2 + new_uuid + input_3;
var temp = $(template).insertBefore('.help-block');
temp.append(minusButton);
});
$('.fields').on('click', '.delete-field', function(){
$(this).parent().remove();
});
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
function uuidv4() {
return ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, c =>
(c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
);
}
</script>

View File

@ -8,20 +8,20 @@
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_cookies in all_cookies %}
{% for dict_cookiejar in all_cookiejar %}
<tr>
<td>{{dict_cookies['description']}}</td>
<td>{{dict_cookiejar['description']}}</td>
<td>
{%if dict_cookies['date']%}
{{dict_cookies['date'][0:4]}}/{{dict_cookies['date'][4:6]}}/{{dict_cookies['date'][6:8]}}
{%if dict_cookiejar['date']%}
{{dict_cookiejar['date'][0:4]}}/{{dict_cookiejar['date'][4:6]}}/{{dict_cookiejar['date'][6:8]}}
{%endif%}
</td>
<td>
<a target="_blank" href="{{ url_for('crawler_splash.crawler_cookies_show') }}?cookies_uuid={{ dict_cookies['cookies_uuid'] }}">
{{ dict_cookies['cookies_uuid']}}
<a target="_blank" href="{{ url_for('crawler_splash.crawler_cookies_show') }}?cookies_uuid={{ dict_cookiejar['cookiejar_uuid'] }}">
{{ dict_cookiejar['cookiejar_uuid']}}
</a>
</td>
<td>{{dict_cookies['user_id']}}</td>
<td>{{dict_cookiejar['user_id']}}</td>
</tr>
{% endfor %}
</tbody>

View File

@ -63,31 +63,9 @@
</li>
</ul>
<h5 class="d-flex text-muted w-100" id="nav_title_cookies">
<span>Cookies </span>
<a class="ml-auto" href="{{url_for('crawler_splash.crawler_cookies_add')}}">
<i class="fas fa-plus-circle ml-auto"></i>
</a>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item">
<a class="nav-link" href="{{url_for('crawler_splash.crawler_cookies_add')}}" id="nav_cookies_add">
<i class="fas fa-cookie"></i>
<span>Add Cookies</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('crawler_splash.crawler_cookies_all')}}" id="nav_cookies_all">
<i class="fas fa-cookie-bite"></i>
<span>All Cookies</span>
</a>
</li>
</ul>
</nav>
<h5 class="d-flex text-muted w-100" id="nav_title_cookies">
<h5 class="d-flex text-muted w-100" id="nav_title_cookiejar">
<span>Cookiejar </span>
<a class="ml-auto" href="{{url_for('crawler_splash.crawler_cookies_add')}}">
<a class="ml-auto" href="{{url_for('crawler_splash.crawler_cookiejar_add')}}">
<i class="fas fa-plus-circle ml-auto"></i>
</a>
</h5>
@ -99,10 +77,11 @@
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('crawler_splash.crawler_cookies_all')}}" id="nav_cookiejar_all">
<a class="nav-link" href="{{url_for('crawler_splash.crawler_cookiejar_all')}}" id="nav_cookiejar_all">
<i class="fas fa-cookie-bite"></i>
<span>All Cookiejar</span>
</a>
</li>
</ul>
</nav>
</div>