2019-02-26 14:50:48 +01:00
|
|
|
<!DOCTYPE html>
|
|
|
|
|
|
|
|
|
|
<html>
|
|
|
|
|
<head>
|
|
|
|
|
<title>AIL-Framework</title>
|
|
|
|
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
|
|
|
|
<!-- Core CSS -->
|
|
|
|
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
|
|
|
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
|
|
|
|
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
|
|
|
|
|
|
|
|
|
<!-- JS -->
|
|
|
|
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
|
|
|
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
|
|
|
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
|
|
|
|
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
|
|
|
|
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
|
|
|
|
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
|
|
|
|
|
|
<body>
|
|
|
|
|
|
|
|
|
|
{% include 'nav_bar.html' %}
|
|
|
|
|
|
|
|
|
|
<div class="container-fluid">
|
|
|
|
|
<div class="row">
|
|
|
|
|
|
2019-02-28 14:56:14 +01:00
|
|
|
{% include 'crawler/menu_sidebar.html' %}
|
2019-02-26 14:50:48 +01:00
|
|
|
|
2019-02-28 14:56:14 +01:00
|
|
|
<div class="col-12 col-lg-10" id="core_content">
|
2019-02-26 14:50:48 +01:00
|
|
|
|
2019-06-04 14:22:46 +02:00
|
|
|
{% include 'crawler/crawler_disabled.html' %}
|
2019-02-26 14:50:48 +01:00
|
|
|
|
|
|
|
|
<div class="card text-white bg-dark mb-3 mt-1">
|
|
|
|
|
<div class="card-header">
|
|
|
|
|
<h5 class="card-title">Crawl a Domain</h5>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="card-body">
|
|
|
|
|
<p class="card-text">Enter a domain and choose what kind of data you want.</p>
|
2020-03-30 18:43:50 +02:00
|
|
|
<form action="{{ url_for('crawler_splash.send_to_spider') }}" method='post'>
|
2019-02-26 14:50:48 +01:00
|
|
|
<div class="row">
|
|
|
|
|
<div class="col-12 col-lg-6">
|
|
|
|
|
<div class="input-group" id="date-range-from">
|
|
|
|
|
<input type="text" class="form-control" id="url_to_crawl" name="url_to_crawl" placeholder="Address or Domain">
|
|
|
|
|
</div>
|
2020-08-24 22:31:41 +02:00
|
|
|
<div class="d-flex mt-2">
|
|
|
|
|
<i class="fas fa-spider mt-1"></i> Crawler Type
|
|
|
|
|
<div class="custom-control custom-switch">
|
|
|
|
|
<input class="custom-control-input" type="checkbox" name="queue_type_selector" value="True" id="queue_type_selector">
|
|
|
|
|
<label class="custom-control-label" for="queue_type_selector">
|
|
|
|
|
<i class="fas fa-splotch"></i> Splash Name
|
|
|
|
|
</label>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
<div id="div_crawler_queue_type">
|
|
|
|
|
<select class="custom-select form-control" name="crawler_queue_type" id="crawler_queue_type">
|
|
|
|
|
{%for crawler_type in all_crawlers_types%}
|
|
|
|
|
<option value="{{crawler_type}}" {%if crawler_type=='tor'%}selected{%endif%}>{{crawler_type}}</option>
|
|
|
|
|
{%endfor%}
|
|
|
|
|
</select>
|
|
|
|
|
</div>
|
|
|
|
|
<div id="div_splash_name">
|
|
|
|
|
<select class="custom-select form-control" name="splash_name" id="splash_name">
|
|
|
|
|
<option value="None" selected>Don't use a special splash crawler</option>
|
|
|
|
|
{%for splash_name in all_splash_name%}
|
|
|
|
|
<option value="{{splash_name}}">{{splash_name}}</option>
|
|
|
|
|
{%endfor%}
|
|
|
|
|
</select>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="d-flex mt-3">
|
2019-02-26 14:50:48 +01:00
|
|
|
<i class="fas fa-user-ninja mt-1"></i> Manual
|
|
|
|
|
<div class="custom-control custom-switch">
|
|
|
|
|
<input class="custom-control-input" type="checkbox" name="crawler_type" value="True" id="crawler_type">
|
|
|
|
|
<label class="custom-control-label" for="crawler_type">
|
|
|
|
|
<i class="fas fa-clock"></i> Automatic
|
|
|
|
|
</label>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="input-group mt-2 mb-2" id="crawler_epoch_input">
|
|
|
|
|
<div class="input-group-prepend">
|
|
|
|
|
<span class="input-group-text bg-light"><i class="fas fa-clock"></i> </span>
|
|
|
|
|
</div>
|
2019-02-26 15:39:52 +01:00
|
|
|
<input class="form-control" type="number" id="crawler_epoch" value="3600" min="1" name="crawler_epoch" required>
|
2019-02-26 14:50:48 +01:00
|
|
|
<div class="input-group-append">
|
|
|
|
|
<span class="input-group-text">Time (seconds) between each crawling</span>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="col-12 col-lg-6 mt-2 mt-lg-0">
|
|
|
|
|
|
|
|
|
|
<div class="row">
|
|
|
|
|
<div class="col-12 col-xl-6">
|
|
|
|
|
<div class="custom-control custom-switch">
|
|
|
|
|
<input class="custom-control-input" type="checkbox" name="html_content" value="True" id="html_content_id" checked disabled>
|
|
|
|
|
<label class="custom-control-label" for="html_content_id">
|
|
|
|
|
<i class="fab fa-html5"></i> HTML
|
|
|
|
|
</label>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="custom-control custom-switch mt-1">
|
|
|
|
|
<input class="custom-control-input" type="checkbox" name="screenshot" value="True" id="screenshot_id">
|
|
|
|
|
<label class="custom-control-label" for="screenshot_id">
|
|
|
|
|
<i class="fas fa-image"></i> Screenshot
|
|
|
|
|
</label>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="custom-control custom-switch mt-1">
|
|
|
|
|
<input class="custom-control-input" type="checkbox" name="har" value="True" id="har_id">
|
|
|
|
|
<label class="custom-control-label" for="har_id">
|
|
|
|
|
<i class="fas fa-file"></i> HAR
|
|
|
|
|
</label>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="col-12 col-xl-6">
|
|
|
|
|
<div class="input-group form-group mb-0">
|
|
|
|
|
<div class="input-group-prepend">
|
|
|
|
|
<span class="input-group-text bg-light"><i class="fas fa-water"></i></span>
|
|
|
|
|
</div>
|
2019-02-26 15:39:52 +01:00
|
|
|
<input class="form-control" type="number" id="depth_limit" name="depth_limit" min="0" value="0" required>
|
2019-02-26 14:50:48 +01:00
|
|
|
<div class="input-group-append">
|
|
|
|
|
<span class="input-group-text">Depth Limit</span>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="input-group mt-2">
|
|
|
|
|
<div class="input-group-prepend">
|
|
|
|
|
<span class="input-group-text bg-light"><i class="fas fa-copy"></i> </span>
|
|
|
|
|
</div>
|
2019-02-26 15:39:52 +01:00
|
|
|
<input class="form-control" type="number" id="max_pages" name="max_pages" min="1" value="1" required>
|
2019-02-26 14:50:48 +01:00
|
|
|
<div class="input-group-append">
|
|
|
|
|
<span class="input-group-text">Max Pages</span>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
2020-03-30 18:43:50 +02:00
|
|
|
|
|
|
|
|
<div class="mt-1">
|
|
|
|
|
<i class="mt-2 text-white fas fa-cookie-bite"></i> Cookiejar:
|
|
|
|
|
<select class="custom-select form-control mt-1" name="cookiejar" id="cookiejar">
|
|
|
|
|
<option value="None" selected>Don't use any cookiejar</option>
|
|
|
|
|
{%for cookiejar in l_cookiejar%}
|
|
|
|
|
<option value="{{cookiejar}}">{{cookiejar}}</option>
|
|
|
|
|
{%endfor%}
|
|
|
|
|
</select>
|
|
|
|
|
</div>
|
|
|
|
|
|
2019-02-26 14:50:48 +01:00
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
<button class="btn btn-primary mt-2">
|
|
|
|
|
<i class="fas fa-spider"></i> Send to Spider
|
|
|
|
|
</button>
|
|
|
|
|
<form>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</body>
|
|
|
|
|
|
|
|
|
|
<script>
|
|
|
|
|
var chart = {};
|
|
|
|
|
$(document).ready(function(){
|
|
|
|
|
$("#page-Crawler").addClass("active");
|
2019-02-28 14:56:14 +01:00
|
|
|
$("#nav_manual_crawler").addClass("active");
|
2020-08-24 22:31:41 +02:00
|
|
|
queue_type_selector_input_controler()
|
2019-02-26 14:50:48 +01:00
|
|
|
manual_crawler_input_controler();
|
|
|
|
|
|
2020-03-04 15:12:32 +01:00
|
|
|
$('#crawler_type').on("change", function () {
|
2019-02-26 14:50:48 +01:00
|
|
|
manual_crawler_input_controler();
|
|
|
|
|
});
|
2020-08-24 22:31:41 +02:00
|
|
|
|
|
|
|
|
$('#queue_type_selector').on("change", function () {
|
|
|
|
|
queue_type_selector_input_controler();
|
|
|
|
|
});
|
2019-02-26 14:50:48 +01:00
|
|
|
});
|
|
|
|
|
|
2019-02-28 14:56:14 +01:00
|
|
|
function toggle_sidebar(){
|
|
|
|
|
if($('#nav_menu').is(':visible')){
|
|
|
|
|
$('#nav_menu').hide();
|
|
|
|
|
$('#side_menu').removeClass('border-right')
|
|
|
|
|
$('#side_menu').removeClass('col-lg-2')
|
|
|
|
|
$('#core_content').removeClass('col-lg-10')
|
|
|
|
|
}else{
|
|
|
|
|
$('#nav_menu').show();
|
|
|
|
|
$('#side_menu').addClass('border-right')
|
|
|
|
|
$('#side_menu').addClass('col-lg-2')
|
|
|
|
|
$('#core_content').addClass('col-lg-10')
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2019-02-26 14:50:48 +01:00
|
|
|
function manual_crawler_input_controler() {
|
|
|
|
|
if($('#crawler_type').is(':checked')){
|
|
|
|
|
$("#crawler_epoch_input").show();
|
|
|
|
|
}else{
|
|
|
|
|
$("#crawler_epoch_input").hide();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-08-24 22:31:41 +02:00
|
|
|
function queue_type_selector_input_controler() {
|
|
|
|
|
if($('#queue_type_selector').is(':checked')){
|
|
|
|
|
$("#div_crawler_queue_type").hide();
|
|
|
|
|
$("#div_splash_name").show();
|
|
|
|
|
}else{
|
|
|
|
|
$("#div_crawler_queue_type").show();
|
|
|
|
|
$("#div_splash_name").hide();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2019-02-26 14:50:48 +01:00
|
|
|
</script>
|
