mirror of https://github.com/CIRCL/AIL-framework
chg: [crawler] auto tag crawled domains
Terrtia
parent
c02b9f2a81
commit
501d10bbbd
|
|
@ -200,10 +200,13 @@ class Crawler(AbstractModule):
|
|||
self.save_capture_response(parent_id, entries)
|
||||
|
||||
self.domain.update_daterange(self.date.replace('/', ''))
|
||||
# Origin + History
|
||||
# Origin + History + tags
|
||||
if self.root_item:
|
||||
self.domain.set_last_origin(parent_id)
|
||||
self.domain.add_history(epoch, root_item=self.root_item)
|
||||
# Tags
|
||||
for tag in task.get_tags():
|
||||
self.domain.add_tag(tag)
|
||||
elif self.domain.was_up():
|
||||
self.domain.add_history(epoch, root_item=epoch)
|
||||
|
||||
|
|
|
|||
|
|
@ -967,6 +967,7 @@ class CrawlerScheduler:
|
|||
task_uuid = create_task(meta['url'], depth=meta['depth'], har=meta['har'], screenshot=meta['screenshot'],
|
||||
header=meta['header'],
|
||||
cookiejar=meta['cookiejar'], proxy=meta['proxy'],
|
||||
tags=meta['tags'],
|
||||
user_agent=meta['user_agent'], parent='scheduler', priority=40)
|
||||
if task_uuid:
|
||||
schedule.set_task(task_uuid)
|
||||
|
|
@ -1069,6 +1070,14 @@ class CrawlerSchedule:
|
|||
def _set_field(self, field, value):
|
||||
return r_crawler.hset(f'schedule:{self.uuid}', field, value)
|
||||
|
||||
def get_tags(self):
|
||||
return r_crawler.smembers(f'schedule:tags:{self.uuid}')
|
||||
|
||||
def set_tags(self, tags=[]):
|
||||
for tag in tags:
|
||||
r_crawler.sadd(f'schedule:tags:{self.uuid}', tag)
|
||||
# Tag.create_custom_tag(tag)
|
||||
|
||||
def get_meta(self, ui=False):
|
||||
meta = {
|
||||
'uuid': self.uuid,
|
||||
|
|
@ -1083,6 +1092,7 @@ class CrawlerSchedule:
|
|||
'cookiejar': self.get_cookiejar(),
|
||||
'header': self.get_header(),
|
||||
'proxy': self.get_proxy(),
|
||||
'tags': self.get_tags(),
|
||||
}
|
||||
status = self.get_status()
|
||||
if ui:
|
||||
|
|
@ -1098,6 +1108,7 @@ class CrawlerSchedule:
|
|||
meta = {'uuid': self.uuid,
|
||||
'url': self.get_url(),
|
||||
'user': self.get_user(),
|
||||
'tags': self.get_tags(),
|
||||
'next_run': self.get_next_run(r_str=True)}
|
||||
status = self.get_status()
|
||||
if isinstance(status, ScheduleStatus):
|
||||
|
|
@ -1106,7 +1117,7 @@ class CrawlerSchedule:
|
|||
return meta
|
||||
|
||||
def create(self, frequency, user, url,
|
||||
depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None):
|
||||
depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None, tags=[]):
|
||||
|
||||
if self.exists():
|
||||
raise Exception('Error: Monitor already exists')
|
||||
|
|
@ -1135,6 +1146,9 @@ class CrawlerSchedule:
|
|||
if user_agent:
|
||||
self._set_field('user_agent', user_agent)
|
||||
|
||||
if tags:
|
||||
self.set_tags(tags)
|
||||
|
||||
r_crawler.sadd('scheduler:schedules', self.uuid)
|
||||
|
||||
def delete(self):
|
||||
|
|
@ -1148,12 +1162,13 @@ class CrawlerSchedule:
|
|||
|
||||
# delete meta
|
||||
r_crawler.delete(f'schedule:{self.uuid}')
|
||||
r_crawler.delete(f'schedule:tags:{self.uuid}')
|
||||
r_crawler.srem('scheduler:schedules', self.uuid)
|
||||
|
||||
def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None):
|
||||
def create_schedule(frequency, user, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None, user_agent=None, tags=[]):
|
||||
schedule_uuid = gen_uuid()
|
||||
schedule = CrawlerSchedule(schedule_uuid)
|
||||
schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent)
|
||||
schedule.create(frequency, user, url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar, proxy=proxy, user_agent=user_agent, tags=tags)
|
||||
return schedule_uuid
|
||||
|
||||
# TODO sanityze UUID
|
||||
|
|
@ -1289,6 +1304,11 @@ def get_captures_status():
|
|||
status.append(meta)
|
||||
return status
|
||||
|
||||
def delete_captures():
|
||||
for capture_uuid in get_crawler_captures():
|
||||
capture = CrawlerCapture(capture_uuid)
|
||||
capture.delete()
|
||||
|
||||
##-- CRAWLER STATE --##
|
||||
|
||||
|
||||
|
|
@ -1371,6 +1391,14 @@ class CrawlerTask:
|
|||
def _set_field(self, field, value):
|
||||
return r_crawler.hset(f'crawler:task:{self.uuid}', field, value)
|
||||
|
||||
def get_tags(self):
|
||||
return r_crawler.smembers(f'crawler:task:tags:{self.uuid}')
|
||||
|
||||
def set_tags(self, tags):
|
||||
for tag in tags:
|
||||
r_crawler.sadd(f'crawler:task:tags:{self.uuid}', tag)
|
||||
# Tag.create_custom_tag(tag)
|
||||
|
||||
def get_meta(self):
|
||||
meta = {
|
||||
'uuid': self.uuid,
|
||||
|
|
@ -1385,6 +1413,7 @@ class CrawlerTask:
|
|||
'header': self.get_header(),
|
||||
'proxy': self.get_proxy(),
|
||||
'parent': self.get_parent(),
|
||||
'tags': self.get_tags(),
|
||||
}
|
||||
return meta
|
||||
|
||||
|
|
@ -1392,7 +1421,7 @@ class CrawlerTask:
|
|||
# TODO SANITIZE PRIORITY
|
||||
# PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100
|
||||
def create(self, url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None,
|
||||
user_agent=None, parent='manual', priority=0):
|
||||
user_agent=None, tags=[], parent='manual', priority=0):
|
||||
if self.exists():
|
||||
raise Exception('Error: Task already exists')
|
||||
|
||||
|
|
@ -1423,7 +1452,7 @@ class CrawlerTask:
|
|||
# TODO SANITIZE COOKIEJAR -> UUID
|
||||
|
||||
# Check if already in queue
|
||||
hash_query = get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header)
|
||||
hash_query = get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header, tags)
|
||||
if r_crawler.hexists(f'crawler:queue:hash', hash_query):
|
||||
self.uuid = r_crawler.hget(f'crawler:queue:hash', hash_query)
|
||||
return self.uuid
|
||||
|
|
@ -1444,6 +1473,9 @@ class CrawlerTask:
|
|||
if user_agent:
|
||||
self._set_field('user_agent', user_agent)
|
||||
|
||||
if tags:
|
||||
self.set_tags(tags)
|
||||
|
||||
r_crawler.hset('crawler:queue:hash', hash_query, self.uuid)
|
||||
self._set_field('hash', hash_query)
|
||||
r_crawler.zadd('crawler:queue', {self.uuid: priority})
|
||||
|
|
@ -1483,10 +1515,10 @@ class CrawlerTask:
|
|||
|
||||
|
||||
# TODO move to class ???
|
||||
def get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header):
|
||||
def get_task_hash(url, domain, depth, har, screenshot, priority, proxy, cookiejar, user_agent, header, tags):
|
||||
to_enqueue = {'domain': domain, 'depth': depth, 'har': har, 'screenshot': screenshot,
|
||||
'priority': priority, 'proxy': proxy, 'cookiejar': cookiejar, 'user_agent': user_agent,
|
||||
'header': header}
|
||||
'header': header, 'tags': tags}
|
||||
if priority != 0:
|
||||
to_enqueue['url'] = url
|
||||
return hashlib.sha512(pickle.dumps(to_enqueue)).hexdigest()
|
||||
|
|
@ -1502,7 +1534,7 @@ def add_task_to_lacus_queue():
|
|||
|
||||
# PRIORITY: discovery = 0/10, feeder = 10, manual = 50, auto = 40, test = 100
|
||||
def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=None, proxy=None,
|
||||
user_agent=None, parent='manual', priority=0, task_uuid=None):
|
||||
user_agent=None, tags=[], parent='manual', priority=0, task_uuid=None):
|
||||
if task_uuid:
|
||||
if CrawlerTask(task_uuid).exists():
|
||||
task_uuid = gen_uuid()
|
||||
|
|
@ -1510,7 +1542,7 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
|
|||
task_uuid = gen_uuid()
|
||||
task = CrawlerTask(task_uuid)
|
||||
task_uuid = task.create(url, depth=depth, har=har, screenshot=screenshot, header=header, cookiejar=cookiejar,
|
||||
proxy=proxy, user_agent=user_agent, parent=parent, priority=priority)
|
||||
proxy=proxy, user_agent=user_agent, tags=tags, parent=parent, priority=priority)
|
||||
return task_uuid
|
||||
|
||||
|
||||
|
|
@ -1586,15 +1618,17 @@ def api_add_crawler_task(data, user_id=None):
|
|||
if verify[1] != 200:
|
||||
return verify
|
||||
|
||||
tags = data.get('tags', [])
|
||||
|
||||
if frequency:
|
||||
# TODO verify user
|
||||
return create_schedule(frequency, user_id, url, depth=depth_limit, har=har, screenshot=screenshot, header=None,
|
||||
cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None), 200
|
||||
cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags), 200
|
||||
else:
|
||||
# TODO HEADERS
|
||||
# TODO USER AGENT
|
||||
return create_task(url, depth=depth_limit, har=har, screenshot=screenshot, header=None,
|
||||
cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None,
|
||||
cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags,
|
||||
parent='manual', priority=90), 200
|
||||
|
||||
|
||||
|
|
@ -1870,6 +1904,8 @@ def test_ail_crawlers():
|
|||
load_blacklist()
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# delete_captures()
|
||||
|
||||
# item_id = 'crawled/2023/02/20/data.gz'
|
||||
# item = Item(item_id)
|
||||
# content = item.get_content()
|
||||
|
|
|
|||
|
|
@ -96,7 +96,8 @@ def manual():
|
|||
is_manager_connected=crawlers.get_lacus_connection_metadata(),
|
||||
crawlers_types=crawlers_types,
|
||||
proxies=proxies,
|
||||
l_cookiejar=l_cookiejar)
|
||||
l_cookiejar=l_cookiejar,
|
||||
tags_selector_data=Tag.get_tags_selector_data())
|
||||
|
||||
|
||||
@crawler_splash.route("/crawlers/send_to_spider", methods=['POST'])
|
||||
|
|
@ -113,6 +114,34 @@ def send_to_spider():
|
|||
depth_limit = request.form.get('depth_limit')
|
||||
cookiejar_uuid = request.form.get('cookiejar')
|
||||
|
||||
# TAGS
|
||||
tags = request.form.get("tags", [])
|
||||
taxonomies_tags = request.form.get('taxonomies_tags')
|
||||
if taxonomies_tags:
|
||||
try:
|
||||
taxonomies_tags = json.loads(taxonomies_tags)
|
||||
except:
|
||||
taxonomies_tags = []
|
||||
else:
|
||||
taxonomies_tags = []
|
||||
galaxies_tags = request.form.get('galaxies_tags')
|
||||
if galaxies_tags:
|
||||
try:
|
||||
galaxies_tags = json.loads(galaxies_tags)
|
||||
except:
|
||||
galaxies_tags = []
|
||||
else:
|
||||
galaxies_tags = []
|
||||
# custom tags
|
||||
if tags:
|
||||
tags = tags.split()
|
||||
else:
|
||||
tags = []
|
||||
escaped = []
|
||||
for tag in tags:
|
||||
escaped.append(tag)
|
||||
tags = escaped + taxonomies_tags + galaxies_tags
|
||||
|
||||
# Frequency
|
||||
if request.form.get('crawler_scheduler'):
|
||||
frequency = request.form.get('frequency')
|
||||
|
|
@ -147,6 +176,8 @@ def send_to_spider():
|
|||
data['proxy'] = proxy
|
||||
if cookiejar_uuid:
|
||||
data['cookiejar'] = cookiejar_uuid
|
||||
if tags:
|
||||
data['tags'] = tags
|
||||
# print(data)
|
||||
res = crawlers.api_add_crawler_task(data, user_id=user_id)
|
||||
|
||||
|
|
@ -163,6 +194,7 @@ def scheduler_dashboard():
|
|||
# print(schedulers)
|
||||
# TODO list currently queued ?
|
||||
return render_template("crawler_scheduler_dashboard.html",
|
||||
bootstrap_label=bootstrap_label,
|
||||
schedulers=schedulers,
|
||||
is_manager_connected=crawlers.get_lacus_connection_metadata())
|
||||
|
||||
|
|
@ -176,6 +208,7 @@ def schedule_show():
|
|||
abort(404)
|
||||
meta = schedule.get_meta(ui=True)
|
||||
return render_template("crawler_schedule_uuid.html",
|
||||
bootstrap_label=bootstrap_label,
|
||||
meta=meta)
|
||||
|
||||
@crawler_splash.route("/crawlers/schedule/delete", methods=['GET'])
|
||||
|
|
|
|||
|
|
@ -8,14 +8,16 @@
|
|||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||
|
||||
</head>
|
||||
|
||||
|
|
@ -119,6 +121,21 @@
|
|||
|
||||
</div>
|
||||
|
||||
<div class="card bg-light my-4">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<b>Tags</b>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
|
||||
</div>
|
||||
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{%for tag in dict_tracker['tags']%}{{tag}} {%endfor%}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
{% include 'tags/block_tags_selector.html' %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -72,6 +72,14 @@
|
|||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Tags</th>
|
||||
<td>
|
||||
{%for tag in meta['tags']%}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
{%endfor%}
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,14 @@
|
|||
<tbody id="tbody_last_crawled">
|
||||
{% for meta in schedulers %}
|
||||
<tr>
|
||||
<td><a href="{{ url_for('crawler_splash.schedule_show') }}?uuid={{ meta['uuid'] }}">{{ meta['url'] }}</a></td>
|
||||
<td>
|
||||
<a href="{{ url_for('crawler_splash.schedule_show') }}?uuid={{ meta['uuid'] }}">{{ meta['url'] }}</a>
|
||||
<div>
|
||||
{% for tag in meta['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
{%endfor%}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ meta['status'] }}</td>
|
||||
<td>
|
||||
{% if not meta['next_run'] %}
|
||||
|
|
|
|||
Loading…
Reference in New Issue