mirror of https://github.com/CIRCL/AIL-framework
951 lines
35 KiB
Python
Executable File
951 lines
35 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
"""
|
|
API Helper
|
|
===================
|
|
|
|
|
|
"""
|
|
import base64
|
|
import gzip
|
|
import json
|
|
import os
|
|
import re
|
|
import redis
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
from datetime import datetime, timedelta
|
|
from urllib.parse import urlparse
|
|
|
|
from pyfaup.faup import Faup
|
|
|
|
# interact with splash_crawler API
|
|
import requests
|
|
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
|
|
|
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
|
import ConfigLoader
|
|
|
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'core/'))
|
|
import screen
|
|
|
|
config_loader = ConfigLoader.ConfigLoader()
|
|
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
|
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
|
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes"))
|
|
config_loader = None
|
|
|
|
# load crawler config
|
|
config_loader = ConfigLoader.ConfigLoader(config_file='crawlers.cfg')
|
|
splash_manager_url = config_loader.get_config_str('Splash_Manager', 'splash_url')
|
|
splash_api_key = config_loader.get_config_str('Splash_Manager', 'api_key')
|
|
config_loader = None
|
|
|
|
faup = Faup()
|
|
|
|
# # # # # # # #
|
|
# #
|
|
# COMMON #
|
|
# #
|
|
# # # # # # # #
|
|
|
|
def generate_uuid():
|
|
return str(uuid.uuid4()).replace('-', '')
|
|
|
|
def get_current_date():
|
|
return datetime.now().strftime("%Y%m%d")
|
|
|
|
##-- COMMON --#
|
|
|
|
################################################################################
|
|
|
|
# # TODO: handle prefix cookies
|
|
# # TODO: fill empty fields
|
|
def create_cookie_crawler(cookie_dict, domain, crawler_type='regular'):
|
|
# check cookie domain filed
|
|
if not 'domain' in cookie_dict:
|
|
cookie_dict['domain'] = '.{}'.format(domain)
|
|
|
|
# tor browser: disable secure cookie
|
|
if crawler_type=='onion':
|
|
cookie_dict['secure'] = False
|
|
|
|
# force cookie domain
|
|
# url = urlparse(browser_cookie['Host raw'])
|
|
# domain = url.netloc.split(':', 1)[0]
|
|
# cookie_dict['domain'] = '.{}'.format(domain)
|
|
|
|
# change expire date
|
|
cookie_dict['expires'] = (datetime.now() + timedelta(days=10)).strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
|
|
return cookie_dict
|
|
|
|
def load_crawler_cookies(cookiejar_uuid, domain, crawler_type='regular'):
|
|
cookies = get_cookiejar_cookies_list(cookiejar_uuid)
|
|
all_cookies = []
|
|
for cookie_dict in cookies:
|
|
all_cookies.append(create_cookie_crawler(cookie_dict, domain, crawler_type=crawler_type))
|
|
return all_cookies
|
|
|
|
################################################################################
|
|
|
|
def get_all_cookiejar():
|
|
r_serv_onion.smembers('cookiejar:all')
|
|
|
|
def get_global_cookiejar():
|
|
res = r_serv_onion.smembers('cookiejar:global')
|
|
if not res:
|
|
res = []
|
|
return res
|
|
|
|
def get_user_cookiejar(user_id):
|
|
res = r_serv_onion.smembers('cookiejar:user:{}'.format(user_id))
|
|
if not res:
|
|
res = []
|
|
return res
|
|
|
|
def exist_cookiejar(cookiejar_uuid):
|
|
return r_serv_onion.exists('cookiejar_metadata:{}'.format(cookiejar_uuid))
|
|
|
|
def create_cookiejar(user_id, level=1, description=None):
|
|
cookiejar_uuid = str(uuid.uuid4())
|
|
|
|
r_serv_onion.sadd('cookiejar:all', cookiejar_uuid)
|
|
if level==0:
|
|
r_serv_onion.sadd('cookiejar:user:{}'.format(user_id), cookiejar_uuid)
|
|
else:
|
|
r_serv_onion.sadd('cookiejar:global', cookiejar_uuid)
|
|
# metadata
|
|
r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'user_id', user_id)
|
|
r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'level', level)
|
|
r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'description', description)
|
|
r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'date', datetime.now().strftime("%Y%m%d"))
|
|
|
|
# if json_cookies:
|
|
# json_cookies = json.loads(json_cookies) # # TODO: catch Exception
|
|
# r_serv_onion.set('cookies:json_cookies:{}'.format(cookies_uuid), json.dumps(json_cookies))
|
|
#
|
|
# for cookie_dict in l_cookies:
|
|
# r_serv_onion.hset('cookies:manual_cookies:{}'.format(cookies_uuid), cookie_dict['name'], cookie_dict['value'])
|
|
return cookiejar_uuid
|
|
|
|
def delete_cookie_jar(cookiejar_uuid):
|
|
level = get_cookiejar_level(cookiejar_uuid)
|
|
if level == 0:
|
|
user_id = get_cookiejar_owner(cookiejar_uuid)
|
|
r_serv_onion.srem('cookiejar:user:{}'.format(user_id), cookiejar_uuid)
|
|
else:
|
|
r_serv_onion.srem('cookiejar:global', cookiejar_uuid)
|
|
|
|
r_serv_onion.delete('cookiejar_metadata:{}'.format(cookiejar_uuid))
|
|
|
|
def get_cookiejar_cookies_uuid(cookiejar_uuid):
|
|
res = r_serv_onion.smembers('cookiejar:{}:cookies:uuid'.format(cookiejar_uuid))
|
|
if not res:
|
|
res = []
|
|
return res
|
|
|
|
def get_cookiejar_cookies_list(cookiejar_uuid, add_cookie_uuid=False):
|
|
l_cookiejar = []
|
|
for cookie_uuid in get_cookiejar_cookies_uuid(cookiejar_uuid):
|
|
if add_cookie_uuid:
|
|
l_cookiejar.append((get_cookie_dict(cookie_uuid), cookie_uuid))
|
|
else:
|
|
l_cookiejar.append(get_cookie_dict(cookie_uuid))
|
|
return l_cookiejar
|
|
|
|
## Cookiejar metadata ##
|
|
def get_cookiejar_description(cookiejar_uuid):
|
|
return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'description')
|
|
|
|
def get_cookiejar_date(cookiejar_uuid):
|
|
return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'date')
|
|
|
|
def get_cookiejar_owner(cookiejar_uuid):
|
|
return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'user_id')
|
|
|
|
def get_cookiejar_date(cookiejar_uuid):
|
|
return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'date')
|
|
|
|
def get_cookiejar_level(cookiejar_uuid):
|
|
res = r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'level')
|
|
if not res:
|
|
res = 1
|
|
return int(res)
|
|
|
|
def get_cookiejar_metadata(cookiejar_uuid, level=False):
|
|
dict_cookiejar = {}
|
|
if exist_cookiejar(cookiejar_uuid):
|
|
dict_cookiejar['cookiejar_uuid'] = cookiejar_uuid
|
|
dict_cookiejar['description'] = get_cookiejar_description(cookiejar_uuid)
|
|
dict_cookiejar['date'] = get_cookiejar_date(cookiejar_uuid)
|
|
dict_cookiejar['user_id'] = get_cookiejar_owner(cookiejar_uuid)
|
|
if level:
|
|
dict_cookiejar['level'] = get_cookies_level(cookiejar_uuid)
|
|
return dict_cookiejar
|
|
|
|
def get_cookiejar_metadata_by_iterator(iter_cookiejar_uuid):
|
|
l_cookiejar_metadata = []
|
|
for cookiejar_uuid in iter_cookiejar_uuid:
|
|
l_cookiejar_metadata.append(get_cookiejar_metadata(cookiejar_uuid))
|
|
return l_cookiejar_metadata
|
|
|
|
def edit_cookiejar_description(cookiejar_uuid, description):
|
|
r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'description', description)
|
|
|
|
# # # # # # # #
|
|
# #
|
|
# COOKIES #
|
|
# #
|
|
# # # # # # # #
|
|
|
|
# # # #
|
|
# Cookies Fields:
|
|
# - name
|
|
# - value
|
|
# - path (optional)
|
|
# - domain (optional)
|
|
# - secure (optional)
|
|
# - httpOnly (optional)
|
|
# - text (optional)
|
|
# # # #
|
|
def get_cookie_all_keys_name():
|
|
return ['name', 'value', 'domain', 'path', 'httpOnly', 'secure']
|
|
|
|
def exists_cookie(cookie_uuid):
|
|
if int(r_serv_onion.scard('cookies:map:cookiejar:{}'.format(cookie_uuid))) > 0:
|
|
return True
|
|
return False
|
|
|
|
def get_cookie_value(cookie_uuid, name):
|
|
return r_serv_onion.hget('cookiejar:cookie:{}'.format(cookie_uuid), name)
|
|
|
|
def set_cookie_value(cookie_uuid, name, value):
|
|
r_serv_onion.hset('cookiejar:cookie:{}'.format(cookie_uuid), name, value)
|
|
|
|
def delete_cookie_value(cookie_uuid, name):
|
|
r_serv_onion.hdel('cookiejar:cookie:{}'.format(cookie_uuid), name)
|
|
|
|
def get_cookie_dict(cookie_uuid):
|
|
cookie_dict = {}
|
|
for key_name in r_serv_onion.hkeys('cookiejar:cookie:{}'.format(cookie_uuid)):
|
|
cookie_dict[key_name] = get_cookie_value(cookie_uuid, key_name)
|
|
return cookie_dict
|
|
|
|
# name, value, path=None, httpOnly=None, secure=None, domain=None, text=None
|
|
def add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict):
|
|
cookie_uuid = generate_uuid()
|
|
r_serv_onion.sadd('cookiejar:{}:cookies:uuid'.format(cookiejar_uuid), cookie_uuid)
|
|
r_serv_onion.sadd('cookies:map:cookiejar:{}'.format(cookie_uuid), cookiejar_uuid)
|
|
|
|
set_cookie_value(cookie_uuid, 'name', cookie_dict['name'])
|
|
set_cookie_value(cookie_uuid, 'value', cookie_dict['value'])
|
|
if 'path' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'path', cookie_dict['path'])
|
|
if 'httpOnly' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'httpOnly', cookie_dict['httpOnly'])
|
|
if 'secure' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'secure', cookie_dict['secure'])
|
|
if 'domain' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'domain', cookie_dict['domain'])
|
|
if 'text' in cookie_dict:
|
|
set_cookie_value(cookie_uuid, 'text', cookie_dict['text'])
|
|
return cookie_uuid
|
|
|
|
def add_cookies_to_cookiejar(cookiejar_uuid, l_cookies):
|
|
for cookie_dict in l_cookies:
|
|
add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict)
|
|
|
|
def delete_all_cookies_from_cookiejar(cookiejar_uuid):
|
|
for cookie_uuid in get_cookiejar_cookies_uuid(cookiejar_uuid):
|
|
delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid)
|
|
|
|
def delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid):
|
|
r_serv_onion.srem('cookiejar:{}:cookies:uuid'.format(cookiejar_uuid), cookie_uuid)
|
|
r_serv_onion.srem('cookies:map:cookiejar:{}'.format(cookie_uuid), cookiejar_uuid)
|
|
if not exists_cookie(cookie_uuid):
|
|
r_serv_onion.delete('cookiejar:cookie:{}'.format(cookie_uuid))
|
|
|
|
def edit_cookie(cookiejar_uuid, cookie_uuid, cookie_dict):
|
|
# delete old keys
|
|
for key_name in r_serv_onion.hkeys('cookiejar:cookie:{}'.format(cookie_uuid)):
|
|
if key_name not in cookie_dict:
|
|
delete_cookie_value(cookie_uuid, key_name)
|
|
# add new keys
|
|
cookie_all_keys_name = get_cookie_all_keys_name()
|
|
for key_name in cookie_dict:
|
|
if key_name in cookie_all_keys_name:
|
|
set_cookie_value(cookie_uuid, key_name, cookie_dict[key_name])
|
|
|
|
## - - ##
|
|
## Cookies import ## # TODO: add browser type ?
|
|
def import_cookies_from_json(json_cookies, cookiejar_uuid):
|
|
for cookie in json_cookies:
|
|
try:
|
|
cookie_dict = unpack_imported_json_cookie(cookie)
|
|
add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict)
|
|
except KeyError:
|
|
return {'error': 'Invalid cookie key, please submit a valid JSON', 'cookiejar_uuid': cookiejar_uuid}
|
|
|
|
# # TODO: add text field
|
|
def unpack_imported_json_cookie(json_cookie):
|
|
cookie_dict = {'name': json_cookie['Name raw'], 'value': json_cookie['Content raw']}
|
|
if 'Path raw' in json_cookie:
|
|
cookie_dict['path'] = json_cookie['Path raw']
|
|
if 'httpOnly' in json_cookie:
|
|
cookie_dict['httpOnly'] = json_cookie['HTTP only raw'] == 'true'
|
|
if 'secure' in json_cookie:
|
|
cookie_dict['secure'] = json_cookie['Send for'] == 'Encrypted connections only'
|
|
if 'Host raw' in json_cookie:
|
|
url = urlparse(json_cookie['Host raw'])
|
|
cookie_dict['domain'] = url.netloc.split(':', 1)[0]
|
|
return cookie_dict
|
|
|
|
def misp_cookie_import(misp_object, cookiejar_uuid):
|
|
pass
|
|
## - - ##
|
|
#### COOKIEJAR API ####
|
|
def api_import_cookies_from_json(json_cookies_str, cookiejar_uuid): # # TODO: add catch
|
|
json_cookies = json.loads(json_cookies_str)
|
|
res = import_cookies_from_json(json_cookies, cookiejar_uuid)
|
|
if res:
|
|
return (res, 400)
|
|
#### ####
|
|
|
|
#### COOKIES API ####
|
|
|
|
def api_verify_basic_cookiejar(cookiejar_uuid, user_id):
|
|
if not exist_cookiejar(cookiejar_uuid):
|
|
return ({'error': 'unknow cookiejar uuid', 'cookiejar_uuid': cookiejar_uuid}, 404)
|
|
level = get_cookiejar_level(cookiejar_uuid)
|
|
if level == 0: # # TODO: check if user is admin
|
|
cookie_owner = get_cookiejar_owner(cookiejar_uuid)
|
|
if cookie_owner != user_id:
|
|
return ({'error': 'The access to this cookiejar is restricted'}, 403)
|
|
|
|
def api_get_cookiejar_cookies(cookiejar_uuid, user_id):
|
|
res = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if res:
|
|
return res
|
|
res = get_cookiejar_cookies_list(cookiejar_uuid)
|
|
return (res, 200)
|
|
|
|
def api_edit_cookiejar_description(user_id, cookiejar_uuid, description):
|
|
res = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if res:
|
|
return res
|
|
edit_cookiejar_description(cookiejar_uuid, description)
|
|
return ({'cookiejar_uuid': cookiejar_uuid}, 200)
|
|
|
|
def api_get_cookiejar_cookies_with_uuid(cookiejar_uuid, user_id):
|
|
res = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if res:
|
|
return res
|
|
res = get_cookiejar_cookies_list(cookiejar_uuid, add_cookie_uuid=True)
|
|
return (res, 200)
|
|
|
|
def api_get_cookies_list_select(user_id):
|
|
l_cookiejar = []
|
|
for cookies_uuid in get_global_cookiejar():
|
|
l_cookiejar.append('{} : {}'.format(get_cookiejar_description(cookies_uuid), cookies_uuid))
|
|
for cookies_uuid in get_user_cookiejar(user_id):
|
|
l_cookiejar.append('{} : {}'.format(get_cookiejar_description(cookies_uuid), cookies_uuid))
|
|
return sorted(l_cookiejar)
|
|
|
|
def api_delete_cookie_from_cookiejar(user_id, cookiejar_uuid, cookie_uuid):
|
|
res = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if res:
|
|
return res
|
|
delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid)
|
|
return ({'cookiejar_uuid': cookiejar_uuid, 'cookie_uuid': cookie_uuid}, 200)
|
|
|
|
def api_delete_cookie_jar(user_id, cookiejar_uuid):
|
|
res = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if res:
|
|
return res
|
|
delete_cookie_jar(cookiejar_uuid)
|
|
return ({'cookiejar_uuid': cookiejar_uuid}, 200)
|
|
|
|
def api_edit_cookie(user_id, cookiejar_uuid, cookie_uuid, cookie_dict):
|
|
res = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if res:
|
|
return res
|
|
if 'name' not in cookie_dict or 'value' not in cookie_dict or cookie_dict['name'] == '':
|
|
({'error': 'cookie name or value not provided'}, 400)
|
|
edit_cookie(cookiejar_uuid, cookie_uuid, cookie_dict)
|
|
return (get_cookie_dict(cookie_uuid), 200)
|
|
|
|
def api_create_cookie(user_id, cookiejar_uuid, cookie_dict):
|
|
res = api_verify_basic_cookiejar(cookiejar_uuid, user_id)
|
|
if res:
|
|
return res
|
|
if 'name' not in cookie_dict or 'value' not in cookie_dict or cookie_dict['name'] == '':
|
|
({'error': 'cookie name or value not provided'}, 400)
|
|
res = add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict)
|
|
return (res, 200)
|
|
|
|
#### ####
|
|
|
|
# # # # # # # #
|
|
# #
|
|
# CRAWLER #
|
|
# #
|
|
# # # # # # # #
|
|
|
|
#### CRAWLER GLOBAL ####
|
|
|
|
def get_all_spash_crawler_status():
|
|
crawler_metadata = []
|
|
all_crawlers = r_cache.smembers('all_splash_crawlers')
|
|
for crawler in all_crawlers:
|
|
crawler_metadata.append(get_splash_crawler_status(crawler))
|
|
return crawler_metadata
|
|
|
|
def reset_all_spash_crawler_status():
|
|
r_cache.delete('all_splash_crawlers')
|
|
|
|
def get_splash_crawler_status(spash_url):
|
|
crawler_type = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'type')
|
|
crawling_domain = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'crawling_domain')
|
|
started_time = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'started_time')
|
|
status_info = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'status')
|
|
crawler_info = '{} - {}'.format(spash_url, started_time)
|
|
if status_info=='Waiting' or status_info=='Crawling':
|
|
status=True
|
|
else:
|
|
status=False
|
|
return {'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status, 'type': crawler_type}
|
|
|
|
def get_stats_last_crawled_domains(crawler_types, date):
|
|
statDomains = {}
|
|
for crawler_type in crawler_types:
|
|
stat_type = {}
|
|
stat_type['domains_up'] = r_serv_onion.scard('{}_up:{}'.format(crawler_type, date))
|
|
stat_type['domains_down'] = r_serv_onion.scard('{}_down:{}'.format(crawler_type, date))
|
|
stat_type['total'] = stat_type['domains_up'] + stat_type['domains_down']
|
|
stat_type['domains_queue'] = get_nb_elem_to_crawl_by_type(crawler_type)
|
|
statDomains[crawler_type] = stat_type
|
|
return statDomains
|
|
|
|
# # TODO: handle custom proxy
|
|
def get_splash_crawler_latest_stats():
|
|
now = datetime.now()
|
|
date = now.strftime("%Y%m%d")
|
|
return get_stats_last_crawled_domains(['onion', 'regular'], date)
|
|
|
|
##-- CRAWLER GLOBAL --##
|
|
|
|
#### CRAWLER TASK ####
|
|
def create_crawler_task(url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, cookiejar_uuid=None, user_agent=None):
|
|
|
|
crawler_config = {}
|
|
crawler_config['depth_limit'] = depth_limit
|
|
crawler_config['closespider_pagecount'] = max_pages
|
|
|
|
if screenshot:
|
|
crawler_config['png'] = True
|
|
else:
|
|
crawler_config['png'] = False
|
|
if har:
|
|
crawler_config['har'] = True
|
|
else:
|
|
crawler_config['har'] = False
|
|
|
|
if user_agent:
|
|
crawler_config['user_agent'] = user_agent
|
|
if cookiejar_uuid:
|
|
crawler_config['cookiejar_uuid'] = cookiejar_uuid
|
|
|
|
if auto_crawler:
|
|
crawler_mode = 'auto'
|
|
crawler_config['time'] = crawler_delta
|
|
else:
|
|
crawler_mode = 'manual'
|
|
|
|
# get crawler_mode
|
|
faup.decode(url)
|
|
unpack_url = faup.get()
|
|
## TODO: # FIXME: remove me
|
|
try:
|
|
domain = unpack_url['domain'].decode()
|
|
except:
|
|
domain = unpack_url['domain']
|
|
|
|
## TODO: # FIXME: remove me
|
|
try:
|
|
tld = unpack_url['tld'].decode()
|
|
except:
|
|
tld = unpack_url['tld']
|
|
if tld == 'onion':
|
|
crawler_type = 'onion'
|
|
else:
|
|
crawler_type = 'regular'
|
|
|
|
save_crawler_config(crawler_mode, crawler_type, crawler_config, domain, url=url)
|
|
send_url_to_crawl_in_queue(crawler_mode, crawler_type, url)
|
|
|
|
def save_crawler_config(crawler_mode, crawler_type, crawler_config, domain, url=None):
|
|
if crawler_mode == 'manual':
|
|
r_cache.set('crawler_config:{}:{}:{}'.format(crawler_mode, crawler_type, domain), json.dumps(crawler_config))
|
|
elif crawler_mode == 'auto':
|
|
r_serv_onion.set('crawler_config:{}:{}:{}:{}'.format(crawler_mode, crawler_type, domain, url), json.dumps(crawler_config))
|
|
|
|
def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url):
|
|
r_serv_onion.sadd('{}_crawler_priority_queue'.format(crawler_type), '{};{}'.format(url, crawler_mode))
|
|
# add auto crawled url for user UI
|
|
if crawler_mode == 'auto':
|
|
r_serv_onion.sadd('auto_crawler_url:{}'.format(crawler_type), url)
|
|
|
|
#### ####
|
|
#### CRAWLER TASK API ####
|
|
def api_create_crawler_task(user_id, url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, cookiejar_uuid=None, user_agent=None):
|
|
# validate url
|
|
if url is None or url=='' or url=='\n':
|
|
return ({'error':'invalid depth limit'}, 400)
|
|
|
|
if depth_limit:
|
|
try:
|
|
depth_limit = int(depth_limit)
|
|
if depth_limit < 0:
|
|
depth_limit = 0
|
|
except ValueError:
|
|
return ({'error':'invalid depth limit'}, 400)
|
|
if max_pages:
|
|
try:
|
|
max_pages = int(max_pages)
|
|
if max_pages < 1:
|
|
max_pages = 1
|
|
except ValueError:
|
|
return ({'error':'invalid max_pages limit'}, 400)
|
|
|
|
if auto_crawler:
|
|
try:
|
|
crawler_delta = int(crawler_delta)
|
|
if crawler_delta < 0:
|
|
return ({'error':'invalid delta bettween two pass of the crawler'}, 400)
|
|
except ValueError:
|
|
return ({'error':'invalid delta bettween two pass of the crawler'}, 400)
|
|
|
|
if cookiejar_uuid:
|
|
if not exist_cookiejar(cookiejar_uuid):
|
|
return ({'error': 'unknow cookiejar uuid', 'cookiejar_uuid': cookiejar_uuid}, 404)
|
|
level = get_cookiejar_level(cookiejar_uuid)
|
|
if level == 0: # # TODO: check if user is admin
|
|
cookie_owner = get_cookiejar_owner(cookiejar_uuid)
|
|
if cookie_owner != user_id:
|
|
return ({'error': 'The access to this cookiejar is restricted'}, 403)
|
|
|
|
create_crawler_task(url, screenshot=screenshot, har=har, depth_limit=depth_limit, max_pages=max_pages,
|
|
auto_crawler=auto_crawler, crawler_delta=crawler_delta, cookiejar_uuid=cookiejar_uuid, user_agent=user_agent)
|
|
return None
|
|
#### ####
|
|
|
|
def is_redirection(domain, last_url):
|
|
url = urlparse(last_url)
|
|
last_domain = url.netloc
|
|
last_domain = last_domain.split('.')
|
|
last_domain = '{}.{}'.format(last_domain[-2], last_domain[-1])
|
|
return domain != last_domain
|
|
|
|
# domain up
|
|
def create_domain_metadata(domain_type, domain, current_port, date, date_month):
|
|
# Add to global set
|
|
r_serv_onion.sadd('{}_up:{}'.format(domain_type, date), domain)
|
|
r_serv_onion.sadd('full_{}_up'.format(domain_type), domain)
|
|
r_serv_onion.sadd('month_{}_up:{}'.format(domain_type, date_month), domain)
|
|
|
|
# create onion metadata
|
|
if not r_serv_onion.exists('{}_metadata:{}'.format(domain_type, domain)):
|
|
r_serv_onion.hset('{}_metadata:{}'.format(domain_type, domain), 'first_seen', date)
|
|
r_serv_onion.hset('{}_metadata:{}'.format(domain_type, domain), 'last_check', date)
|
|
|
|
# Update domain port number
|
|
all_domain_ports = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'ports')
|
|
if all_domain_ports:
|
|
all_domain_ports = all_domain_ports.split(';')
|
|
else:
|
|
all_domain_ports = []
|
|
if current_port not in all_domain_ports:
|
|
all_domain_ports.append(current_port)
|
|
r_serv_onion.hset('{}_metadata:{}'.format(domain_type, domain), 'ports', ';'.join(all_domain_ports))
|
|
|
|
# add root_item to history
|
|
def add_domain_root_item(root_item, domain_type, domain, epoch_date, port):
|
|
# Create/Update crawler history
|
|
r_serv_onion.zadd('crawler_history_{}:{}:{}'.format(domain_type, domain, port), epoch_date, root_item)
|
|
|
|
def create_item_metadata(item_id, domain, url, port, item_father):
|
|
r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', item_father)
|
|
r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'domain', '{}:{}'.format(domain, port))
|
|
r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'real_link', url)
|
|
# add this item_id to his father
|
|
r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_id)
|
|
|
|
def create_item_id(item_dir, domain):
|
|
if len(domain) > 215:
|
|
UUID = domain[-215:]+str(uuid.uuid4())
|
|
else:
|
|
UUID = domain+str(uuid.uuid4())
|
|
return os.path.join(item_dir, UUID)
|
|
|
|
def save_crawled_item(item_id, item_content):
|
|
try:
|
|
gzipencoded = gzip.compress(item_content.encode())
|
|
gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
|
|
return gzip64encoded
|
|
except:
|
|
print("file error: {}".format(item_id))
|
|
return False
|
|
|
|
def save_har(har_dir, item_id, har_content):
|
|
if not os.path.exists(har_dir):
|
|
os.makedirs(har_dir)
|
|
item_id = item_id.split('/')[-1]
|
|
filename = os.path.join(har_dir, item_id + '.json')
|
|
with open(filename, 'w') as f:
|
|
f.write(json.dumps(har_content))
|
|
|
|
#### CRAWLER QUEUES ####
|
|
def get_crawler_queue_type_by_proxy(splash_name, proxy_type):
|
|
all_domain_type = []
|
|
if splash_name != 'default_splash' and splash_name != 'default_splash_tor':
|
|
all_domain_type.append(splash_name)
|
|
# check if can be used for discovery
|
|
if not is_splash_used_in_discovery(splash_name):
|
|
return all_domain_type
|
|
if proxy_type == 'tor':
|
|
all_domain_type.append('onion')
|
|
all_domain_type.append('regular')
|
|
# proxy_type = web
|
|
else:
|
|
all_domain_type.append('regular')
|
|
return all_domain_type
|
|
|
|
def get_elem_to_crawl_by_queue_type(l_queue_type):
|
|
## queues priority:
|
|
# 1 - priority queue
|
|
# 2 - discovery queue
|
|
# 3 - normal queue
|
|
##
|
|
all_queue_key = ['{}_crawler_priority_queue', '{}_crawler_discovery_queue', '{}_crawler_queue']
|
|
|
|
for queue_key in all_queue_key:
|
|
for queue_type in l_queue_type:
|
|
message = r_serv_onion.spop(queue_key.format(queue_type))
|
|
if message:
|
|
dict_to_crawl = {}
|
|
splitted = message.rsplit(';', 1)
|
|
if len(splitted) == 2:
|
|
url, item_id = splitted
|
|
item_id = item_id.replace(PASTES_FOLDER+'/', '')
|
|
else:
|
|
# # TODO: to check/refractor
|
|
item_id = None
|
|
url = message
|
|
return {'url': url, 'paste': item_id, 'type_service': queue_type, 'original_message': message}
|
|
return None
|
|
|
|
def get_nb_elem_to_crawl_by_type(queue_type):
|
|
nb = r_serv_onion.scard('{}_crawler_priority_queue'.format(queue_type))
|
|
nb += r_serv_onion.scard('{}_crawler_discovery_queue'.format(queue_type))
|
|
nb += r_serv_onion.scard('{}_crawler_queue'.format(queue_type))
|
|
return nb
|
|
|
|
#### ---- ####
|
|
|
|
# # # # # # # # # # # #
|
|
# #
|
|
# SPLASH MANAGER #
|
|
# #
|
|
# # # # # # # # # # # #
|
|
|
|
def get_splash_manager_url(reload=False): # TODO: add in db config
|
|
return splash_manager_url
|
|
|
|
def get_splash_api_key(reload=False): # TODO: add in db config
|
|
return splash_api_key
|
|
|
|
def get_hidden_splash_api_key(): # TODO: add in db config
|
|
key = get_splash_api_key()
|
|
if len(key)==41:
|
|
return f'{key[:4]}*********************************{key[-4:]}'
|
|
else:
|
|
return None
|
|
|
|
def get_splash_url_from_manager_url(splash_manager_url, splash_port):
|
|
url = urlparse(splash_manager_url)
|
|
host = url.netloc.split(':', 1)[0]
|
|
return '{}:{}'.format(host, splash_port)
|
|
|
|
def is_splash_used_in_discovery(splash_name):
|
|
res = r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'discovery_queue')
|
|
if res == 'True':
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def restart_splash_docker(splash_url):
|
|
splash_port = splash_url.split(':')[-1]
|
|
return _restart_splash_docker(splash_port)
|
|
|
|
def is_splash_manager_connected(delta_check=30):
|
|
last_check = r_cache.hget('crawler:splash:manager', 'last_check')
|
|
if last_check:
|
|
if int(time.time()) - int(last_check) > delta_check:
|
|
ping_splash_manager()
|
|
else:
|
|
ping_splash_manager()
|
|
res = r_cache.hget('crawler:splash:manager', 'connected')
|
|
return res == 'True'
|
|
|
|
def update_splash_manager_connection_status(is_connected):
|
|
r_cache.hset('crawler:splash:manager', 'connected', is_connected)
|
|
r_cache.hset('crawler:splash:manager', 'last_check', int(time.time()))
|
|
|
|
## API ##
|
|
def ping_splash_manager():
|
|
try:
|
|
req = requests.get('{}/api/v1/ping'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False)
|
|
if req.status_code == 200:
|
|
update_splash_manager_connection_status(True)
|
|
return True
|
|
else:
|
|
print(req.json())
|
|
update_splash_manager_connection_status(False)
|
|
return False
|
|
except requests.exceptions.ConnectionError:
|
|
pass
|
|
# splash manager unreachable
|
|
update_splash_manager_connection_status(False)
|
|
return False
|
|
|
|
def get_splash_manager_session_uuid():
|
|
try:
|
|
req = requests.get('{}/api/v1/get/session_uuid'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False)
|
|
if req.status_code == 200:
|
|
res = req.json()
|
|
if res:
|
|
return res['session_uuid']
|
|
else:
|
|
print(req.json())
|
|
except requests.exceptions.ConnectionError:
|
|
# splash manager unreachable
|
|
update_splash_manager_connection_status(False)
|
|
|
|
def get_all_splash_manager_containers_name():
|
|
req = requests.get('{}/api/v1/get/splash/name/all'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False)
|
|
if req.status_code == 200:
|
|
return req.json()
|
|
else:
|
|
print(req.json())
|
|
|
|
def get_all_splash_manager_proxies():
|
|
req = requests.get('{}/api/v1/get/proxies/all'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False)
|
|
if req.status_code == 200:
|
|
return req.json()
|
|
else:
|
|
print(req.json())
|
|
|
|
def _restart_splash_docker(splash_port):
|
|
dict_to_send = {'docker_port': splash_port}
|
|
req = requests.post('{}/api/v1/splash/restart'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False, json=dict_to_send)
|
|
if req.status_code == 200:
|
|
return req.json()
|
|
else:
|
|
print(req.json())
|
|
## -- ##
|
|
|
|
## SPLASH ##
|
|
def get_all_splash(r_list=False):
|
|
res = r_serv_onion.smembers('all_splash')
|
|
if res:
|
|
if r_list:
|
|
return list(res)
|
|
else:
|
|
return res
|
|
else:
|
|
return []
|
|
|
|
def get_splash_proxy(splash_name):
|
|
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'proxy')
|
|
|
|
def get_splash_all_url(splash_name, r_list=False):
|
|
res = r_serv_onion.smembers('splash:url:{}'.format(splash_name))
|
|
if res:
|
|
if r_list:
|
|
return list(res)
|
|
else:
|
|
return res
|
|
else:
|
|
return []
|
|
|
|
def get_splash_name_by_url(splash_url):
|
|
return r_serv_onion.get('splash:map:url:name:{}'.format(splash_url))
|
|
|
|
def get_splash_crawler_type(splash_name):
|
|
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'crawler_type')
|
|
|
|
def get_splash_crawler_description(splash_name):
|
|
return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'description')
|
|
|
|
def get_splash_crawler_metadata(splash_name):
|
|
dict_splash = {}
|
|
dict_splash['proxy'] = get_splash_proxy(splash_name)
|
|
dict_splash['type'] = get_splash_crawler_type(splash_name)
|
|
dict_splash['description'] = get_splash_crawler_description(splash_name)
|
|
return dict_splash
|
|
|
|
def get_all_splash_crawler_metadata():
|
|
dict_splash = {}
|
|
for splash_name in get_all_splash():
|
|
dict_splash[splash_name] = get_splash_crawler_metadata(splash_name)
|
|
return dict_splash
|
|
|
|
def get_all_splash_by_proxy(proxy_name, r_list=False):
|
|
res = r_serv_onion.smembers('proxy:splash:{}'.format(proxy_name))
|
|
if res:
|
|
if r_list:
|
|
return list(res)
|
|
else:
|
|
return res
|
|
else:
|
|
return []
|
|
|
|
def delete_all_splash_containers():
|
|
for splash_name in get_all_splash():
|
|
delete_splash_container(splash_name)
|
|
|
|
def delete_splash_container(splash_name):
|
|
r_serv_onion.srem('proxy:splash:{}'.format(get_splash_proxy(splash_name)), splash_name)
|
|
r_serv_onion.delete('splash:metadata:{}'.format(splash_name))
|
|
|
|
for splash_url in get_splash_all_url(splash_name):
|
|
r_serv_onion.delete('splash:map:url:name:{}'.format(splash_url), splash_name)
|
|
r_serv_onion.srem('splash:url:{}'.format(splash_name), splash_url)
|
|
r_serv_onion.srem('all_splash', splash_name)
|
|
## -- ##
|
|
|
|
## PROXY ##
|
|
def get_all_proxies(r_list=False):
|
|
res = r_serv_onion.smembers('all_proxy')
|
|
if res:
|
|
return list(res)
|
|
else:
|
|
return []
|
|
|
|
def delete_all_proxies():
|
|
for proxy_name in get_all_proxies():
|
|
delete_proxy(proxy_name)
|
|
|
|
def get_proxy_host(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'host')
|
|
|
|
def get_proxy_port(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'port')
|
|
|
|
def get_proxy_type(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'type')
|
|
|
|
def get_proxy_crawler_type(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'crawler_type')
|
|
|
|
def get_proxy_description(proxy_name):
|
|
return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'description')
|
|
|
|
def get_proxy_metadata(proxy_name):
|
|
meta_dict = {}
|
|
meta_dict['host'] = get_proxy_host(proxy_name)
|
|
meta_dict['port'] = get_proxy_port(proxy_name)
|
|
meta_dict['type'] = get_proxy_type(proxy_name)
|
|
meta_dict['crawler_type'] = get_proxy_crawler_type(proxy_name)
|
|
meta_dict['description'] = get_proxy_description(proxy_name)
|
|
return meta_dict
|
|
|
|
def get_all_proxies_metadata():
|
|
all_proxy_dict = {}
|
|
for proxy_name in get_all_proxies():
|
|
all_proxy_dict[proxy_name] = get_proxy_metadata(proxy_name)
|
|
return all_proxy_dict
|
|
|
|
def set_proxy_used_in_discovery(proxy_name, value):
|
|
r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'discovery_queue', value)
|
|
|
|
def delete_proxy(proxy_name): # # TODO: force delete (delete all proxy)
|
|
proxy_splash = get_all_splash_by_proxy(proxy_name)
|
|
if proxy_splash:
|
|
print('error, a splash container is using this proxy')
|
|
r_serv_onion.delete('proxy:metadata:{}'.format(proxy_name))
|
|
r_serv_onion.srem('all_proxy', proxy_name)
|
|
## -- ##
|
|
|
|
## LOADER ##
|
|
def load_all_splash_containers():
|
|
all_splash_containers_name = get_all_splash_manager_containers_name()
|
|
for splash_name in all_splash_containers_name:
|
|
r_serv_onion.sadd('all_splash', splash_name)
|
|
|
|
proxy = all_splash_containers_name[splash_name]['proxy']
|
|
if not proxy:
|
|
proxy = {'name': 'no_proxy', 'crawler_type': 'web'}
|
|
|
|
r_serv_onion.sadd('proxy:splash:{}'.format(proxy['name']), splash_name)
|
|
|
|
r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'crawler_type', proxy['crawler_type'])
|
|
r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'proxy', proxy['name'])
|
|
description = all_splash_containers_name[splash_name].get('description', None)
|
|
if description:
|
|
r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'description', description)
|
|
|
|
for port in all_splash_containers_name[splash_name]['ports']:
|
|
splash_url = get_splash_url_from_manager_url(get_splash_manager_url(), port)
|
|
r_serv_onion.sadd('splash:url:{}'.format(splash_name), splash_url)
|
|
r_serv_onion.set('splash:map:url:name:{}'.format(splash_url), splash_name)
|
|
|
|
def load_all_proxy():
|
|
delete_all_proxies()
|
|
all_proxies = get_all_splash_manager_proxies()
|
|
for proxy_name in all_proxies:
|
|
proxy_dict = all_proxies[proxy_name]
|
|
r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'host', proxy_dict['host'])
|
|
r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'port', proxy_dict['port'])
|
|
r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'type', proxy_dict['type'])
|
|
r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'crawler_type', proxy_dict['crawler_type'])
|
|
description = all_proxies[proxy_name].get('description', None)
|
|
if description:
|
|
r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'description', description)
|
|
r_serv_onion.sadd('all_proxy', proxy_name)
|
|
|
|
def reload_splash_and_proxies_list():
|
|
if ping_splash_manager():
|
|
# LOAD SPLASH containers
|
|
delete_all_splash_containers()
|
|
load_all_splash_containers()
|
|
# LOAD PROXIES containers
|
|
delete_all_proxies()
|
|
load_all_proxy()
|
|
return True
|
|
else:
|
|
return False
|
|
# # TODO: kill crawler screen ?
|
|
## -- ##
|
|
|
|
## SPLASH CONTROLLER ##
|
|
def launch_ail_splash_crawler(splash_url, script_options=''):
|
|
screen_name = 'Crawler_AIL'
|
|
dir_project = os.environ['AIL_HOME']
|
|
script_location = os.path.join(os.environ['AIL_BIN'])
|
|
script_name = 'Crawler.py'
|
|
screen.create_screen(screen_name)
|
|
screen.launch_uniq_windows_script(screen_name, splash_url, dir_project, script_location, script_name, script_options=script_options, kill_previous_windows=True)
|
|
|
|
|
|
## -- ##
|
|
|
|
#### ---- ####
|
|
|
|
#### CRAWLER PROXY ####
|
|
|
|
#### ---- ####
|