#!/usr/bin/python3 """ API Helper =================== """ import base64 import gzip import json import os import re import redis import sys import time import uuid from datetime import datetime, timedelta from urllib.parse import urlparse from pyfaup.faup import Faup # interact with splash_crawler API import requests requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_BIN'], 'core/')) import screen config_loader = ConfigLoader.ConfigLoader() r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") r_cache = config_loader.get_redis_conn("Redis_Cache") PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) config_loader = None faup = Faup() # # # # # # # # # # # COMMON # # # # # # # # # # # def generate_uuid(): return str(uuid.uuid4()).replace('-', '') # # TODO: remove me ? def get_current_date(): return datetime.now().strftime("%Y%m%d") def is_valid_onion_domain(domain): if not domain.endswith('.onion'): return False domain = domain.replace('.onion', '', 1) if len(domain) == 16: # v2 address r_onion = r'[a-z0-9]{16}' if re.match(r_onion, domain): return True elif len(domain) == 56: # v3 address r_onion = r'[a-z0-9]{56}' if re.fullmatch(r_onion, domain): return True return False ################################################################################ # # TODO: handle prefix cookies # # TODO: fill empty fields def create_cookie_crawler(cookie_dict, domain, crawler_type='regular'): # check cookie domain filed if not 'domain' in cookie_dict: cookie_dict['domain'] = '.{}'.format(domain) # tor browser: disable secure cookie if crawler_type=='onion': cookie_dict['secure'] = False # force cookie domain # url = urlparse(browser_cookie['Host raw']) # domain = url.netloc.split(':', 1)[0] # cookie_dict['domain'] = '.{}'.format(domain) # change expire date cookie_dict['expires'] = (datetime.now() + timedelta(days=10)).strftime('%Y-%m-%dT%H:%M:%S') + 'Z' return cookie_dict def load_crawler_cookies(cookiejar_uuid, domain, crawler_type='regular'): cookies = get_cookiejar_cookies_list(cookiejar_uuid) all_cookies = [] for cookie_dict in cookies: all_cookies.append(create_cookie_crawler(cookie_dict, domain, crawler_type=crawler_type)) return all_cookies ################################################################################ def get_all_cookiejar(): r_serv_onion.smembers('cookiejar:all') def get_global_cookiejar(): res = r_serv_onion.smembers('cookiejar:global') if not res: res = [] return res def get_user_cookiejar(user_id): res = r_serv_onion.smembers('cookiejar:user:{}'.format(user_id)) if not res: res = [] return res def exist_cookiejar(cookiejar_uuid): return r_serv_onion.exists('cookiejar_metadata:{}'.format(cookiejar_uuid)) def create_cookiejar(user_id, level=1, description=None): cookiejar_uuid = str(uuid.uuid4()) r_serv_onion.sadd('cookiejar:all', cookiejar_uuid) if level==0: r_serv_onion.sadd('cookiejar:user:{}'.format(user_id), cookiejar_uuid) else: r_serv_onion.sadd('cookiejar:global', cookiejar_uuid) # metadata r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'user_id', user_id) r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'level', level) r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'description', description) r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'date', datetime.now().strftime("%Y%m%d")) # if json_cookies: # json_cookies = json.loads(json_cookies) # # TODO: catch Exception # r_serv_onion.set('cookies:json_cookies:{}'.format(cookies_uuid), json.dumps(json_cookies)) # # for cookie_dict in l_cookies: # r_serv_onion.hset('cookies:manual_cookies:{}'.format(cookies_uuid), cookie_dict['name'], cookie_dict['value']) return cookiejar_uuid def delete_cookie_jar(cookiejar_uuid): level = get_cookiejar_level(cookiejar_uuid) if level == 0: user_id = get_cookiejar_owner(cookiejar_uuid) r_serv_onion.srem('cookiejar:user:{}'.format(user_id), cookiejar_uuid) else: r_serv_onion.srem('cookiejar:global', cookiejar_uuid) r_serv_onion.delete('cookiejar_metadata:{}'.format(cookiejar_uuid)) def get_cookiejar_cookies_uuid(cookiejar_uuid): res = r_serv_onion.smembers('cookiejar:{}:cookies:uuid'.format(cookiejar_uuid)) if not res: res = [] return res def get_cookiejar_cookies_list(cookiejar_uuid, add_cookie_uuid=False): l_cookiejar = [] for cookie_uuid in get_cookiejar_cookies_uuid(cookiejar_uuid): if add_cookie_uuid: l_cookiejar.append((get_cookie_dict(cookie_uuid), cookie_uuid)) else: l_cookiejar.append(get_cookie_dict(cookie_uuid)) return l_cookiejar ## Cookiejar metadata ## def get_cookiejar_description(cookiejar_uuid): return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'description') def get_cookiejar_date(cookiejar_uuid): return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'date') def get_cookiejar_owner(cookiejar_uuid): return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'user_id') def get_cookiejar_date(cookiejar_uuid): return r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'date') def get_cookiejar_level(cookiejar_uuid): res = r_serv_onion.hget('cookiejar_metadata:{}'.format(cookiejar_uuid), 'level') if not res: res = 1 return int(res) def get_cookiejar_metadata(cookiejar_uuid, level=False): dict_cookiejar = {} if exist_cookiejar(cookiejar_uuid): dict_cookiejar['cookiejar_uuid'] = cookiejar_uuid dict_cookiejar['description'] = get_cookiejar_description(cookiejar_uuid) dict_cookiejar['date'] = get_cookiejar_date(cookiejar_uuid) dict_cookiejar['user_id'] = get_cookiejar_owner(cookiejar_uuid) if level: dict_cookiejar['level'] = get_cookies_level(cookiejar_uuid) return dict_cookiejar def get_cookiejar_metadata_by_iterator(iter_cookiejar_uuid): l_cookiejar_metadata = [] for cookiejar_uuid in iter_cookiejar_uuid: l_cookiejar_metadata.append(get_cookiejar_metadata(cookiejar_uuid)) return l_cookiejar_metadata def edit_cookiejar_description(cookiejar_uuid, description): r_serv_onion.hset('cookiejar_metadata:{}'.format(cookiejar_uuid), 'description', description) # # # # # # # # # # # COOKIES # # # # # # # # # # # # # # # # Cookies Fields: # - name # - value # - path (optional) # - domain (optional) # - secure (optional) # - httpOnly (optional) # - text (optional) # # # # def get_cookie_all_keys_name(): return ['name', 'value', 'domain', 'path', 'httpOnly', 'secure'] def exists_cookie(cookie_uuid): if int(r_serv_onion.scard('cookies:map:cookiejar:{}'.format(cookie_uuid))) > 0: return True return False def get_cookie_value(cookie_uuid, name): return r_serv_onion.hget('cookiejar:cookie:{}'.format(cookie_uuid), name) def set_cookie_value(cookie_uuid, name, value): r_serv_onion.hset('cookiejar:cookie:{}'.format(cookie_uuid), name, value) def delete_cookie_value(cookie_uuid, name): r_serv_onion.hdel('cookiejar:cookie:{}'.format(cookie_uuid), name) def get_cookie_dict(cookie_uuid): cookie_dict = {} for key_name in r_serv_onion.hkeys('cookiejar:cookie:{}'.format(cookie_uuid)): cookie_dict[key_name] = get_cookie_value(cookie_uuid, key_name) return cookie_dict # name, value, path=None, httpOnly=None, secure=None, domain=None, text=None def add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict): cookie_uuid = generate_uuid() r_serv_onion.sadd('cookiejar:{}:cookies:uuid'.format(cookiejar_uuid), cookie_uuid) r_serv_onion.sadd('cookies:map:cookiejar:{}'.format(cookie_uuid), cookiejar_uuid) set_cookie_value(cookie_uuid, 'name', cookie_dict['name']) set_cookie_value(cookie_uuid, 'value', cookie_dict['value']) if 'path' in cookie_dict: set_cookie_value(cookie_uuid, 'path', cookie_dict['path']) if 'httpOnly' in cookie_dict: set_cookie_value(cookie_uuid, 'httpOnly', cookie_dict['httpOnly']) if 'secure' in cookie_dict: set_cookie_value(cookie_uuid, 'secure', cookie_dict['secure']) if 'domain' in cookie_dict: set_cookie_value(cookie_uuid, 'domain', cookie_dict['domain']) if 'text' in cookie_dict: set_cookie_value(cookie_uuid, 'text', cookie_dict['text']) return cookie_uuid def add_cookies_to_cookiejar(cookiejar_uuid, l_cookies): for cookie_dict in l_cookies: add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict) def delete_all_cookies_from_cookiejar(cookiejar_uuid): for cookie_uuid in get_cookiejar_cookies_uuid(cookiejar_uuid): delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid) def delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid): r_serv_onion.srem('cookiejar:{}:cookies:uuid'.format(cookiejar_uuid), cookie_uuid) r_serv_onion.srem('cookies:map:cookiejar:{}'.format(cookie_uuid), cookiejar_uuid) if not exists_cookie(cookie_uuid): r_serv_onion.delete('cookiejar:cookie:{}'.format(cookie_uuid)) def edit_cookie(cookiejar_uuid, cookie_uuid, cookie_dict): # delete old keys for key_name in r_serv_onion.hkeys('cookiejar:cookie:{}'.format(cookie_uuid)): if key_name not in cookie_dict: delete_cookie_value(cookie_uuid, key_name) # add new keys cookie_all_keys_name = get_cookie_all_keys_name() for key_name in cookie_dict: if key_name in cookie_all_keys_name: set_cookie_value(cookie_uuid, key_name, cookie_dict[key_name]) ## - - ## ## Cookies import ## # TODO: add browser type ? def import_cookies_from_json(json_cookies, cookiejar_uuid): for cookie in json_cookies: try: cookie_dict = unpack_imported_json_cookie(cookie) add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict) except KeyError: return {'error': 'Invalid cookie key, please submit a valid JSON', 'cookiejar_uuid': cookiejar_uuid} # # TODO: add text field def unpack_imported_json_cookie(json_cookie): cookie_dict = {'name': json_cookie['Name raw'], 'value': json_cookie['Content raw']} if 'Path raw' in json_cookie: cookie_dict['path'] = json_cookie['Path raw'] if 'httpOnly' in json_cookie: cookie_dict['httpOnly'] = json_cookie['HTTP only raw'] == 'true' if 'secure' in json_cookie: cookie_dict['secure'] = json_cookie['Send for'] == 'Encrypted connections only' if 'Host raw' in json_cookie: url = urlparse(json_cookie['Host raw']) cookie_dict['domain'] = url.netloc.split(':', 1)[0] return cookie_dict def misp_cookie_import(misp_object, cookiejar_uuid): pass ## - - ## #### COOKIEJAR API #### def api_import_cookies_from_json(json_cookies_str, cookiejar_uuid): # # TODO: add catch json_cookies = json.loads(json_cookies_str) res = import_cookies_from_json(json_cookies, cookiejar_uuid) if res: return (res, 400) #### #### #### COOKIES API #### def api_verify_basic_cookiejar(cookiejar_uuid, user_id): if not exist_cookiejar(cookiejar_uuid): return ({'error': 'unknow cookiejar uuid', 'cookiejar_uuid': cookiejar_uuid}, 404) level = get_cookiejar_level(cookiejar_uuid) if level == 0: # # TODO: check if user is admin cookie_owner = get_cookiejar_owner(cookiejar_uuid) if cookie_owner != user_id: return ({'error': 'The access to this cookiejar is restricted'}, 403) def api_get_cookiejar_cookies(cookiejar_uuid, user_id): res = api_verify_basic_cookiejar(cookiejar_uuid, user_id) if res: return res res = get_cookiejar_cookies_list(cookiejar_uuid) return (res, 200) def api_edit_cookiejar_description(user_id, cookiejar_uuid, description): res = api_verify_basic_cookiejar(cookiejar_uuid, user_id) if res: return res edit_cookiejar_description(cookiejar_uuid, description) return ({'cookiejar_uuid': cookiejar_uuid}, 200) def api_get_cookiejar_cookies_with_uuid(cookiejar_uuid, user_id): res = api_verify_basic_cookiejar(cookiejar_uuid, user_id) if res: return res res = get_cookiejar_cookies_list(cookiejar_uuid, add_cookie_uuid=True) return (res, 200) def api_get_cookies_list_select(user_id): l_cookiejar = [] for cookies_uuid in get_global_cookiejar(): l_cookiejar.append('{} : {}'.format(get_cookiejar_description(cookies_uuid), cookies_uuid)) for cookies_uuid in get_user_cookiejar(user_id): l_cookiejar.append('{} : {}'.format(get_cookiejar_description(cookies_uuid), cookies_uuid)) return sorted(l_cookiejar) def api_delete_cookie_from_cookiejar(user_id, cookiejar_uuid, cookie_uuid): res = api_verify_basic_cookiejar(cookiejar_uuid, user_id) if res: return res delete_cookie_from_cookiejar(cookiejar_uuid, cookie_uuid) return ({'cookiejar_uuid': cookiejar_uuid, 'cookie_uuid': cookie_uuid}, 200) def api_delete_cookie_jar(user_id, cookiejar_uuid): res = api_verify_basic_cookiejar(cookiejar_uuid, user_id) if res: return res delete_cookie_jar(cookiejar_uuid) return ({'cookiejar_uuid': cookiejar_uuid}, 200) def api_edit_cookie(user_id, cookiejar_uuid, cookie_uuid, cookie_dict): res = api_verify_basic_cookiejar(cookiejar_uuid, user_id) if res: return res if 'name' not in cookie_dict or 'value' not in cookie_dict or cookie_dict['name'] == '': ({'error': 'cookie name or value not provided'}, 400) edit_cookie(cookiejar_uuid, cookie_uuid, cookie_dict) return (get_cookie_dict(cookie_uuid), 200) def api_create_cookie(user_id, cookiejar_uuid, cookie_dict): res = api_verify_basic_cookiejar(cookiejar_uuid, user_id) if res: return res if 'name' not in cookie_dict or 'value' not in cookie_dict or cookie_dict['name'] == '': ({'error': 'cookie name or value not provided'}, 400) res = add_cookie_to_cookiejar(cookiejar_uuid, cookie_dict) return (res, 200) #### #### # # # # # # # # # # # CRAWLER # # # # # # # # # # # #### CRAWLER GLOBAL #### def get_all_spash_crawler_status(): crawler_metadata = [] all_crawlers = r_cache.smembers('all_splash_crawlers') for crawler in all_crawlers: crawler_metadata.append(get_splash_crawler_status(crawler)) return crawler_metadata def reset_all_spash_crawler_status(): r_cache.delete('all_splash_crawlers') def get_splash_crawler_status(spash_url): crawler_type = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'type') crawling_domain = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'crawling_domain') started_time = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'started_time') status_info = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'status') crawler_info = '{} - {}'.format(spash_url, started_time) if status_info=='Waiting' or status_info=='Crawling': status=True else: status=False return {'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status, 'type': crawler_type} def get_stats_last_crawled_domains(crawler_types, date): statDomains = {} for crawler_type in crawler_types: stat_type = {} stat_type['domains_up'] = r_serv_onion.scard('{}_up:{}'.format(crawler_type, date)) stat_type['domains_down'] = r_serv_onion.scard('{}_down:{}'.format(crawler_type, date)) stat_type['total'] = stat_type['domains_up'] + stat_type['domains_down'] stat_type['domains_queue'] = get_nb_elem_to_crawl_by_type(crawler_type) statDomains[crawler_type] = stat_type return statDomains # # TODO: handle custom proxy def get_splash_crawler_latest_stats(): now = datetime.now() date = now.strftime("%Y%m%d") return get_stats_last_crawled_domains(['onion', 'regular'], date) def get_nb_crawlers_to_launch_by_splash_name(splash_name): res = r_serv_onion.hget('all_crawlers_to_launch', splash_name) if res: return int(res) else: return 0 def get_all_crawlers_to_launch_splash_name(): return r_serv_onion.hkeys('all_crawlers_to_launch') def get_nb_crawlers_to_launch(): nb_crawlers_to_launch = r_serv_onion.hgetall('all_crawlers_to_launch') for splash_name in nb_crawlers_to_launch: nb_crawlers_to_launch[splash_name] = int(nb_crawlers_to_launch[splash_name]) return nb_crawlers_to_launch def get_nb_crawlers_to_launch_ui(): nb_crawlers_to_launch = get_nb_crawlers_to_launch() for splash_name in get_all_splash(): if splash_name not in nb_crawlers_to_launch: nb_crawlers_to_launch[splash_name] = 0 return nb_crawlers_to_launch def set_nb_crawlers_to_launch(dict_splash_name): r_serv_onion.delete('all_crawlers_to_launch') for splash_name in dict_splash_name: r_serv_onion.hset('all_crawlers_to_launch', splash_name, int(dict_splash_name[splash_name])) relaunch_crawlers() def relaunch_crawlers(): all_crawlers_to_launch = get_nb_crawlers_to_launch() for splash_name in all_crawlers_to_launch: nb_crawlers = int(all_crawlers_to_launch[splash_name]) all_crawler_urls = get_splash_all_url(splash_name, r_list=True) if nb_crawlers > len(all_crawler_urls): print('Error, can\'t launch all Splash Dockers') print('Please launch {} additional {} Dockers'.format( nb_crawlers - len(all_crawler_urls), splash_name)) nb_crawlers = len(all_crawler_urls) reset_all_spash_crawler_status() for i in range(0, int(nb_crawlers)): splash_url = all_crawler_urls[i] print(all_crawler_urls[i]) launch_ail_splash_crawler(splash_url, script_options='{}'.format(splash_url)) def api_set_nb_crawlers_to_launch(dict_splash_name): # TODO: check if is dict dict_crawlers_to_launch = {} all_splash = get_all_splash() crawlers_to_launch = list(all_splash & set(dict_splash_name.keys())) for splash_name in crawlers_to_launch: try: nb_to_launch = int(dict_splash_name.get(splash_name, 0)) if nb_to_launch < 0: return ({'error':'The number of crawlers to launch is negative'}, 400) except: return ({'error':'invalid number of crawlers to launch'}, 400) if nb_to_launch > 0: dict_crawlers_to_launch[splash_name] = nb_to_launch if dict_crawlers_to_launch: set_nb_crawlers_to_launch(dict_crawlers_to_launch) return (dict_crawlers_to_launch, 200) else: return ({'error':'invalid input'}, 400) ##-- CRAWLER GLOBAL --## #### CRAWLER TASK #### def create_crawler_task(url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, crawler_type=None, cookiejar_uuid=None, user_agent=None): crawler_config = {} crawler_config['depth_limit'] = depth_limit crawler_config['closespider_pagecount'] = max_pages if screenshot: crawler_config['png'] = True else: crawler_config['png'] = False if har: crawler_config['har'] = True else: crawler_config['har'] = False if user_agent: crawler_config['user_agent'] = user_agent if cookiejar_uuid: crawler_config['cookiejar_uuid'] = cookiejar_uuid if auto_crawler: crawler_mode = 'auto' crawler_config['time'] = crawler_delta else: crawler_mode = 'manual' # get crawler_mode faup.decode(url) unpack_url = faup.get() ## TODO: # FIXME: remove me try: domain = unpack_url['domain'].decode() except: domain = unpack_url['domain'] ## TODO: # FIXME: remove me try: tld = unpack_url['tld'].decode() except: tld = unpack_url['tld'] if crawler_type=='None': crawler_type = None if crawler_type: if crawler_type=='tor': crawler_type = 'onion' else: if tld == 'onion': crawler_type = 'onion' else: crawler_type = 'regular' save_crawler_config(crawler_mode, crawler_type, crawler_config, domain, url=url) send_url_to_crawl_in_queue(crawler_mode, crawler_type, url) def save_crawler_config(crawler_mode, crawler_type, crawler_config, domain, url=None): if crawler_mode == 'manual': r_cache.set('crawler_config:{}:{}:{}'.format(crawler_mode, crawler_type, domain), json.dumps(crawler_config)) elif crawler_mode == 'auto': r_serv_onion.set('crawler_config:{}:{}:{}:{}'.format(crawler_mode, crawler_type, domain, url), json.dumps(crawler_config)) def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url): print('{}_crawler_priority_queue'.format(crawler_type), '{};{}'.format(url, crawler_mode)) r_serv_onion.sadd('{}_crawler_priority_queue'.format(crawler_type), '{};{}'.format(url, crawler_mode)) # add auto crawled url for user UI if crawler_mode == 'auto': r_serv_onion.sadd('auto_crawler_url:{}'.format(crawler_type), url) #### #### #### CRAWLER TASK API #### def api_create_crawler_task(user_id, url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, crawler_type=None, cookiejar_uuid=None, user_agent=None): # validate url if url is None or url=='' or url=='\n': return ({'error':'invalid depth limit'}, 400) if depth_limit: try: depth_limit = int(depth_limit) if depth_limit < 0: depth_limit = 0 except ValueError: return ({'error':'invalid depth limit'}, 400) if max_pages: try: max_pages = int(max_pages) if max_pages < 1: max_pages = 1 except ValueError: return ({'error':'invalid max_pages limit'}, 400) if auto_crawler: try: crawler_delta = int(crawler_delta) if crawler_delta < 0: return ({'error':'invalid delta bettween two pass of the crawler'}, 400) except ValueError: return ({'error':'invalid delta bettween two pass of the crawler'}, 400) if cookiejar_uuid: if not exist_cookiejar(cookiejar_uuid): return ({'error': 'unknow cookiejar uuid', 'cookiejar_uuid': cookiejar_uuid}, 404) level = get_cookiejar_level(cookiejar_uuid) if level == 0: # # TODO: check if user is admin cookie_owner = get_cookiejar_owner(cookiejar_uuid) if cookie_owner != user_id: return ({'error': 'The access to this cookiejar is restricted'}, 403) # # TODO: verify splash name/crawler type create_crawler_task(url, screenshot=screenshot, har=har, depth_limit=depth_limit, max_pages=max_pages, crawler_type=crawler_type, auto_crawler=auto_crawler, crawler_delta=crawler_delta, cookiejar_uuid=cookiejar_uuid, user_agent=user_agent) return None #### #### #### SPLASH API #### def is_splash_reachable(splash_url, timeout=1.0): try: r = requests.get(splash_url , timeout=timeout) except Exception: return False if r.status_code == 200: return True else: return False #### #### def is_redirection(domain, last_url): url = urlparse(last_url) last_domain = url.netloc last_domain = last_domain.split('.') last_domain = '{}.{}'.format(last_domain[-2], last_domain[-1]) return domain != last_domain # domain up def create_domain_metadata(domain_type, domain, current_port, date, date_month): # Add to global set r_serv_onion.sadd('{}_up:{}'.format(domain_type, date), domain) r_serv_onion.sadd('full_{}_up'.format(domain_type), domain) r_serv_onion.sadd('month_{}_up:{}'.format(domain_type, date_month), domain) # create onion metadata if not r_serv_onion.exists('{}_metadata:{}'.format(domain_type, domain)): r_serv_onion.hset('{}_metadata:{}'.format(domain_type, domain), 'first_seen', date) r_serv_onion.hset('{}_metadata:{}'.format(domain_type, domain), 'last_check', date) # Update domain port number all_domain_ports = r_serv_onion.hget('{}_metadata:{}'.format(domain_type, domain), 'ports') if all_domain_ports: all_domain_ports = all_domain_ports.split(';') else: all_domain_ports = [] if current_port not in all_domain_ports: all_domain_ports.append(current_port) r_serv_onion.hset('{}_metadata:{}'.format(domain_type, domain), 'ports', ';'.join(all_domain_ports)) # add root_item to history def add_domain_root_item(root_item, domain_type, domain, epoch_date, port): # Create/Update crawler history r_serv_onion.zadd('crawler_history_{}:{}:{}'.format(domain_type, domain, port), epoch_date, root_item) def create_item_metadata(item_id, domain, url, port, item_father): r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', item_father) r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'domain', '{}:{}'.format(domain, port)) r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'real_link', url) # add this item_id to his father r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_id) def create_item_id(item_dir, domain): if len(domain) > 215: UUID = domain[-215:]+str(uuid.uuid4()) else: UUID = domain+str(uuid.uuid4()) return os.path.join(item_dir, UUID) def save_crawled_item(item_id, item_content): try: gzipencoded = gzip.compress(item_content.encode()) gzip64encoded = base64.standard_b64encode(gzipencoded).decode() return gzip64encoded except: print("file error: {}".format(item_id)) return False def save_har(har_dir, item_id, har_content): if not os.path.exists(har_dir): os.makedirs(har_dir) item_id = item_id.split('/')[-1] filename = os.path.join(har_dir, item_id + '.json') with open(filename, 'w') as f: f.write(json.dumps(har_content)) # # TODO: FIXME def api_add_crawled_item(dict_crawled): domain = None # create item_id item_id = save_crawled_item(item_id, response.data['html']) create_item_metadata(item_id, domain, 'last_url', port, 'father') #### CRAWLER QUEUES #### def get_all_crawlers_queues_types(): all_queues_types = set() all_splash_name = get_all_crawlers_to_launch_splash_name() for splash_name in all_splash_name: all_queues_types.add(get_splash_crawler_type(splash_name)) all_splash_name = list() return all_queues_types def get_crawler_queue_types_by_splash_name(splash_name): all_domain_type = [splash_name] crawler_type = get_splash_crawler_type(splash_name) #if not is_splash_used_in_discovery(splash_name) if crawler_type == 'tor': all_domain_type.append('onion') all_domain_type.append('regular') else: all_domain_type.append('regular') return all_domain_type def get_crawler_type_by_url(url): faup.decode(url) unpack_url = faup.get() ## TODO: # FIXME: remove me try: tld = unpack_url['tld'].decode() except: tld = unpack_url['tld'] if tld == 'onion': crawler_type = 'onion' else: crawler_type = 'regular' return crawler_type def get_elem_to_crawl_by_queue_type(l_queue_type): ## queues priority: # 1 - priority queue # 2 - discovery queue # 3 - normal queue ## all_queue_key = ['{}_crawler_priority_queue', '{}_crawler_discovery_queue', '{}_crawler_queue'] for queue_key in all_queue_key: for queue_type in l_queue_type: message = r_serv_onion.spop(queue_key.format(queue_type)) if message: dict_to_crawl = {} splitted = message.rsplit(';', 1) if len(splitted) == 2: url, item_id = splitted item_id = item_id.replace(PASTES_FOLDER+'/', '') else: # # TODO: to check/refractor item_id = None url = message crawler_type = get_crawler_type_by_url(url) return {'url': url, 'paste': item_id, 'type_service': crawler_type, 'queue_type': queue_type, 'original_message': message} return None def get_nb_elem_to_crawl_by_type(queue_type): nb = r_serv_onion.scard('{}_crawler_priority_queue'.format(queue_type)) nb += r_serv_onion.scard('{}_crawler_discovery_queue'.format(queue_type)) nb += r_serv_onion.scard('{}_crawler_queue'.format(queue_type)) return nb #### ---- #### # # # # # # # # # # # # # # # SPLASH MANAGER # # # # # # # # # # # # # # # def get_splash_manager_url(reload=False): # TODO: add in db config return r_serv_onion.get('crawler:splash:manager:url') def get_splash_api_key(reload=False): # TODO: add in db config return r_serv_onion.get('crawler:splash:manager:key') def get_hidden_splash_api_key(): # TODO: add in db config key = get_splash_api_key() if key: if len(key)==41: return f'{key[:4]}*********************************{key[-4:]}' def is_valid_api_key(api_key, search=re.compile(r'[^a-zA-Z0-9_-]').search): if len(api_key) != 41: return False return not bool(search(api_key)) def save_splash_manager_url_api(url, api_key): r_serv_onion.set('crawler:splash:manager:url', url) r_serv_onion.set('crawler:splash:manager:key', api_key) def get_splash_url_from_manager_url(splash_manager_url, splash_port): url = urlparse(splash_manager_url) host = url.netloc.split(':', 1)[0] return '{}:{}'.format(host, splash_port) # def is_splash_used_in_discovery(splash_name): # res = r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'discovery_queue') # if res == 'True': # return True # else: # return False def restart_splash_docker(splash_url): splash_port = splash_url.split(':')[-1] return _restart_splash_docker(splash_port) def is_splash_manager_connected(delta_check=30): last_check = r_cache.hget('crawler:splash:manager', 'last_check') if last_check: if int(time.time()) - int(last_check) > delta_check: ping_splash_manager() else: ping_splash_manager() res = r_cache.hget('crawler:splash:manager', 'connected') return res == 'True' def update_splash_manager_connection_status(is_connected, req_error=None): r_cache.hset('crawler:splash:manager', 'connected', is_connected) r_cache.hset('crawler:splash:manager', 'last_check', int(time.time())) if not req_error: r_cache.hdel('crawler:splash:manager', 'error') else: r_cache.hset('crawler:splash:manager', 'status_code', req_error['status_code']) r_cache.hset('crawler:splash:manager', 'error', req_error['error']) def get_splash_manager_connection_metadata(force_ping=False): dict_manager={} if force_ping: dict_manager['status'] = ping_splash_manager() else: dict_manager['status'] = is_splash_manager_connected() if not dict_manager['status']: dict_manager['status_code'] = r_cache.hget('crawler:splash:manager', 'status_code') dict_manager['error'] = r_cache.hget('crawler:splash:manager', 'error') return dict_manager ## API ## def ping_splash_manager(): splash_manager_url = get_splash_manager_url() if not splash_manager_url: return False try: req = requests.get('{}/api/v1/ping'.format(splash_manager_url), headers={"Authorization": get_splash_api_key()}, verify=False) if req.status_code == 200: update_splash_manager_connection_status(True) return True else: res = req.json() if 'reason' in res: req_error = {'status_code': req.status_code, 'error': res['reason']} else: print(req.json()) req_error = {'status_code': req.status_code, 'error': json.dumps(req.json())} update_splash_manager_connection_status(False, req_error=req_error) return False except requests.exceptions.ConnectionError: pass # splash manager unreachable req_error = {'status_code': 500, 'error': 'splash manager unreachable'} update_splash_manager_connection_status(False, req_error=req_error) return False def get_splash_manager_session_uuid(): try: req = requests.get('{}/api/v1/get/session_uuid'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False) if req.status_code == 200: res = req.json() if res: return res['session_uuid'] else: print(req.json()) except requests.exceptions.ConnectionError: # splash manager unreachable update_splash_manager_connection_status(False) def get_splash_manager_version(): splash_manager_url = get_splash_manager_url() if splash_manager_url: try: req = requests.get('{}/api/v1/version'.format(splash_manager_url), headers={"Authorization": get_splash_api_key()}, verify=False) if req.status_code == 200: return req.json()['message'] else: print(req.json()) except requests.exceptions.ConnectionError: pass def get_all_splash_manager_containers_name(): req = requests.get('{}/api/v1/get/splash/name/all'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False) if req.status_code == 200: return req.json() else: print(req.json()) def get_all_splash_manager_proxies(): req = requests.get('{}/api/v1/get/proxies/all'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False) if req.status_code == 200: return req.json() else: print(req.json()) def _restart_splash_docker(splash_port): dict_to_send = {'docker_port': splash_port} req = requests.post('{}/api/v1/splash/restart'.format(get_splash_manager_url()), headers={"Authorization": get_splash_api_key()}, verify=False, json=dict_to_send) if req.status_code == 200: return req.json() else: print(req.json()) def api_save_splash_manager_url_api(data): # unpack json manager_url = data.get('url', None) api_key = data.get('api_key', None) if not manager_url or not api_key: return ({'status': 'error', 'reason': 'No url or API key supplied'}, 400) # check if is valid url try: result = urlparse(manager_url) if not all([result.scheme, result.netloc]): return ({'status': 'error', 'reason': 'Invalid url'}, 400) except: return ({'status': 'error', 'reason': 'Invalid url'}, 400) # check if is valid key if not is_valid_api_key(api_key): return ({'status': 'error', 'reason': 'Invalid API key'}, 400) save_splash_manager_url_api(manager_url, api_key) return ({'url': manager_url, 'api_key': get_hidden_splash_api_key()}, 200) ## -- ## ## SPLASH ## def get_all_splash(r_list=False): res = r_serv_onion.smembers('all_splash') if res: if r_list: return list(res) else: return res else: return [] def get_splash_proxy(splash_name): return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'proxy') def get_splash_all_url(splash_name, r_list=False): res = r_serv_onion.smembers('splash:url:{}'.format(splash_name)) if res: if r_list: return list(res) else: return res else: return [] def get_splash_name_by_url(splash_url): return r_serv_onion.get('splash:map:url:name:{}'.format(splash_url)) def get_splash_crawler_type(splash_name): return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'crawler_type') def get_splash_crawler_description(splash_name): return r_serv_onion.hget('splash:metadata:{}'.format(splash_name), 'description') def get_splash_crawler_metadata(splash_name): dict_splash = {} dict_splash['proxy'] = get_splash_proxy(splash_name) dict_splash['type'] = get_splash_crawler_type(splash_name) dict_splash['description'] = get_splash_crawler_description(splash_name) return dict_splash def get_all_splash_crawler_metadata(): dict_splash = {} for splash_name in get_all_splash(): dict_splash[splash_name] = get_splash_crawler_metadata(splash_name) return dict_splash def get_all_splash_by_proxy(proxy_name, r_list=False): res = r_serv_onion.smembers('proxy:splash:{}'.format(proxy_name)) if res: if r_list: return list(res) else: return res else: return [] def delete_all_splash_containers(): for splash_name in get_all_splash(): delete_splash_container(splash_name) def delete_splash_container(splash_name): r_serv_onion.srem('proxy:splash:{}'.format(get_splash_proxy(splash_name)), splash_name) r_serv_onion.delete('splash:metadata:{}'.format(splash_name)) for splash_url in get_splash_all_url(splash_name): r_serv_onion.delete('splash:map:url:name:{}'.format(splash_url), splash_name) r_serv_onion.srem('splash:url:{}'.format(splash_name), splash_url) r_serv_onion.srem('all_splash', splash_name) ## -- ## ## PROXY ## def get_all_proxies(r_list=False): res = r_serv_onion.smembers('all_proxy') if res: return list(res) else: return [] def delete_all_proxies(): for proxy_name in get_all_proxies(): delete_proxy(proxy_name) def get_proxy_host(proxy_name): return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'host') def get_proxy_port(proxy_name): return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'port') def get_proxy_type(proxy_name): return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'type') def get_proxy_crawler_type(proxy_name): return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'crawler_type') def get_proxy_description(proxy_name): return r_serv_onion.hget('proxy:metadata:{}'.format(proxy_name), 'description') def get_proxy_metadata(proxy_name): meta_dict = {} meta_dict['host'] = get_proxy_host(proxy_name) meta_dict['port'] = get_proxy_port(proxy_name) meta_dict['type'] = get_proxy_type(proxy_name) meta_dict['crawler_type'] = get_proxy_crawler_type(proxy_name) meta_dict['description'] = get_proxy_description(proxy_name) return meta_dict def get_all_proxies_metadata(): all_proxy_dict = {} for proxy_name in get_all_proxies(): all_proxy_dict[proxy_name] = get_proxy_metadata(proxy_name) return all_proxy_dict # def set_proxy_used_in_discovery(proxy_name, value): # r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'discovery_queue', value) def delete_proxy(proxy_name): # # TODO: force delete (delete all proxy) proxy_splash = get_all_splash_by_proxy(proxy_name) #if proxy_splash: # print('error, a splash container is using this proxy') r_serv_onion.delete('proxy:metadata:{}'.format(proxy_name)) r_serv_onion.srem('all_proxy', proxy_name) ## -- ## ## LOADER ## def load_all_splash_containers(): all_splash_containers_name = get_all_splash_manager_containers_name() for splash_name in all_splash_containers_name: r_serv_onion.sadd('all_splash', splash_name) proxy = all_splash_containers_name[splash_name]['proxy'] if not proxy: proxy = {'name': 'no_proxy', 'crawler_type': 'web'} r_serv_onion.sadd('proxy:splash:{}'.format(proxy['name']), splash_name) r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'crawler_type', proxy['crawler_type']) r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'proxy', proxy['name']) description = all_splash_containers_name[splash_name].get('description', None) if description: r_serv_onion.hset('splash:metadata:{}'.format(splash_name), 'description', description) for port in all_splash_containers_name[splash_name]['ports']: splash_url = get_splash_url_from_manager_url(get_splash_manager_url(), port) r_serv_onion.sadd('splash:url:{}'.format(splash_name), splash_url) r_serv_onion.set('splash:map:url:name:{}'.format(splash_url), splash_name) def load_all_proxy(): delete_all_proxies() all_proxies = get_all_splash_manager_proxies() for proxy_name in all_proxies: proxy_dict = all_proxies[proxy_name] r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'host', proxy_dict['host']) r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'port', proxy_dict['port']) r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'type', proxy_dict['type']) r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'crawler_type', proxy_dict['crawler_type']) description = all_proxies[proxy_name].get('description', None) if description: r_serv_onion.hset('proxy:metadata:{}'.format(proxy_name), 'description', description) r_serv_onion.sadd('all_proxy', proxy_name) def reload_splash_and_proxies_list(): if ping_splash_manager(): # LOAD SPLASH containers delete_all_splash_containers() load_all_splash_containers() # LOAD PROXIES containers delete_all_proxies() load_all_proxy() return True else: return False # # TODO: kill crawler screen ? ## -- ## ## SPLASH CONTROLLER ## def launch_ail_splash_crawler(splash_url, script_options=''): screen_name = 'Crawler_AIL' dir_project = os.environ['AIL_HOME'] script_location = os.path.join(os.environ['AIL_BIN']) script_name = 'Crawler.py' screen.create_screen(screen_name) screen.launch_uniq_windows_script(screen_name, splash_url, dir_project, script_location, script_name, script_options=script_options, kill_previous_windows=True) ## -- ## #### ---- #### #### CRAWLER PROXY #### #### ---- #### if __name__ == '__main__': res = get_splash_manager_version() print(res)