CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [api] rename domain lookup

pull/607/merge
terrtia 2024-10-04 11:53:55 +02:00
parent 6f2a59ccc0
commit 5a052c47d9
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bin/lib/crawlers.py
View File

@ -52,6 +52,8 @@ r_cache = config_loader.get_redis_conn("Redis_Cache")
ITEMS_FOLDER = config_loader.get_config_str("Directories", "pastes") ITEMS_FOLDER = config_loader.get_config_str("Directories", "pastes")
HAR_DIR = config_loader.get_files_directory('har') HAR_DIR = config_loader.get_files_directory('har')
activate_crawler = config_loader.get_config_str("Crawler", "activate_crawler") activate_crawler = config_loader.get_config_str("Crawler", "activate_crawler")
D_HAR = config_loader.get_config_boolean('Crawler', 'default_har')
D_SCREENSHOT = config_loader.get_config_boolean('Crawler', 'default_screenshot')
config_loader = None config_loader = None
faup = Faup() faup = Faup()
@ -65,9 +67,14 @@ faup = Faup()
# is safe ??? # is safe ???
# TODO FILTER URL ??? # TODO FILTER URL ???
def api_get_domain_lookup_meta(domain): def api_get_onion_lookup(domain):
domain = domain.lower()
dom = Domain(domain) dom = Domain(domain)
if not is_valid_onion_v3_domain(domain):
return {'error': 'Invalid Domain', 'domain': domain}, 404
if not dom.exists(): if not dom.exists():
if is_crawler_activated():
create_task(domain, parent='lookup', priority=0, har=D_HAR, screenshot=D_SCREENSHOT)
return {'error': 'domain not found', 'domain': domain}, 404 return {'error': 'domain not found', 'domain': domain}, 404
meta = dom.get_meta(options={'languages'}) meta = dom.get_meta(options={'languages'})
meta['first_seen'] = meta['first_seen'].replace('/', '-') meta['first_seen'] = meta['first_seen'].replace('/', '-')
@ -109,6 +116,11 @@ def get_date_crawled_items_source(date):
def get_har_dir(): def get_har_dir():
return HAR_DIR return HAR_DIR
def is_valid_onion_v3_domain(domain):
if len(domain) == 62: # v3 address
return domain[:56].isalnum()
return False
def is_valid_onion_domain(domain): def is_valid_onion_domain(domain):
if not domain.endswith('.onion'): if not domain.endswith('.onion'):
return False return False

6
var/www/blueprints/api_rest.py
View File

@ -218,10 +218,10 @@ def objects_chat_thread_messages():
# # # # # # # # # # # # # # # DOMAINS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # DOMAINS # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@api_rest.route("api/v1/domain/lookup/<domain>", methods=['GET']) @api_rest.route("api/v1/lookup/onion/<domain>", methods=['GET'])
@token_required('user') @token_required('user')
def api_domain_lookup(domain): def api_lookup_onion(domain):
return create_json_response(crawlers.api_get_domain_lookup_meta(domain), 200) return create_json_response(crawlers.api_get_onion_lookup(domain), 200)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # TITLES # # # # # # # # # # # # # # # # # # # TODO TO REVIEW # # # # # # # # # # # # # # # TITLES # # # # # # # # # # # # # # # # # # # TODO TO REVIEW