mirror of https://github.com/CIRCL/lookyloo
chg: normalize output to get categories, properly re-trigger indexes
parent
13651dcfdb
commit
11fcc9dd8b
|
@ -11,7 +11,7 @@ import re
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from datetime import datetime, timedelta, date
|
from datetime import datetime, timedelta, date
|
||||||
from functools import lru_cache
|
from functools import lru_cache, cache
|
||||||
from importlib.metadata import version
|
from importlib.metadata import version
|
||||||
from io import BufferedIOBase
|
from io import BufferedIOBase
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
@ -31,6 +31,7 @@ from werkzeug.user_agent import UserAgent
|
||||||
from werkzeug.utils import cached_property
|
from werkzeug.utils import cached_property
|
||||||
|
|
||||||
from .default import get_homedir, safe_create_dir, get_config, LookylooException
|
from .default import get_homedir, safe_create_dir, get_config, LookylooException
|
||||||
|
from .indexing import Indexing
|
||||||
# from .exceptions import InvalidCaptureSetting
|
# from .exceptions import InvalidCaptureSetting
|
||||||
|
|
||||||
|
|
||||||
|
@ -436,3 +437,12 @@ def load_user_config(username: str) -> dict[str, Any] | None:
|
||||||
return None
|
return None
|
||||||
with user_config_path.open() as _c:
|
with user_config_path.open() as _c:
|
||||||
return json.load(_c)
|
return json.load(_c)
|
||||||
|
|
||||||
|
|
||||||
|
@cache
|
||||||
|
def get_indexing(full: bool=False) -> Indexing:
|
||||||
|
if not get_config('generic', 'index_everything'):
|
||||||
|
return Indexing()
|
||||||
|
if full:
|
||||||
|
return Indexing(full_index=True)
|
||||||
|
return Indexing()
|
||||||
|
|
|
@ -59,7 +59,8 @@ from .helpers import (get_captures_dir, get_email_template,
|
||||||
get_resources_hashes, get_taxonomies,
|
get_resources_hashes, get_taxonomies,
|
||||||
uniq_domains, ParsedUserAgent, UserAgents,
|
uniq_domains, ParsedUserAgent, UserAgents,
|
||||||
get_useragent_for_requests, load_takedown_filters,
|
get_useragent_for_requests, load_takedown_filters,
|
||||||
CaptureSettings, load_user_config
|
CaptureSettings, load_user_config,
|
||||||
|
get_indexing
|
||||||
)
|
)
|
||||||
from .modules import (MISPs, PhishingInitiative, UniversalWhois,
|
from .modules import (MISPs, PhishingInitiative, UniversalWhois,
|
||||||
UrlScan, VirusTotal, Phishtank, Hashlookup,
|
UrlScan, VirusTotal, Phishtank, Hashlookup,
|
||||||
|
@ -335,6 +336,9 @@ class Lookyloo():
|
||||||
current_categories.add(category)
|
current_categories.add(category)
|
||||||
with categ_file.open('w') as f:
|
with categ_file.open('w') as f:
|
||||||
f.writelines(f'{t}\n' for t in current_categories)
|
f.writelines(f'{t}\n' for t in current_categories)
|
||||||
|
get_indexing().reindex_categories_capture(capture_uuid)
|
||||||
|
if get_config('generic', 'index_everything'):
|
||||||
|
get_indexing(full=True).reindex_categories_capture(capture_uuid)
|
||||||
|
|
||||||
def uncategorize_capture(self, capture_uuid: str, /, category: str) -> None:
|
def uncategorize_capture(self, capture_uuid: str, /, category: str) -> None:
|
||||||
'''Remove a category (MISP Taxonomy tag) from a capture.'''
|
'''Remove a category (MISP Taxonomy tag) from a capture.'''
|
||||||
|
@ -351,6 +355,9 @@ class Lookyloo():
|
||||||
current_categories.remove(category)
|
current_categories.remove(category)
|
||||||
with categ_file.open('w') as f:
|
with categ_file.open('w') as f:
|
||||||
f.writelines(f'{t}\n' for t in current_categories)
|
f.writelines(f'{t}\n' for t in current_categories)
|
||||||
|
get_indexing().reindex_categories_capture(capture_uuid)
|
||||||
|
if get_config('generic', 'index_everything'):
|
||||||
|
get_indexing(full=True).reindex_categories_capture(capture_uuid)
|
||||||
|
|
||||||
def trigger_modules(self, capture_uuid: str, /, force: bool=False, auto_trigger: bool=False) -> dict[str, Any]:
|
def trigger_modules(self, capture_uuid: str, /, force: bool=False, auto_trigger: bool=False) -> dict[str, Any]:
|
||||||
'''Launch the 3rd party modules on a capture.
|
'''Launch the 3rd party modules on a capture.
|
||||||
|
|
|
@ -694,7 +694,6 @@ def categories_capture(tree_uuid: str, query: str) -> str | WerkzeugResponse | R
|
||||||
categories.append(category)
|
categories.append(category)
|
||||||
for category in categories:
|
for category in categories:
|
||||||
lookyloo.categorize_capture(tree_uuid, category)
|
lookyloo.categorize_capture(tree_uuid, category)
|
||||||
get_indexing(flask_login.current_user).reindex_categories_capture(tree_uuid)
|
|
||||||
if 'query' in request.form and request.form.get('query', '').strip():
|
if 'query' in request.form and request.form.get('query', '').strip():
|
||||||
matching_categories = {}
|
matching_categories = {}
|
||||||
t = get_taxonomies()
|
t = get_taxonomies()
|
||||||
|
@ -714,7 +713,6 @@ def uncategorize_capture(tree_uuid: str, category: str) -> str | WerkzeugRespons
|
||||||
if not enable_categorization:
|
if not enable_categorization:
|
||||||
return jsonify({'response': 'Categorization not enabled.'})
|
return jsonify({'response': 'Categorization not enabled.'})
|
||||||
lookyloo.uncategorize_capture(tree_uuid, category)
|
lookyloo.uncategorize_capture(tree_uuid, category)
|
||||||
get_indexing(flask_login.current_user).reindex_categories_capture(tree_uuid)
|
|
||||||
return jsonify({'response': f'{category} successfully removed from {tree_uuid}'})
|
return jsonify({'response': f'{category} successfully removed from {tree_uuid}'})
|
||||||
|
|
||||||
|
|
||||||
|
@ -725,7 +723,6 @@ def categorize_capture(tree_uuid: str, category: str) -> str | WerkzeugResponse
|
||||||
if not enable_categorization:
|
if not enable_categorization:
|
||||||
return jsonify({'response': 'Categorization not enabled.'})
|
return jsonify({'response': 'Categorization not enabled.'})
|
||||||
lookyloo.categorize_capture(tree_uuid, category)
|
lookyloo.categorize_capture(tree_uuid, category)
|
||||||
get_indexing(flask_login.current_user).reindex_categories_capture(tree_uuid)
|
|
||||||
return jsonify({'response': f'{category} successfully added to {tree_uuid}'})
|
return jsonify({'response': f'{category} successfully added to {tree_uuid}'})
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -802,23 +802,11 @@ class RecentCaptures(Resource): # type: ignore[misc]
|
||||||
params={'category': 'The category according to which the uuids are to be returned.'},
|
params={'category': 'The category according to which the uuids are to be returned.'},
|
||||||
required=False)
|
required=False)
|
||||||
class CategoriesCaptures(Resource): # type: ignore[misc]
|
class CategoriesCaptures(Resource): # type: ignore[misc]
|
||||||
def get(self, category: str | None=None) -> list[str] | dict[str, Any]:
|
def get(self, category: str | None=None) -> list[str] | dict[str, list[str]] | tuple[dict[str, str], int]:
|
||||||
categories = ['legitimate', 'parking-page', 'default-page', 'insti_usertution', 'captcha',
|
existing_categories = get_indexing(flask_login.current_user).categories
|
||||||
'authentication-form', 'adult-content', 'shop', 'malicious', 'clone', 'phishing', 'unclear']
|
if category:
|
||||||
if not category:
|
if category not in existing_categories:
|
||||||
all_categorized_uuids: dict[str, set[str]] = {}
|
return {'error': f'Invalid category: {category}, must be in {", ".join(existing_categories)}.'}, 400
|
||||||
for c in categories:
|
return list(get_indexing(flask_login.current_user).get_captures_category(category))
|
||||||
one_categorie = get_indexing(flask_login.current_user).get_captures_category(c)
|
return {c: list(get_indexing(flask_login.current_user).get_captures_category(c))
|
||||||
if not one_categorie:
|
for c in existing_categories}
|
||||||
continue
|
|
||||||
for uuid in one_categorie:
|
|
||||||
if uuid not in all_categorized_uuids:
|
|
||||||
all_categorized_uuids[uuid] = {c}
|
|
||||||
else:
|
|
||||||
all_categorized_uuids[uuid].add(c)
|
|
||||||
all_categorized_uuids_list = {uuid: list(categories) for uuid, categories in all_categorized_uuids.items()}
|
|
||||||
return all_categorized_uuids_list
|
|
||||||
if not category in categories:
|
|
||||||
return {'error': f'Invalid category: {category}'}
|
|
||||||
return list(get_indexing(flask_login.current_user).get_captures_category(category))
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue