CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Add capture meta on the related cookies page

pull/78/head
Raphaël Vinot 2020-06-15 01:34:59 +02:00
parent 5ae7f0f7e4
commit e6c9f6dada
2 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lookyloo/indexing.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from typing import Set, Tuple, List from typing import Set, Tuple, List, Optional, Dict, Any
from redis import Redis from redis import Redis
@ -28,8 +28,18 @@ class Indexing():
def get_cookie_domains(self, cookie_name: str) -> List[Tuple[str, float]]: def get_cookie_domains(self, cookie_name: str) -> List[Tuple[str, float]]:
return self.redis.zrevrange(f'cn|{cookie_name}', 0, -1, withscores=True) return self.redis.zrevrange(f'cn|{cookie_name}', 0, -1, withscores=True)
def get_capture_cache(self, capture_uuid: str) -> Optional[Dict[str, Any]]:
capture_dir = self.lookyloo.lookup_capture_dir(capture_uuid)
if capture_dir:
return self.lookyloo.capture_cache(capture_dir)
return {}
def get_cookies_names_captures(self, cookie_name: str) -> List[Tuple[str, str]]:
return [uuids.split('|')for uuids in self.redis.smembers(f'cn|{cookie_name}|captures')]
def index_cookies(self) -> None: def index_cookies(self) -> None:
for capture_dir in self.lookyloo.capture_dirs: for capture_dir in self.lookyloo.capture_dirs:
print(f'Processing {capture_dir}')
try: try:
crawled_tree = self.lookyloo.get_crawled_tree(capture_dir) crawled_tree = self.lookyloo.get_crawled_tree(capture_dir)
except Exception as e: except Exception as e:
@ -47,6 +57,7 @@ class Indexing():
already_loaded.add((name, domain)) already_loaded.add((name, domain))
pipeline.zincrby('cookies_names', 1, name) pipeline.zincrby('cookies_names', 1, name)
pipeline.zincrby(f'cn|{name}', 1, domain) pipeline.zincrby(f'cn|{name}', 1, domain)
pipeline.sadd(f'cn|{name}|captures', f'{crawled_tree.uuid}|{urlnode.uuid}')
pipeline.zincrby(f'cn|{name}|{domain}', 1, value) pipeline.zincrby(f'cn|{name}|{domain}', 1, value)
pipeline.sadd('lookyloo_domains', domain) pipeline.sadd('lookyloo_domains', domain)

3
website/web/__init__.py
View File

@ -431,9 +431,10 @@ def cookies_lookup():
@app.route('/cookies/<string:cookie_name>', methods=['GET']) @app.route('/cookies/<string:cookie_name>', methods=['GET'])
def cookies_name_detail(cookie_name: str): def cookies_name_detail(cookie_name: str):
i = Indexing() i = Indexing()
captures = [i.get_capture_cache(capture) for capture, url in i.get_cookies_names_captures(cookie_name)]
domains = [(domain, freq, i.cookies_names_domains_values(cookie_name, domain)) domains = [(domain, freq, i.cookies_names_domains_values(cookie_name, domain))
for domain, freq in i.get_cookie_domains(cookie_name)] for domain, freq in i.get_cookie_domains(cookie_name)]
return render_template('cookie_name.html', cookie_name=cookie_name, domains=domains) return render_template('cookie_name.html', cookie_name=cookie_name, domains=domains, captures=captures)
# Query API # Query API