diff --git a/config/generic.json.sample b/config/generic.json.sample index 507fa011..c6fa359a 100644 --- a/config/generic.json.sample +++ b/config/generic.json.sample @@ -14,6 +14,7 @@ "max_depth": 1, "use_user_agents_users": false, "enable_default_blur_screenshot": false, + "enable_context_by_users": false, "enable_mail_notification": false, "email": { "from": "Lookyloo ", @@ -35,6 +36,7 @@ "max_depth": "Maximum depth for scraping. Anything > 1 will be exponentially bigger.", "use_user_agents_users": "Only usable for medium/high use instances: use the user agents of the users of the platform", "enable_default_blur_screenshot": "If true, blur the screenshot by default (useful on public instances)", + "enable_context_by_users": "Allow the users to add context to a response body", "enable_mail_notification": "Enable email notification or not", "email": "Configuration for sending email notifications." } diff --git a/known_content/generic.json b/known_content/generic.json new file mode 100644 index 00000000..6dd96b58 --- /dev/null +++ b/known_content/generic.json @@ -0,0 +1,66 @@ +{ + "1px_gif": { + "description": "1 pixel GIF", + "entries": [ + "717ea0ff7f3f624c268eccb244e24ec1305ab21557abb3d6f1a7e183ff68a2d28f13d1d2af926c9ef6d1fb16dd8cbe34cd98cacf79091dddc7874dcee21ecfdc", + "e508d5d17e94d14b126164082342a9ca4774f404e87a3dd56c26812493ee18d9c3d6daacca979134a94a003066aca24116de874596d00d1e52130c1283d54209", + "2d073e10ae40fde434eb31cbedd581a35cd763e51fb7048b88caa5f949b1e6105e37a228c235bc8976e8db58ed22149cfccf83b40ce93a28390566a28975744a", + "84e24a70b78e9de9c9d0dfeb49f3f4247dbc1c715d8844471ee40669270682e199d48f5fbec62bd984c9c0270534b407c4d2561dd6c05adec3c83c1534f32d5c", + "d5da26b5d496edb0221df1a4057a8b0285d15592a8f8dc7016a294df37ed335f3fde6a2252962e0df38b62847f8b771463a0124ef3f84299f262ed9d9d3cee4c", + "f7a5f748f4c0d3096a3ca972886fe9a9dff5dce7792779ec6ffc42fa880b3815e2e4c3bdea452352f3844b81864c9bfb7861f66ac961cfa66cb9cb4febe568e8", + "b2ca25a3311dc42942e046eb1a27038b71d689925b7d6b3ebb4d7cd2c7b9a0c7de3d10175790ac060dc3f8acf3c1708c336626be06879097f4d0ecaa7f567041", + "b8d82d64ec656c63570b82215564929adad167e61643fd72283b94f3e448ef8ab0ad42202f3537a0da89960bbdc69498608fc6ec89502c6c338b6226c8bf5e14", + "2991c3aa1ba61a62c1cccd990c0679a1fb8dccd547d153ec0920b91a75ba20820de1d1c206f66d083bf2585d35050f0a39cd7a3e11c03882dafec907d27a0180", + "b1a6cfa7b21dbb0b281d241af609f3ba7f3a63e5668095bba912bf7cfd7f0320baf7c3b0bfabd0f8609448f39902baeb145ba7a2d8177fe22a6fcea03dd29be1", + "ebfe0c0df4bcc167d5cb6ebdd379f9083df62bef63a23818e1c6adf0f64b65467ea58b7cd4d03cf0a1b1a2b07fb7b969bf35f25f1f8538cc65cf3eebdf8a0910", + "1d68b92e8d822fe82dc7563edd7b37f3418a02a89f1a9f0454cca664c2fc2565235e0d85540ff9be0b20175be3f5b7b4eae1175067465d5cca13486aab4c582c", + "ac44da7f455bfae52b883639964276026fb259320902aa813d0333e021c356a7b3e3537b297f9a2158e588c302987ce0854866c039d1bb0ffb27f67560739db2", + "921944dc10fbfb6224d69f0b3ac050f4790310fd1bcac3b87c96512ad5ed9a268824f3f5180563d372642071b4704c979d209baf40bc0b1c9a714769aba7dfc7", + "89dfc38ec77cf258362e4db7c8203cae8a02c0fe4f99265b0539ec4f810c84f8451e22c9bef1ebc59b4089af7e93e378e053c542a5967ec4912d4c1fc5de22f0", + "280ea4383ee6b37051d91c5af30a5ce72aa4439340fc6d31a4fbe7ba8a8156eb7893891d5b2371b9fc4934a78f08de3d57e5b63fa9d279a317dcbefb8a07a6b0", + "3844065e1dd778a05e8cc39901fbf3191ded380d594359df137901ec56ca52e03d57eb60acc2421a0ee74f0733bbb5d781b7744685c26fb013a236f49b02fed3", + "bd9ab35dde3a5242b04c159187732e13b0a6da50ddcff7015dfb78cdd68743e191eaf5cddedd49bef7d2d5a642c217272a40e5ba603fe24ca676a53f8c417c5d", + "d052ecec2839340876eb57247cfc2e777dd7f2e868dc37cd3f3f740c8deb94917a0c9f2a4fc8229987a0b91b04726de2d1e9f6bcbe3f9bef0e4b7e0d7f65ea12", + "8717074ddf1198d27b9918132a550cb4ba343794cc3d304a793f9d78c9ff6c4929927b414141d40b6f6ad296725520f4c63edeb660ed530267766c2ab74ee4a9", + "6834f1548f26b94357fcc3312a3491e8c87080a84f678f990beb2c745899a01e239964521e64a534d7d5554222f728af966ec6ec8291bc64d2005861bcfd78ec", + "3be8176915593e79bc280d08984a16c29c495bc53be9b439276094b8dcd3764a3c72a046106a06b958e08e67451fe02743175c621a1faa261fe7a9691cc77141", + "826225fc21717d8861a05b9d2f959539aad2d2b131b2afed75d88fbca535e1b0d5a0da8ac69713a0876a0d467848a37a0a7f926aeafad8cf28201382d16466ab", + "202612457d9042fe853daab3ddcc1f0f960c5ffdbe8462fa435713e4d1d85ff0c3f197daf8dba15bda9f5266d7e1f9ecaeee045cbc156a4892d2f931fe6fa1bb", + "b82c6aa1ae927ade5fadbbab478cfaef26d21c1ac441f48e69cfc04cdb779b1e46d7668b4368b933213276068e52f9060228907720492a70fd9bc897191ee77c", + "763de1053a56a94eef4f72044adb2aa370b98ffa6e0add0b1cead7ee27da519e223921c681ae1db3311273f45d0dd3dc022d102d42ce210c90cb3e761b178438", + "69e2da5cdc318fc237eaa243b6ea7ecc83b68dbdea8478dc69154abdda86ecb4e16c35891cc1facb3ce7e0cf19d5abf189c50f59c769777706f4558f6442abbc", + "16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516", + "01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb", + "71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746", + "49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a", + "c57ebbadcf59f982ba28da35fdbd5e5369a8500a2e1edad0dc9c9174de6fd99f437953732e545b95d3de5943c61077b6b949c989f49553ff2e483f68fcc30641", + "c87bf81fd70cf6434ca3a6c05ad6e9bd3f1d96f77dddad8d45ee043b126b2cb07a5cf23b4137b9d8462cd8a9adf2b463ab6de2b38c93db72d2d511ca60e3b57e", + "fd8b021f0236e487bfee13bf8f0ae98760abc492f7ca3023e292631979e135cb4ccb0c89b6234971b060ad72c0ca4474cbb5092c6c7a3255d81a54a36277b486", + "235479f42cbbe0a4b0100167fece0d14c9b47d272b3ba8322bcfe8539f055bf31d500e7b2995cc968ebf73034e039f59c5f0f9410428663034bf119d74b5672c", + "a85e09c3b5dbb560f4e03ba880047dbc8b4999a64c1f54fbfbca17ee0bcbed3bc6708d699190b56668e464a59358d6b534c3963a1329ba01db21075ef5bedace", + "27656d6106a6da0c84174ba7a6307e6f1c4b3f2cc085c8466b6a25d54331035dabc7081aac208d960d8d37c5577547628c0d1c4b77bb4cf254c71859673feec1", + "41edf618eb0ba5158411c5ac3e900904bbf36cbb4be1347dc5281f4722244ad0b9880f0cf4fbec70089b0b7ba3b8aae6f92be7379e72db325c2802250b5e529e" + ] + }, + "1px_png": { + "description": "1 pixel PNG", + "entries": [ + "f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498", + "dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99", + "c2c239cb5cdd0b670780ad6414ef6be9ccd4c21ce46bb93d1fa3120ac812f1679445162978c3df05cb2e1582a1844cc4c41cf74960b8fdae3123999c5d2176cc", + "6ad523f5b65487369d305613366b9f68dcdeee225291766e3b25faf45439ca069f614030c08ca54c714fdbf7a944fac489b1515a8bf9e0d3191e1bcbbfe6a9df" + ] + }, + "empty_file": { + "description": "empty file", + "entries": [ + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" + ] + }, + "single_space": { + "description": "Empty file with a single space", + "entries": [ + "f90ddd77e400dfe6a3fcf479b00b1ee29e7015c5bb8cd70f5f15b4886cc339275ff553fc8a053f8ddc7324f45168cffaf81f8c3ac93996f6536eef38e5e40768" + ] + } +} diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index 34f5b336..247455ec 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -183,6 +183,14 @@ def get_user_agents(directory: str='user_agents') -> Dict[str, Any]: return json.load(f) +def load_known_content(directory: str='known_content') -> Dict[str, Dict[str, Any]]: + to_return: Dict[str, Dict[str, Any]] = {} + for known_content_file in (get_homedir() / directory).glob('*.json'): + with known_content_file.open() as f: + to_return[known_content_file.stem] = json.load(f) + return to_return + + def load_cookies(cookie_pseudofile: Optional[BufferedIOBase]=None) -> List[Dict[str, str]]: if cookie_pseudofile: cookies = json.load(cookie_pseudofile) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 9aa7e4b8..bb3dd410 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -15,7 +15,7 @@ from pathlib import Path import pickle import smtplib import socket -from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set, Iterable +from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set, Iterable, Iterator from urllib.parse import urlsplit from uuid import uuid4 from zipfile import ZipFile @@ -29,10 +29,15 @@ from scrapysplashwrapper import crawl from werkzeug.useragents import UserAgent from .exceptions import NoValidHarFile, MissingUUID -from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template, load_pickle_tree, remove_pickle_tree +from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template, load_pickle_tree, remove_pickle_tree, load_known_content from .modules import VirusTotal, SaneJavaScript, PhishingInitiative +def dump_to_json(obj: Union[Set]) -> Union[List]: + if isinstance(obj, set): + return list(obj) + + class Indexing(): def __init__(self) -> None: @@ -114,21 +119,22 @@ class Indexing(): pipeline = self.redis.pipeline() for urlnode in crawled_tree.root_hartree.url_tree.traverse(): - if not urlnode.empty_response: - pipeline.zincrby('body_hashes', 1, urlnode.body_hash) - pipeline.zincrby(f'bh|{urlnode.body_hash}', 1, urlnode.hostname) - # set of all captures with this hash - pipeline.sadd(f'bh|{urlnode.body_hash}|captures', crawled_tree.uuid) - # ZSet of all urlnode_UUIDs|full_url - pipeline.zincrby(f'bh|{urlnode.body_hash}|captures|{crawled_tree.uuid}', 1, f'{urlnode.uuid}|{urlnode.hostnode_uuid}|{urlnode.name}') - if hasattr(urlnode, 'embedded_ressources') and urlnode.embedded_ressources: - for mimetype, blobs in urlnode.embedded_ressources.items(): - for h, body in blobs: - pipeline.zincrby('body_hashes', 1, h) - pipeline.zincrby(f'bh|{h}', 1, urlnode.hostname) - pipeline.sadd(f'bh|{h}|captures', crawled_tree.uuid) - pipeline.zincrby(f'bh|{h}|captures|{crawled_tree.uuid}', 1, - f'{urlnode.uuid}|{urlnode.hostnode_uuid}|{urlnode.name}') + if urlnode.empty_response: + continue + pipeline.zincrby('body_hashes', 1, urlnode.body_hash) + pipeline.zincrby(f'bh|{urlnode.body_hash}', 1, urlnode.hostname) + # set of all captures with this hash + pipeline.sadd(f'bh|{urlnode.body_hash}|captures', crawled_tree.uuid) + # ZSet of all urlnode_UUIDs|full_url + pipeline.zincrby(f'bh|{urlnode.body_hash}|captures|{crawled_tree.uuid}', 1, f'{urlnode.uuid}|{urlnode.hostnode_uuid}|{urlnode.name}') + if hasattr(urlnode, 'embedded_ressources') and urlnode.embedded_ressources: + for mimetype, blobs in urlnode.embedded_ressources.items(): + for h, body in blobs: + pipeline.zincrby('body_hashes', 1, h) + pipeline.zincrby(f'bh|{h}', 1, urlnode.hostname) + pipeline.sadd(f'bh|{h}|captures', crawled_tree.uuid) + pipeline.zincrby(f'bh|{h}|captures|{crawled_tree.uuid}', 1, + f'{urlnode.uuid}|{urlnode.hostnode_uuid}|{urlnode.name}') pipeline.execute() @@ -147,6 +153,289 @@ class Indexing(): return self.redis.zrevrange(f'bh|{body_hash}', 0, -1, withscores=True) +class Context(): + + def __init__(self, sanejs: Optional[SaneJavaScript] = None): + self.redis: Redis = Redis(unix_socket_path=get_socket_path('indexing'), db=1, decode_responses=True) + self.sanejs = sanejs + self._cache_known_content() + + def clear_context(self): + self.redis.flushdb() + + def _get_resources_hashes(self, har2tree_container: Union[CrawledTree, HostNode, URLNode]) -> Set[str]: + if isinstance(har2tree_container, CrawledTree): + urlnodes = har2tree_container.root_hartree.url_tree.traverse() + elif isinstance(har2tree_container, HostNode): + urlnodes = har2tree_container.urls + elif isinstance(har2tree_container, URLNode): + urlnodes = [har2tree_container] + else: + raise Exception(f'har2tree_container cannot be {type(har2tree_container)}') + all_ressources_hashes: Set[str] = set() + for urlnode in urlnodes: + if hasattr(urlnode, 'resources_hashes'): + all_ressources_hashes.update(urlnode.resources_hashes) + return all_ressources_hashes + + def _cache_known_content(self) -> None: + p = self.redis.pipeline() + for filename, file_content in load_known_content().items(): + if filename == 'generic': + for k, type_content in file_content.items(): + p.hmset('known_content', {h: type_content['description'] for h in type_content['entries']}) + elif filename == 'malicious': + for h, details in file_content.items(): + p.sadd('bh|malicious', h) + elif filename == 'legitimate': + for h, details in file_content.items(): + if 'domain' in details and details['domain']: + p.sadd(f'bh|{h}|legitimate', *details['domain']) + elif 'description' in details: + p.hset('known_content', h, details['description']) + else: + for h, details in file_content.items(): + p.sadd(f'bh|{h}|legitimate', *details['hostnames']) + p.execute() + + def find_known_content(self, har2tree_container: Union[CrawledTree, HostNode, URLNode]) -> Dict[str, Union[str, List[str]]]: + """Return a dictionary of content resources found in the local known_content database, or in SaneJS (if enabled)""" + all_ressources_hashes = self._get_resources_hashes(har2tree_container) + # Get from local cache of known content all descriptions related to the ressources. + if not all_ressources_hashes: + return {} + known_content_table = dict(zip(all_ressources_hashes, + self.redis.hmget('known_content', all_ressources_hashes))) + + if self.sanejs and self.sanejs.available: + # Query sanejs on the remaining ones + to_lookup = [h for h, description in known_content_table.items() if not description] + for h, entry in self.sanejs.hashes_lookup(to_lookup).items(): + libname, version, path = entry[0].split("|") + known_content_table[h] = (libname, version, path, len(entry)) + return {h: details for h, details in known_content_table.items() if details} + + def _filter(self, urlnodes: Union[URLNode, List[URLNode]], known_hashes: Iterable[str]) -> Iterator[Tuple[URLNode, str]]: + if isinstance(urlnodes, URLNode): + _urlnodes = [urlnodes] + else: + _urlnodes = urlnodes + for urlnode in _urlnodes: + for h in urlnode.resources_hashes: + if h not in known_hashes: + yield urlnode, h + + def store_known_legitimate_tree(self, tree: CrawledTree): + known_content = self.find_known_content(tree) + urlnodes = tree.root_hartree.url_tree.traverse() + root_hostname = urlsplit(tree.root_url).hostname + known_content_file: Path = get_homedir() / 'known_content' / f'{root_hostname}.json' + if known_content_file.exists(): + with open(known_content_file) as f: + to_store = json.load(f) + else: + to_store = {} + for urlnode, h in self._filter(urlnodes, known_content): + mimetype = '' + if h != urlnode.body_hash: + # this is the hash of an embeded content so it won't have a filename but has a different mimetype + # FIXME: this is ugly. + for ressource_mimetype, blobs in urlnode.embedded_ressources.items(): + for ressource_h, b in blobs: + if ressource_h == h: + mimetype = ressource_mimetype.split(';')[0] + break + if mimetype: + break + else: + if urlnode.mimetype: + mimetype = urlnode.mimetype.split(';')[0] + if h not in to_store: + to_store[h] = {'filenames': set(), 'description': '', 'hostnames': set(), 'mimetype': mimetype} + else: + to_store[h]['filenames'] = set(to_store[h]['filenames']) + to_store[h]['hostnames'] = set(to_store[h]['hostnames']) + + to_store[h]['hostnames'].add(urlnode.hostname) + if urlnode.url_split.path: + filename = Path(urlnode.url_split.path).name + if filename: + to_store[h]['filenames'].add(filename) + + with open(known_content_file, 'w') as f: + json.dump(to_store, f, indent=2, default=dump_to_json) + + def mark_as_legitimate(self, tree: CrawledTree, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> None: + if hostnode_uuid: + urlnodes = tree.root_hartree.get_host_node_by_uuid(hostnode_uuid).urls + elif urlnode_uuid: + urlnodes = [tree.root_hartree.get_url_node_by_uuid(urlnode_uuid)] + else: + urlnodes = tree.root_hartree.url_tree.traverse() + self.store_known_legitimate_tree(tree) + known_content = self.find_known_content(tree) + pipeline = self.redis.pipeline() + for urlnode, h in self._filter(urlnodes, known_content): + # Note: we can have multiple hahes on the same urlnode (see embedded resources). + # They are expected to be on the same domain as urlnode. This code work as expected. + pipeline.sadd(f'bh|{h}|legitimate', urlnode.hostname) + pipeline.execute() + + def contextualize_tree(self, tree: CrawledTree) -> CrawledTree: + hostnodes_with_malicious_content = set() + known_content = self.find_known_content(tree) + for urlnode in tree.root_hartree.url_tree.traverse(): + if urlnode.empty_response: + continue + + malicious = self.is_malicious(urlnode, known_content) + if malicious is True: + urlnode.add_feature('malicious', malicious) + hostnodes_with_malicious_content.add(urlnode.hostnode_uuid) + elif malicious is False: + # Marked as legitimate + urlnode.add_feature('legitimate', True) + elif not urlnode.empty_response and urlnode.body_hash in known_content: + urlnode.add_feature('legitimate', True) + + for hostnode_with_malicious_content in hostnodes_with_malicious_content: + hostnode = tree.root_hartree.get_host_node_by_uuid(hostnode_with_malicious_content) + hostnode.add_feature('malicious', malicious) + + for hostnode in tree.root_hartree.hostname_tree.traverse(): + if 'malicious' not in hostnode.features: + if all(urlnode.empty_response for urlnode in hostnode.urls): + hostnode.add_feature('all_empty', True) + continue + + legit = [urlnode.legitimate for urlnode in hostnode.urls if hasattr(urlnode, 'legitimate')] + if len(legit) == len(hostnode.urls) and all(legit): + hostnode.add_feature('legitimate', True) + + return tree + + def legitimate_body(self, body_hash: str, legitimate_hostname: str) -> None: + self.redis.sadd(f'bh|{body_hash}|legitimate', legitimate_hostname) + + def store_known_malicious_ressource(self, ressource_hash: str, details: Dict[str, str]): + known_malicious_ressource_file = get_homedir() / 'known_content' / 'malicious.json' + if known_malicious_ressource_file.exists(): + with open(known_malicious_ressource_file) as f: + to_store = json.load(f) + else: + to_store = {} + + if ressource_hash not in to_store: + to_store[ressource_hash] = {'target': set(), 'tag': set()} + else: + to_store[ressource_hash]['target'] = set(to_store[ressource_hash]['target']) + to_store[ressource_hash]['tag'] = set(to_store[ressource_hash]['tag']) + + if 'target' in details: + to_store[ressource_hash]['target'].add(details['target']) + if 'type' in details: + to_store[ressource_hash]['tag'].add(details['type']) + + with open(known_malicious_ressource_file, 'w') as f: + json.dump(to_store, f, indent=2, default=dump_to_json) + + def add_malicious(self, ressource_hash: str, details: Dict[str, str]): + self.store_known_malicious_ressource(ressource_hash, details) + p = self.redis.pipeline() + p.sadd('bh|malicious', ressource_hash) + if 'target' in details: + p.sadd(f'{ressource_hash}|target', details['target']) + if 'type' in details: + p.sadd(f'{ressource_hash}|tag', details['type']) + p.execute() + + def store_known_legitimate_ressource(self, ressource_hash: str, details: Dict[str, str]): + known_legitimate_ressource_file = get_homedir() / 'known_content' / 'legitimate.json' + if known_legitimate_ressource_file.exists(): + with open(known_legitimate_ressource_file) as f: + to_store = json.load(f) + else: + to_store = {} + + if ressource_hash not in to_store: + to_store[ressource_hash] = {'domain': set(), 'description': ''} + else: + to_store[ressource_hash]['domain'] = set(to_store[ressource_hash]['domain']) + + if 'domain' in details: + to_store[ressource_hash]['domain'].add(details['domain']) + if 'description' in details: + to_store[ressource_hash]['description'] = details['description'] + + with open(known_legitimate_ressource_file, 'w') as f: + json.dump(to_store, f, indent=2, default=dump_to_json) + + def add_legitimate(self, ressource_hash: str, details: Dict[str, str]): + self.store_known_legitimate_ressource(ressource_hash, details) + if 'domain' in details: + self.redis.sadd(f'bh|{ressource_hash}|legitimate', details['domain']) + elif 'description' in details: + # Library + self.redis.hset('known_content', ressource_hash, details['description']) + + # Query DB + + def is_legitimate(self, urlnode: URLNode, known_hashes: Iterable[str]) -> Optional[bool]: + """3 cases: + * True if *all* the contents are known legitimate + * False if *any* content is malicious + * None in all other cases + """ + status: List[Optional[bool]] = [] + for urlnode, h in self._filter(urlnode, known_hashes): + # Note: we can have multiple hahes on the same urlnode (see embedded resources). + # They are expected to be on the same domain as urlnode. This code work as expected. + if self.redis.sismember('bh|malicious', h): + # Malicious, no need to go any further + return False + hostnames = self.redis.smembers(f'bh|{h}|legitimate') + if hostnames: + if urlnode.hostname in hostnames: + status.append(True) # legitimate + continue + else: + return False # Malicious + else: + # NOTE: we do not return here, because we want to return False if *any* of the contents is malicious + status.append(None) # Unknown + if status and all(status): + return True # All the contents are known legitimate + return None + + def is_malicious(self, urlnode: URLNode, known_hashes: Iterable[str]) -> Optional[bool]: + """3 cases: + * True if *any* content is malicious + * False if *all* the contents are known legitimate + * None in all other cases + """ + legitimate = self.is_legitimate(urlnode, known_hashes) + if legitimate: + return False + elif legitimate is False: + return True + return None + + def legitimacy_details(self, urlnode: URLNode, known_hashes: Iterable[str]) -> Dict[str, Tuple[bool, Optional[List[str]]]]: + to_return = {} + for urlnode, h in self._filter(urlnode, known_hashes): + # Note: we can have multiple hahes on the same urlnode (see embedded resources). + # They are expected to be on the same domain as urlnode. This code work as expected. + hostnames = self.redis.smembers(f'bh|{h}|legitimate') + if hostnames: + if urlnode.hostname in hostnames: + to_return[h] = (True, hostnames) + else: + to_return[h] = (False, hostnames) + elif self.redis.sismember('bh|malicious', urlnode.body_hash): + to_return[h] = (False, None) + return to_return + + class Lookyloo(): def __init__(self) -> None: @@ -184,6 +473,11 @@ class Lookyloo(): if not self.sanejs.available: self.logger.warning('Unable to setup the SaneJS module') + if hasattr(self, 'sanejs') and self.sanejs.available: + self.context = Context(self.sanejs) + else: + self.context = Context() + if not self.redis.exists('cache_loaded'): self._init_existing_dumps() @@ -259,6 +553,16 @@ class Lookyloo(): return ct + def add_context(self, capture_uuid: str, urlnode_uuid: str, ressource_hash: str, legitimate: bool, malicious: bool, details: Dict[str, Dict[str, str]]): + if malicious: + self.context.add_malicious(ressource_hash, details['malicious']) + if legitimate: + self.context.add_legitimate(ressource_hash, details['legitimate']) + + def add_to_legitimate(self, capture_uuid: str, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None): + ct = self.get_crawled_tree(capture_uuid) + self.context.mark_as_legitimate(ct, hostnode_uuid, urlnode_uuid) + def load_tree(self, capture_uuid: str) -> Tuple[str, str, str, str, Dict[str, str]]: capture_dir = self.lookup_capture_dir(capture_uuid) if not capture_dir: @@ -268,6 +572,7 @@ class Lookyloo(): with open((capture_dir / 'meta'), 'r') as f: meta = json.load(f) ct = self.get_crawled_tree(capture_uuid) + ct = self.context.contextualize_tree(ct) return ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta def remove_pickle(self, capture_uuid: str) -> None: @@ -739,17 +1044,6 @@ class Lookyloo(): captures_list['different_url'].append((h_capture_uuid, url_uuid, cache['title'], cache['timestamp'], url_hostname)) return captures_list - def _format_sane_js_response(self, lookup_table: Dict, h: str) -> Optional[Union[str, Tuple]]: - if lookup_table.get(h): - if isinstance(lookup_table[h], list): - libname, version, path = lookup_table[h][0].split("|") - other_files = len(lookup_table[h]) - return libname, version, path, other_files - else: - # Predefined generic file - return lookup_table[h] - return None - def get_hostnode_investigator(self, capture_uuid: str, node_uuid: str) -> Tuple[HostNode, List[Dict[str, Any]]]: capture_dir = self.lookup_capture_dir(capture_uuid) if not capture_dir: @@ -762,10 +1056,7 @@ class Lookyloo(): if not hostnode: raise MissingUUID(f'Unable to find UUID {node_uuid} in {capture_dir}') - sanejs_lookups: Dict[str, List[str]] = {} - if hasattr(self, 'sanejs') and self.sanejs.available: - to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')] - sanejs_lookups = self.sanejs.hashes_lookup(to_lookup) + known_content = self.context.find_known_content(hostnode) urls: List[Dict[str, Any]] = [] for url in hostnode.urls: @@ -773,10 +1064,11 @@ class Lookyloo(): # * https vs http # * everything after the domain # * the full URL + legit_details = self.context.legitimacy_details(url, known_content) to_append: Dict[str, Any] = { 'encrypted': url.name.startswith('https'), 'url_path': url.name.split('/', 3)[-1], - 'url_object': url + 'url_object': url, } if not url.empty_response: @@ -797,21 +1089,16 @@ class Lookyloo(): continue freq_embedded = self.indexing.body_hash_fequency(h) to_append['embedded_ressources'][h] = freq_embedded + to_append['embedded_ressources'][h]['body_size'] = blob.getbuffer().nbytes to_append['embedded_ressources'][h]['type'] = mimetype if freq_embedded['hash_freq'] > 1: to_append['embedded_ressources'][h]['other_captures'] = self.hash_lookup(h, url.name, capture_uuid) - if hasattr(self, 'sanejs') and self.sanejs.available: - to_lookup = list(to_append['embedded_ressources'].keys()) - sanejs_lookups_embedded = self.sanejs.hashes_lookup(to_lookup) - for h in to_append['embedded_ressources'].keys(): - sane_js_match = self._format_sane_js_response(sanejs_lookups_embedded, h) - if sane_js_match: - to_append['embedded_ressources'][h]['sane_js'] = sane_js_match + for h in to_append['embedded_ressources'].keys(): + to_append['embedded_ressources'][h]['known_content'] = known_content.get(h) + to_append['embedded_ressources'][h]['legitimacy'] = legit_details.get(h) - # Optional: SaneJS information - sane_js_match = self._format_sane_js_response(sanejs_lookups, url.body_hash) - if sane_js_match: - to_append['sane_js'] = sane_js_match + to_append['known_content'] = known_content.get(url.body_hash) + to_append['legitimacy'] = legit_details.get(url.body_hash) # Optional: Cookies sent to server in request -> map to nodes who set the cookie in response if hasattr(url, 'cookies_sent'): diff --git a/lookyloo/modules.py b/lookyloo/modules.py index a14ee57e..c70c2398 100644 --- a/lookyloo/modules.py +++ b/lookyloo/modules.py @@ -19,53 +19,6 @@ from pyeupi import PyEUPI class SaneJavaScript(): - skip_lookup: Dict[str, str] = { - "717ea0ff7f3f624c268eccb244e24ec1305ab21557abb3d6f1a7e183ff68a2d28f13d1d2af926c9ef6d1fb16dd8cbe34cd98cacf79091dddc7874dcee21ecfdc": "This is a 1*1 pixel GIF", - "e508d5d17e94d14b126164082342a9ca4774f404e87a3dd56c26812493ee18d9c3d6daacca979134a94a003066aca24116de874596d00d1e52130c1283d54209": "This is a 1*1 pixel GIF", - "2d073e10ae40fde434eb31cbedd581a35cd763e51fb7048b88caa5f949b1e6105e37a228c235bc8976e8db58ed22149cfccf83b40ce93a28390566a28975744a": "This is a 1*1 pixel GIF", - "84e24a70b78e9de9c9d0dfeb49f3f4247dbc1c715d8844471ee40669270682e199d48f5fbec62bd984c9c0270534b407c4d2561dd6c05adec3c83c1534f32d5c": "This is a 1*1 pixel GIF", - "d5da26b5d496edb0221df1a4057a8b0285d15592a8f8dc7016a294df37ed335f3fde6a2252962e0df38b62847f8b771463a0124ef3f84299f262ed9d9d3cee4c": "This is a 1*1 pixel GIF", - "f7a5f748f4c0d3096a3ca972886fe9a9dff5dce7792779ec6ffc42fa880b3815e2e4c3bdea452352f3844b81864c9bfb7861f66ac961cfa66cb9cb4febe568e8": "This is a 1*1 pixel GIF", - "b2ca25a3311dc42942e046eb1a27038b71d689925b7d6b3ebb4d7cd2c7b9a0c7de3d10175790ac060dc3f8acf3c1708c336626be06879097f4d0ecaa7f567041": "This is a 1*1 pixel GIF", - "b8d82d64ec656c63570b82215564929adad167e61643fd72283b94f3e448ef8ab0ad42202f3537a0da89960bbdc69498608fc6ec89502c6c338b6226c8bf5e14": "This is a 1*1 pixel GIF", - "2991c3aa1ba61a62c1cccd990c0679a1fb8dccd547d153ec0920b91a75ba20820de1d1c206f66d083bf2585d35050f0a39cd7a3e11c03882dafec907d27a0180": "This is a 1*1 pixel GIF", - "b1a6cfa7b21dbb0b281d241af609f3ba7f3a63e5668095bba912bf7cfd7f0320baf7c3b0bfabd0f8609448f39902baeb145ba7a2d8177fe22a6fcea03dd29be1": "This is a 1*1 pixel GIF", - "ebfe0c0df4bcc167d5cb6ebdd379f9083df62bef63a23818e1c6adf0f64b65467ea58b7cd4d03cf0a1b1a2b07fb7b969bf35f25f1f8538cc65cf3eebdf8a0910": "This is a 1*1 pixel GIF", - "1d68b92e8d822fe82dc7563edd7b37f3418a02a89f1a9f0454cca664c2fc2565235e0d85540ff9be0b20175be3f5b7b4eae1175067465d5cca13486aab4c582c": "This is a 1*1 pixel GIF", - "ac44da7f455bfae52b883639964276026fb259320902aa813d0333e021c356a7b3e3537b297f9a2158e588c302987ce0854866c039d1bb0ffb27f67560739db2": "This is a 1*1 pixel GIF", - "921944dc10fbfb6224d69f0b3ac050f4790310fd1bcac3b87c96512ad5ed9a268824f3f5180563d372642071b4704c979d209baf40bc0b1c9a714769aba7dfc7": "This is a 1*1 pixel GIF", - "89dfc38ec77cf258362e4db7c8203cae8a02c0fe4f99265b0539ec4f810c84f8451e22c9bef1ebc59b4089af7e93e378e053c542a5967ec4912d4c1fc5de22f0": "This is a 1*1 pixel GIF", - "280ea4383ee6b37051d91c5af30a5ce72aa4439340fc6d31a4fbe7ba8a8156eb7893891d5b2371b9fc4934a78f08de3d57e5b63fa9d279a317dcbefb8a07a6b0": "This is a 1*1 pixel GIF", - "3844065e1dd778a05e8cc39901fbf3191ded380d594359df137901ec56ca52e03d57eb60acc2421a0ee74f0733bbb5d781b7744685c26fb013a236f49b02fed3": "This is a 1*1 pixel GIF", - "bd9ab35dde3a5242b04c159187732e13b0a6da50ddcff7015dfb78cdd68743e191eaf5cddedd49bef7d2d5a642c217272a40e5ba603fe24ca676a53f8c417c5d": "This is a 1*1 pixel GIF", - "d052ecec2839340876eb57247cfc2e777dd7f2e868dc37cd3f3f740c8deb94917a0c9f2a4fc8229987a0b91b04726de2d1e9f6bcbe3f9bef0e4b7e0d7f65ea12": "This is a 1*1 pixel GIF", - "8717074ddf1198d27b9918132a550cb4ba343794cc3d304a793f9d78c9ff6c4929927b414141d40b6f6ad296725520f4c63edeb660ed530267766c2ab74ee4a9": "This is a 1*1 pixel GIF", - "6834f1548f26b94357fcc3312a3491e8c87080a84f678f990beb2c745899a01e239964521e64a534d7d5554222f728af966ec6ec8291bc64d2005861bcfd78ec": "This is a 1*1 pixel GIF", - "3be8176915593e79bc280d08984a16c29c495bc53be9b439276094b8dcd3764a3c72a046106a06b958e08e67451fe02743175c621a1faa261fe7a9691cc77141": "This is a 1*1 pixel GIF", - "826225fc21717d8861a05b9d2f959539aad2d2b131b2afed75d88fbca535e1b0d5a0da8ac69713a0876a0d467848a37a0a7f926aeafad8cf28201382d16466ab": "This is a 1*1 pixel GIF", - "202612457d9042fe853daab3ddcc1f0f960c5ffdbe8462fa435713e4d1d85ff0c3f197daf8dba15bda9f5266d7e1f9ecaeee045cbc156a4892d2f931fe6fa1bb": "This is a 1*1 pixel GIF", - "b82c6aa1ae927ade5fadbbab478cfaef26d21c1ac441f48e69cfc04cdb779b1e46d7668b4368b933213276068e52f9060228907720492a70fd9bc897191ee77c": "This is a 1*1 pixel GIF", - "763de1053a56a94eef4f72044adb2aa370b98ffa6e0add0b1cead7ee27da519e223921c681ae1db3311273f45d0dd3dc022d102d42ce210c90cb3e761b178438": "This is a 1*1 pixel GIF", - "69e2da5cdc318fc237eaa243b6ea7ecc83b68dbdea8478dc69154abdda86ecb4e16c35891cc1facb3ce7e0cf19d5abf189c50f59c769777706f4558f6442abbc": "This is a 1*1 pixel GIF", - "16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516": "This is a 1*1 pixel GIF", - "01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb": "This is a 1*1 pixel GIF", - "71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746": "This is a 1*1 pixel GIF", - "49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a": "This is a 1*1 pixel GIF", - "c57ebbadcf59f982ba28da35fdbd5e5369a8500a2e1edad0dc9c9174de6fd99f437953732e545b95d3de5943c61077b6b949c989f49553ff2e483f68fcc30641": "This is a 1*1 pixel GIF", - "c87bf81fd70cf6434ca3a6c05ad6e9bd3f1d96f77dddad8d45ee043b126b2cb07a5cf23b4137b9d8462cd8a9adf2b463ab6de2b38c93db72d2d511ca60e3b57e": "This is a 1*1 pixel GIF", - "fd8b021f0236e487bfee13bf8f0ae98760abc492f7ca3023e292631979e135cb4ccb0c89b6234971b060ad72c0ca4474cbb5092c6c7a3255d81a54a36277b486": "This is a 1*1 pixel GIF", - "235479f42cbbe0a4b0100167fece0d14c9b47d272b3ba8322bcfe8539f055bf31d500e7b2995cc968ebf73034e039f59c5f0f9410428663034bf119d74b5672c": "This is a 1*1 pixel GIF", - "a85e09c3b5dbb560f4e03ba880047dbc8b4999a64c1f54fbfbca17ee0bcbed3bc6708d699190b56668e464a59358d6b534c3963a1329ba01db21075ef5bedace": "This is a 1*1 pixel GIF", - "27656d6106a6da0c84174ba7a6307e6f1c4b3f2cc085c8466b6a25d54331035dabc7081aac208d960d8d37c5577547628c0d1c4b77bb4cf254c71859673feec1": "This is a 1*1 pixel GIF", - # "": "This is a 1*1 pixel GIF", - "f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498": "This is a 1*1 pixel PNG", - "dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99": "This is a 1*1 pixel PNG", - "c2c239cb5cdd0b670780ad6414ef6be9ccd4c21ce46bb93d1fa3120ac812f1679445162978c3df05cb2e1582a1844cc4c41cf74960b8fdae3123999c5d2176cc": "This is a 1*1 pixel PNG", - "6ad523f5b65487369d305613366b9f68dcdeee225291766e3b25faf45439ca069f614030c08ca54c714fdbf7a944fac489b1515a8bf9e0d3191e1bcbbfe6a9df": "This is a 1*1 pixel PNG", - # "": "This is a 1*1 pixel PNG", - "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": "This is an empty file" - } - def __init__(self, config: Dict[str, Any]): if not ('enabled' in config or config['enabled']): self.available = False @@ -78,7 +31,7 @@ class SaneJavaScript(): self.storage_dir = get_homedir() / 'sanejs' self.storage_dir.mkdir(parents=True, exist_ok=True) - def hashes_lookup(self, sha512: Union[List[str], str], force: bool=False) -> Dict[str, Any]: + def hashes_lookup(self, sha512: Union[List[str], str], force: bool=False) -> Dict[str, List[str]]: if isinstance(sha512, str): hashes = [sha512] else: @@ -92,12 +45,13 @@ class SaneJavaScript(): with sanejs_unknowns.open() as f: unknown_hashes = [line.strip() for line in f.readlines()] - to_return: Dict[str, Union[str, List[str]]] = {h: details for h, details in self.skip_lookup.items() if h in sha512} + to_return: Dict[str, List[str]] = {} - to_lookup = [h for h in hashes if h not in self.skip_lookup] - if not force: - to_lookup = [h for h in to_lookup if (h not in unknown_hashes - and not (today_dir / h).exists())] + if force: + to_lookup = hashes + else: + to_lookup = [h for h in hashes if (h not in unknown_hashes + and not (today_dir / h).exists())] for h in to_lookup: response = self.client.sha512(h) if 'error' in response: diff --git a/poetry.lock b/poetry.lock index d8f4707f..35c52787 100644 --- a/poetry.lock +++ b/poetry.lock @@ -56,13 +56,12 @@ description = "Classes Without Boilerplate" name = "attrs" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "19.3.0" +version = "20.1.0" [package.extras] -azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] -dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] -docs = ["sphinx", "zope.interface"] -tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] [[package]] category = "main" @@ -109,7 +108,7 @@ description = "Bootstrap helper for Flask/Jinja2." name = "bootstrap-flask" optional = false python-versions = "*" -version = "1.4" +version = "1.5" [package.dependencies] Flask = "*" @@ -140,7 +139,7 @@ description = "Foreign Function Interface for Python calling C code." name = "cffi" optional = false python-versions = "*" -version = "1.14.1" +version = "1.14.2" [package.dependencies] pycparser = "*" @@ -198,7 +197,7 @@ description = "cryptography is a package which provides cryptographic recipes an name = "cryptography" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" -version = "3.0" +version = "3.1" [package.dependencies] cffi = ">=1.8,<1.11.3 || >1.11.3" @@ -207,7 +206,6 @@ six = ">=1.4.1" [package.extras] docs = ["sphinx (>=1.6.5,<1.8.0 || >1.8.0,<3.1.0 || >3.1.0,<3.1.1 || >3.1.1)", "sphinx-rtd-theme"] docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] -idna = ["idna (>=2.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] ssh = ["bcrypt (>=3.1.5)"] test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"] @@ -242,7 +240,15 @@ description = "A Python Environment for (phylogenetic) Tree Exploration" name = "ete3" optional = false python-versions = "*" -version = "3.1.1" +version = "3.1.2" + +[[package]] +category = "main" +description = "Infer file type and MIME type of any file/buffer. No external dependencies." +name = "filetype" +optional = false +python-versions = "*" +version = "1.0.7" [[package]] category = "main" @@ -297,12 +303,14 @@ description = "HTTP Archive (HAR) to ETE Toolkit generator" name = "har2tree" optional = false python-versions = ">=3.6,<4.0" -version = "1.2.1" +version = "1.2.2" [package.dependencies] beautifulsoup4 = ">=4.8.2,<5.0.0" ete3 = ">=3.1.1,<4.0.0" +filetype = ">=1.0.7,<2.0.0" lxml = ">=4.4.2,<5.0.0" +numpy = ">=1.19.1,<2.0.0" publicsuffix2 = ">=2.20191221,<3.0" six = ">=1.14.0,<2.0.0" @@ -368,7 +376,7 @@ description = "IPython: Productive Interactive Computing" name = "ipython" optional = false python-versions = ">=3.7" -version = "7.17.0" +version = "7.18.1" [package.dependencies] appnope = "*" @@ -376,7 +384,7 @@ backcall = "*" colorama = "*" decorator = "*" jedi = ">=0.10" -pexpect = "*" +pexpect = ">4.3" pickleshare = "*" prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0" pygments = "*" @@ -467,7 +475,7 @@ description = "More routines for operating on iterables, beyond itertools" name = "more-itertools" optional = false python-versions = ">=3.5" -version = "8.4.0" +version = "8.5.0" [[package]] category = "main" @@ -501,6 +509,14 @@ optional = false python-versions = "*" version = "0.4.3" +[[package]] +category = "main" +description = "NumPy is the fundamental package for array computing with Python." +name = "numpy" +optional = false +python-versions = ">=3.6" +version = "1.19.1" + [[package]] category = "main" description = "Core utilities for Python packages" @@ -580,7 +596,7 @@ description = "Library for building powerful interactive command lines in Python name = "prompt-toolkit" optional = false python-versions = ">=3.6.1" -version = "3.0.5" +version = "3.0.7" [package.dependencies] wcwidth = "*" @@ -986,7 +1002,7 @@ description = "Backported and Experimental Type Hints for Python 3.5+" name = "typing-extensions" optional = false python-versions = "*" -version = "3.7.4.2" +version = "3.7.4.3" [[package]] category = "main" @@ -1130,8 +1146,8 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, - {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, + {file = "attrs-20.1.0-py2.py3-none-any.whl", hash = "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"}, + {file = "attrs-20.1.0.tar.gz", hash = "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a"}, ] automat = [ {file = "Automat-20.2.0-py2.py3-none-any.whl", hash = "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"}, @@ -1147,8 +1163,8 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.1.tar.gz", hash = "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7"}, ] bootstrap-flask = [ - {file = "Bootstrap-Flask-1.4.tar.gz", hash = "sha256:f5e75b7fb976ba2268eac88240e1a9b249c7848a0cb639f45a6be14d913c88c2"}, - {file = "Bootstrap_Flask-1.4-py2.py3-none-any.whl", hash = "sha256:a64368423512bff9e12b2d9f63be510bd3926f31e61b6be3760761d67a47a786"}, + {file = "Bootstrap-Flask-1.5.tar.gz", hash = "sha256:94b8e67f7ba15e8e6ba83e7ca30aa784f45c8d713a18d8fbf013a59ce9370954"}, + {file = "Bootstrap_Flask-1.5-py2.py3-none-any.whl", hash = "sha256:77f26a4ecd749063433b0e8780652c41c1c29bc91bf88756f07330fef3158cbb"}, ] cchardet = [ {file = "cchardet-2.1.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:2aa1b008965c703ad6597361b0f6d427c8971fe94a2c99ec3724c228ae50d6a6"}, @@ -1186,34 +1202,34 @@ certifi = [ {file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"}, ] cffi = [ - {file = "cffi-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:66dd45eb9530e3dde8f7c009f84568bc7cac489b93d04ac86e3111fb46e470c2"}, - {file = "cffi-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:4f53e4128c81ca3212ff4cf097c797ab44646a40b42ec02a891155cd7a2ba4d8"}, - {file = "cffi-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:833401b15de1bb92791d7b6fb353d4af60dc688eaa521bd97203dcd2d124a7c1"}, - {file = "cffi-1.14.1-cp27-cp27m-win32.whl", hash = "sha256:26f33e8f6a70c255767e3c3f957ccafc7f1f706b966e110b855bfe944511f1f9"}, - {file = "cffi-1.14.1-cp27-cp27m-win_amd64.whl", hash = "sha256:b87dfa9f10a470eee7f24234a37d1d5f51e5f5fa9eeffda7c282e2b8f5162eb1"}, - {file = "cffi-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:effd2ba52cee4ceff1a77f20d2a9f9bf8d50353c854a282b8760ac15b9833168"}, - {file = "cffi-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bac0d6f7728a9cc3c1e06d4fcbac12aaa70e9379b3025b27ec1226f0e2d404cf"}, - {file = "cffi-1.14.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:d6033b4ffa34ef70f0b8086fd4c3df4bf801fee485a8a7d4519399818351aa8e"}, - {file = "cffi-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:8416ed88ddc057bab0526d4e4e9f3660f614ac2394b5e019a628cdfff3733849"}, - {file = "cffi-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:892daa86384994fdf4856cb43c93f40cbe80f7f95bb5da94971b39c7f54b3a9c"}, - {file = "cffi-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:c991112622baee0ae4d55c008380c32ecfd0ad417bcd0417ba432e6ba7328caa"}, - {file = "cffi-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:fcf32bf76dc25e30ed793145a57426064520890d7c02866eb93d3e4abe516948"}, - {file = "cffi-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f960375e9823ae6a07072ff7f8a85954e5a6434f97869f50d0e41649a1c8144f"}, - {file = "cffi-1.14.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a6d28e7f14ecf3b2ad67c4f106841218c8ab12a0683b1528534a6c87d2307af3"}, - {file = "cffi-1.14.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:cda422d54ee7905bfc53ee6915ab68fe7b230cacf581110df4272ee10462aadc"}, - {file = "cffi-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:4a03416915b82b81af5502459a8a9dd62a3c299b295dcdf470877cb948d655f2"}, - {file = "cffi-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:4ce1e995aeecf7cc32380bc11598bfdfa017d592259d5da00fc7ded11e61d022"}, - {file = "cffi-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e23cb7f1d8e0f93addf0cae3c5b6f00324cccb4a7949ee558d7b6ca973ab8ae9"}, - {file = "cffi-1.14.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ddff0b2bd7edcc8c82d1adde6dbbf5e60d57ce985402541cd2985c27f7bec2a0"}, - {file = "cffi-1.14.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:f90c2267101010de42f7273c94a1f026e56cbc043f9330acd8a80e64300aba33"}, - {file = "cffi-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:3cd2c044517f38d1b577f05927fb9729d3396f1d44d0c659a445599e79519792"}, - {file = "cffi-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fa72a52a906425416f41738728268072d5acfd48cbe7796af07a923236bcf96"}, - {file = "cffi-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:267adcf6e68d77ba154334a3e4fc921b8e63cbb38ca00d33d40655d4228502bc"}, - {file = "cffi-1.14.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:d3148b6ba3923c5850ea197a91a42683f946dba7e8eb82dfa211ab7e708de939"}, - {file = "cffi-1.14.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:98be759efdb5e5fa161e46d404f4e0ce388e72fbf7d9baf010aff16689e22abe"}, - {file = "cffi-1.14.1-cp38-cp38-win32.whl", hash = "sha256:6923d077d9ae9e8bacbdb1c07ae78405a9306c8fd1af13bfa06ca891095eb995"}, - {file = "cffi-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:b1d6ebc891607e71fd9da71688fcf332a6630b7f5b7f5549e6e631821c0e5d90"}, - {file = "cffi-1.14.1.tar.gz", hash = "sha256:b2a2b0d276a136146e012154baefaea2758ef1f56ae9f4e01c612b0831e0bd2f"}, + {file = "cffi-1.14.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:da9d3c506f43e220336433dffe643fbfa40096d408cb9b7f2477892f369d5f82"}, + {file = "cffi-1.14.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23e44937d7695c27c66a54d793dd4b45889a81b35c0751ba91040fe825ec59c4"}, + {file = "cffi-1.14.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:0da50dcbccd7cb7e6c741ab7912b2eff48e85af217d72b57f80ebc616257125e"}, + {file = "cffi-1.14.2-cp27-cp27m-win32.whl", hash = "sha256:76ada88d62eb24de7051c5157a1a78fd853cca9b91c0713c2e973e4196271d0c"}, + {file = "cffi-1.14.2-cp27-cp27m-win_amd64.whl", hash = "sha256:15a5f59a4808f82d8ec7364cbace851df591c2d43bc76bcbe5c4543a7ddd1bf1"}, + {file = "cffi-1.14.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:e4082d832e36e7f9b2278bc774886ca8207346b99f278e54c9de4834f17232f7"}, + {file = "cffi-1.14.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:57214fa5430399dffd54f4be37b56fe22cedb2b98862550d43cc085fb698dc2c"}, + {file = "cffi-1.14.2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:6843db0343e12e3f52cc58430ad559d850a53684f5b352540ca3f1bc56df0731"}, + {file = "cffi-1.14.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:577791f948d34d569acb2d1add5831731c59d5a0c50a6d9f629ae1cefd9ca4a0"}, + {file = "cffi-1.14.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8662aabfeab00cea149a3d1c2999b0731e70c6b5bac596d95d13f643e76d3d4e"}, + {file = "cffi-1.14.2-cp35-cp35m-win32.whl", hash = "sha256:837398c2ec00228679513802e3744d1e8e3cb1204aa6ad408b6aff081e99a487"}, + {file = "cffi-1.14.2-cp35-cp35m-win_amd64.whl", hash = "sha256:bf44a9a0141a082e89c90e8d785b212a872db793a0080c20f6ae6e2a0ebf82ad"}, + {file = "cffi-1.14.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:29c4688ace466a365b85a51dcc5e3c853c1d283f293dfcc12f7a77e498f160d2"}, + {file = "cffi-1.14.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:99cc66b33c418cd579c0f03b77b94263c305c389cb0c6972dac420f24b3bf123"}, + {file = "cffi-1.14.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:65867d63f0fd1b500fa343d7798fa64e9e681b594e0a07dc934c13e76ee28fb1"}, + {file = "cffi-1.14.2-cp36-cp36m-win32.whl", hash = "sha256:f5033952def24172e60493b68717792e3aebb387a8d186c43c020d9363ee7281"}, + {file = "cffi-1.14.2-cp36-cp36m-win_amd64.whl", hash = "sha256:7057613efefd36cacabbdbcef010e0a9c20a88fc07eb3e616019ea1692fa5df4"}, + {file = "cffi-1.14.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6539314d84c4d36f28d73adc1b45e9f4ee2a89cdc7e5d2b0a6dbacba31906798"}, + {file = "cffi-1.14.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:672b539db20fef6b03d6f7a14b5825d57c98e4026401fce838849f8de73fe4d4"}, + {file = "cffi-1.14.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:95e9094162fa712f18b4f60896e34b621df99147c2cee216cfa8f022294e8e9f"}, + {file = "cffi-1.14.2-cp37-cp37m-win32.whl", hash = "sha256:b9aa9d8818c2e917fa2c105ad538e222a5bce59777133840b93134022a7ce650"}, + {file = "cffi-1.14.2-cp37-cp37m-win_amd64.whl", hash = "sha256:e4b9b7af398c32e408c00eb4e0d33ced2f9121fd9fb978e6c1b57edd014a7d15"}, + {file = "cffi-1.14.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e613514a82539fc48291d01933951a13ae93b6b444a88782480be32245ed4afa"}, + {file = "cffi-1.14.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9b219511d8b64d3fa14261963933be34028ea0e57455baf6781fe399c2c3206c"}, + {file = "cffi-1.14.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:c0b48b98d79cf795b0916c57bebbc6d16bb43b9fc9b8c9f57f4cf05881904c75"}, + {file = "cffi-1.14.2-cp38-cp38-win32.whl", hash = "sha256:15419020b0e812b40d96ec9d369b2bc8109cc3295eac6e013d3261343580cc7e"}, + {file = "cffi-1.14.2-cp38-cp38-win_amd64.whl", hash = "sha256:12a453e03124069b6896107ee133ae3ab04c624bb10683e1ed1c1663df17c13c"}, + {file = "cffi-1.14.2.tar.gz", hash = "sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b"}, ] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, @@ -1236,25 +1252,28 @@ constantly = [ {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"}, ] cryptography = [ - {file = "cryptography-3.0-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:ab49edd5bea8d8b39a44b3db618e4783ef84c19c8b47286bf05dfdb3efb01c83"}, - {file = "cryptography-3.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:124af7255ffc8e964d9ff26971b3a6153e1a8a220b9a685dc407976ecb27a06a"}, - {file = "cryptography-3.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:51e40123083d2f946794f9fe4adeeee2922b581fa3602128ce85ff813d85b81f"}, - {file = "cryptography-3.0-cp27-cp27m-win32.whl", hash = "sha256:dea0ba7fe6f9461d244679efa968d215ea1f989b9c1957d7f10c21e5c7c09ad6"}, - {file = "cryptography-3.0-cp27-cp27m-win_amd64.whl", hash = "sha256:8ecf9400d0893836ff41b6f977a33972145a855b6efeb605b49ee273c5e6469f"}, - {file = "cryptography-3.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:0c608ff4d4adad9e39b5057de43657515c7da1ccb1807c3a27d4cf31fc923b4b"}, - {file = "cryptography-3.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:bec7568c6970b865f2bcebbe84d547c52bb2abadf74cefce396ba07571109c67"}, - {file = "cryptography-3.0-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:0cbfed8ea74631fe4de00630f4bb592dad564d57f73150d6f6796a24e76c76cd"}, - {file = "cryptography-3.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:a09fd9c1cca9a46b6ad4bea0a1f86ab1de3c0c932364dbcf9a6c2a5eeb44fa77"}, - {file = "cryptography-3.0-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:ce82cc06588e5cbc2a7df3c8a9c778f2cb722f56835a23a68b5a7264726bb00c"}, - {file = "cryptography-3.0-cp35-cp35m-win32.whl", hash = "sha256:9367d00e14dee8d02134c6c9524bb4bd39d4c162456343d07191e2a0b5ec8b3b"}, - {file = "cryptography-3.0-cp35-cp35m-win_amd64.whl", hash = "sha256:384d7c681b1ab904fff3400a6909261cae1d0939cc483a68bdedab282fb89a07"}, - {file = "cryptography-3.0-cp36-cp36m-win32.whl", hash = "sha256:4d355f2aee4a29063c10164b032d9fa8a82e2c30768737a2fd56d256146ad559"}, - {file = "cryptography-3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:45741f5499150593178fc98d2c1a9c6722df88b99c821ad6ae298eff0ba1ae71"}, - {file = "cryptography-3.0-cp37-cp37m-win32.whl", hash = "sha256:8ecef21ac982aa78309bb6f092d1677812927e8b5ef204a10c326fc29f1367e2"}, - {file = "cryptography-3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4b9303507254ccb1181d1803a2080a798910ba89b1a3c9f53639885c90f7a756"}, - {file = "cryptography-3.0-cp38-cp38-win32.whl", hash = "sha256:8713ddb888119b0d2a1462357d5946b8911be01ddbf31451e1d07eaa5077a261"}, - {file = "cryptography-3.0-cp38-cp38-win_amd64.whl", hash = "sha256:bea0b0468f89cdea625bb3f692cd7a4222d80a6bdafd6fb923963f2b9da0e15f"}, - {file = "cryptography-3.0.tar.gz", hash = "sha256:8e924dbc025206e97756e8903039662aa58aa9ba357d8e1d8fc29e3092322053"}, + {file = "cryptography-3.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:969ae512a250f869c1738ca63be843488ff5cc031987d302c1f59c7dbe1b225f"}, + {file = "cryptography-3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:b45ab1c6ece7c471f01c56f5d19818ca797c34541f0b2351635a5c9fe09ac2e0"}, + {file = "cryptography-3.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:247df238bc05c7d2e934a761243bfdc67db03f339948b1e2e80c75d41fc7cc36"}, + {file = "cryptography-3.1-cp27-cp27m-win32.whl", hash = "sha256:10c9775a3f31610cf6b694d1fe598f2183441de81cedcf1814451ae53d71b13a"}, + {file = "cryptography-3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:9f734423eb9c2ea85000aa2476e0d7a58e021bc34f0a373ac52a5454cd52f791"}, + {file = "cryptography-3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e7563eb7bc5c7e75a213281715155248cceba88b11cb4b22957ad45b85903761"}, + {file = "cryptography-3.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:94191501e4b4009642be21dde2a78bd3c2701a81ee57d3d3d02f1d99f8b64a9e"}, + {file = "cryptography-3.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:dc3f437ca6353979aace181f1b790f0fc79e446235b14306241633ab7d61b8f8"}, + {file = "cryptography-3.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:725875681afe50b41aee7fdd629cedbc4720bab350142b12c55c0a4d17c7416c"}, + {file = "cryptography-3.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:321761d55fb7cb256b771ee4ed78e69486a7336be9143b90c52be59d7657f50f"}, + {file = "cryptography-3.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:2a27615c965173c4c88f2961cf18115c08fedfb8bdc121347f26e8458dc6d237"}, + {file = "cryptography-3.1-cp35-cp35m-win32.whl", hash = "sha256:e7dad66a9e5684a40f270bd4aee1906878193ae50a4831922e454a2a457f1716"}, + {file = "cryptography-3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4005b38cd86fc51c955db40b0f0e52ff65340874495af72efabb1bb8ca881695"}, + {file = "cryptography-3.1-cp36-abi3-win32.whl", hash = "sha256:cc6096c86ec0de26e2263c228fb25ee01c3ff1346d3cfc219d67d49f303585af"}, + {file = "cryptography-3.1-cp36-abi3-win_amd64.whl", hash = "sha256:2e26223ac636ca216e855748e7d435a1bf846809ed12ed898179587d0cf74618"}, + {file = "cryptography-3.1-cp36-cp36m-win32.whl", hash = "sha256:7a63e97355f3cd77c94bd98c59cb85fe0efd76ea7ef904c9b0316b5bbfde6ed1"}, + {file = "cryptography-3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:4b9e96543d0784acebb70991ebc2dbd99aa287f6217546bb993df22dd361d41c"}, + {file = "cryptography-3.1-cp37-cp37m-win32.whl", hash = "sha256:eb80a288e3cfc08f679f95da72d2ef90cb74f6d8a8ba69d2f215c5e110b2ca32"}, + {file = "cryptography-3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:180c9f855a8ea280e72a5d61cf05681b230c2dce804c48e9b2983f491ecc44ed"}, + {file = "cryptography-3.1-cp38-cp38-win32.whl", hash = "sha256:fa7fbcc40e2210aca26c7ac8a39467eae444d90a2c346cbcffd9133a166bcc67"}, + {file = "cryptography-3.1-cp38-cp38-win_amd64.whl", hash = "sha256:548b0818e88792318dc137d8b1ec82a0ab0af96c7f0603a00bb94f896fbf5e10"}, + {file = "cryptography-3.1.tar.gz", hash = "sha256:26409a473cc6278e4c90f782cd5968ebad04d3911ed1c402fc86908c17633e08"}, ] cssselect = [ {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"}, @@ -1268,7 +1287,11 @@ defang = [ {file = "defang-0.5.3.tar.gz", hash = "sha256:86aeff658d7cd4c3b61d16089872e1c1f0a1b7b3c64d4ca9525c017caeb284d7"}, ] ete3 = [ - {file = "ete3-3.1.1.tar.gz", hash = "sha256:870a3d4b496a36fbda4b13c7c6b9dfa7638384539ae93551ec7acb377fb9c385"}, + {file = "ete3-3.1.2.tar.gz", hash = "sha256:4fc987b8c529889d6608fab1101f1455cb5cbd42722788de6aea9c7d0a8e59e9"}, +] +filetype = [ + {file = "filetype-1.0.7-py2.py3-none-any.whl", hash = "sha256:353369948bb1c09b8b3ea3d78390b5586e9399bff9aab894a1dff954e31a66f6"}, + {file = "filetype-1.0.7.tar.gz", hash = "sha256:da393ece8d98b47edf2dd5a85a2c8733e44b769e32c71af4cd96ed8d38d96aa7"}, ] flask = [ {file = "Flask-1.1.2-py2.py3-none-any.whl", hash = "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557"}, @@ -1283,8 +1306,8 @@ gunicorn = [ {file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"}, ] har2tree = [ - {file = "har2tree-1.2.1-py3-none-any.whl", hash = "sha256:180d090a455c435f5bd8dd20e8e66239722177e60e99d1e3dc89a21796aad11c"}, - {file = "har2tree-1.2.1.tar.gz", hash = "sha256:7201a1b9764127b4ef1d0ea7f541eeac3a617eb61c156d26e17878b77395ac2b"}, + {file = "har2tree-1.2.2-py3-none-any.whl", hash = "sha256:0cdd46181dad9ae738eeedba054584299e3a60233cc15d22f163039974ca1daf"}, + {file = "har2tree-1.2.2.tar.gz", hash = "sha256:5ae6ccad4b93bcd53b57cc4c371cc8ae1a1d614240e3228d24ca36aea5927291"}, ] hyperlink = [ {file = "hyperlink-20.0.1-py2.py3-none-any.whl", hash = "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"}, @@ -1307,8 +1330,8 @@ iniconfig = [ {file = "iniconfig-1.0.1.tar.gz", hash = "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"}, ] ipython = [ - {file = "ipython-7.17.0-py3-none-any.whl", hash = "sha256:5a8f159ca8b22b9a0a1f2a28befe5ad2b703339afb58c2ffe0d7c8d7a3af5999"}, - {file = "ipython-7.17.0.tar.gz", hash = "sha256:b70974aaa2674b05eb86a910c02ed09956a33f2dd6c71afc60f0b128a77e7f28"}, + {file = "ipython-7.18.1-py3-none-any.whl", hash = "sha256:2e22c1f74477b5106a6fb301c342ab8c64bb75d702e350f05a649e8cb40a0fb8"}, + {file = "ipython-7.18.1.tar.gz", hash = "sha256:a331e78086001931de9424940699691ad49dfb457cea31f5471eae7b78222d5e"}, ] ipython-genutils = [ {file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"}, @@ -1386,8 +1409,8 @@ markupsafe = [ {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, ] more-itertools = [ - {file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"}, - {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"}, + {file = "more-itertools-8.5.0.tar.gz", hash = "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20"}, + {file = "more_itertools-8.5.0-py3-none-any.whl", hash = "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"}, ] multidict = [ {file = "multidict-4.7.6-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000"}, @@ -1428,6 +1451,34 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] +numpy = [ + {file = "numpy-1.19.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b1cca51512299841bf69add3b75361779962f9cee7d9ee3bb446d5982e925b69"}, + {file = "numpy-1.19.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c9591886fc9cbe5532d5df85cb8e0cc3b44ba8ce4367bd4cf1b93dc19713da72"}, + {file = "numpy-1.19.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:cf1347450c0b7644ea142712619533553f02ef23f92f781312f6a3553d031fc7"}, + {file = "numpy-1.19.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:ed8a311493cf5480a2ebc597d1e177231984c818a86875126cfd004241a73c3e"}, + {file = "numpy-1.19.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3673c8b2b29077f1b7b3a848794f8e11f401ba0b71c49fbd26fb40b71788b132"}, + {file = "numpy-1.19.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:56ef7f56470c24bb67fb43dae442e946a6ce172f97c69f8d067ff8550cf782ff"}, + {file = "numpy-1.19.1-cp36-cp36m-win32.whl", hash = "sha256:aaf42a04b472d12515debc621c31cf16c215e332242e7a9f56403d814c744624"}, + {file = "numpy-1.19.1-cp36-cp36m-win_amd64.whl", hash = "sha256:082f8d4dd69b6b688f64f509b91d482362124986d98dc7dc5f5e9f9b9c3bb983"}, + {file = "numpy-1.19.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e4f6d3c53911a9d103d8ec9518190e52a8b945bab021745af4939cfc7c0d4a9e"}, + {file = "numpy-1.19.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:5b6885c12784a27e957294b60f97e8b5b4174c7504665333c5e94fbf41ae5d6a"}, + {file = "numpy-1.19.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1bc0145999e8cb8aed9d4e65dd8b139adf1919e521177f198529687dbf613065"}, + {file = "numpy-1.19.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:5a936fd51049541d86ccdeef2833cc89a18e4d3808fe58a8abeb802665c5af93"}, + {file = "numpy-1.19.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:ef71a1d4fd4858596ae80ad1ec76404ad29701f8ca7cdcebc50300178db14dfc"}, + {file = "numpy-1.19.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b9792b0ac0130b277536ab8944e7b754c69560dac0415dd4b2dbd16b902c8954"}, + {file = "numpy-1.19.1-cp37-cp37m-win32.whl", hash = "sha256:b12e639378c741add21fbffd16ba5ad25c0a1a17cf2b6fe4288feeb65144f35b"}, + {file = "numpy-1.19.1-cp37-cp37m-win_amd64.whl", hash = "sha256:8343bf67c72e09cfabfab55ad4a43ce3f6bf6e6ced7acf70f45ded9ebb425055"}, + {file = "numpy-1.19.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e45f8e981a0ab47103181773cc0a54e650b2aef8c7b6cd07405d0fa8d869444a"}, + {file = "numpy-1.19.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:667c07063940e934287993366ad5f56766bc009017b4a0fe91dbd07960d0aba7"}, + {file = "numpy-1.19.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:480fdd4dbda4dd6b638d3863da3be82873bba6d32d1fc12ea1b8486ac7b8d129"}, + {file = "numpy-1.19.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:935c27ae2760c21cd7354402546f6be21d3d0c806fffe967f745d5f2de5005a7"}, + {file = "numpy-1.19.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:309cbcfaa103fc9a33ec16d2d62569d541b79f828c382556ff072442226d1968"}, + {file = "numpy-1.19.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7ed448ff4eaffeb01094959b19cbaf998ecdee9ef9932381420d514e446601cd"}, + {file = "numpy-1.19.1-cp38-cp38-win32.whl", hash = "sha256:de8b4a9b56255797cbddb93281ed92acbc510fb7b15df3f01bd28f46ebc4edae"}, + {file = "numpy-1.19.1-cp38-cp38-win_amd64.whl", hash = "sha256:92feb989b47f83ebef246adabc7ff3b9a59ac30601c3f6819f8913458610bdcc"}, + {file = "numpy-1.19.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:e1b1dc0372f530f26a03578ac75d5e51b3868b9b76cd2facba4c9ee0eb252ab1"}, + {file = "numpy-1.19.1.zip", hash = "sha256:b8456987b637232602ceb4d663cb34106f7eb780e247d51a260b84760fd8f491"}, +] packaging = [ {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, @@ -1453,8 +1504,8 @@ pluggy = [ {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] prompt-toolkit = [ - {file = "prompt_toolkit-3.0.5-py3-none-any.whl", hash = "sha256:df7e9e63aea609b1da3a65641ceaf5bc7d05e0a04de5bd45d05dbeffbabf9e04"}, - {file = "prompt_toolkit-3.0.5.tar.gz", hash = "sha256:563d1a4140b63ff9dd587bda9557cffb2fe73650205ab6f4383092fb882e7dc8"}, + {file = "prompt_toolkit-3.0.7-py3-none-any.whl", hash = "sha256:83074ee28ad4ba6af190593d4d4c607ff525272a504eb159199b6dd9f950c950"}, + {file = "prompt_toolkit-3.0.7.tar.gz", hash = "sha256:822f4605f28f7d2ba6b0b09a31e25e140871e96364d1d377667b547bb3bf4489"}, ] protego = [ {file = "Protego-0.1.16.tar.gz", hash = "sha256:a682771bc7b51b2ff41466460896c1a5a653f9a1e71639ef365a72e66d8734b4"}, @@ -1642,9 +1693,9 @@ typed-ast = [ {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] typing-extensions = [ - {file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"}, - {file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"}, - {file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"}, + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, ] urllib3 = [ {file = "urllib3-1.25.10-py2.py3-none-any.whl", hash = "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"}, diff --git a/website/web/__init__.py b/website/web/__init__.py index 7a8a5556..06c07224 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -191,16 +191,21 @@ def hostnode_popup(tree_uuid: str, node_uuid: str): keys_request = { 'request_cookie': "/static/cookie_read.png", } + if lookyloo.get_config('enable_context_by_users'): + enable_context_by_users = True + else: + enable_context_by_users = False hostnode, urls = lookyloo.get_hostnode_investigator(tree_uuid, node_uuid) return render_template('hostname_popup.html', tree_uuid=tree_uuid, - hostname_uuid=node_uuid, + hostnode_uuid=node_uuid, hostname=hostnode.name, urls=urls, keys_response=keys_response, - keys_request=keys_request) + keys_request=keys_request, + enable_context_by_users=enable_context_by_users) @app.route('/tree//url//request_cookies', methods=['GET']) @@ -492,6 +497,43 @@ def body_hash_details(body_hash: str): return render_template('body_hash.html', body_hash=body_hash, domains=domains, captures=captures) +@app.route('/tree//mark_as_legitimate', methods=['POST']) +def mark_as_legitimate(tree_uuid: str): + if request.data: + legitimate_entries = request.get_json(force=True) + lookyloo.add_to_legitimate(tree_uuid, **legitimate_entries) + else: + lookyloo.add_to_legitimate(tree_uuid) + return jsonify({'message': 'Legitimate entry added.'}) + + +@app.route('/tree//add_context/', methods=['POST']) +@auth.login_required +def add_context(tree_uuid: str, urlnode_uuid: str): + context_data = request.form + ressource_hash: str = context_data.get('hash_to_contextualize') # type: ignore + hostnode_uuid: str = context_data.get('hostnode_uuid') # type: ignore + legitimate: bool = True if context_data.get('legitimate') else False + malicious: bool = True if context_data.get('malicious') else False + details: Dict[str, Dict] = {'malicious': {}, 'legitimate': {}} + if malicious: + malicious_details = {} + if context_data.get('malicious_type'): + malicious_details['type'] = context_data['malicious_type'] + if context_data.get('malicious_target'): + malicious_details['target'] = context_data['malicious_target'] + details['malicious'] = malicious_details + if legitimate: + legitimate_details = {} + if context_data.get('legitimate_domain'): + legitimate_details['domain'] = context_data['legitimate_domain'] + if context_data.get('legitimate_description'): + legitimate_details['description'] = context_data['legitimate_description'] + details['legitimate'] = legitimate_details + lookyloo.add_context(tree_uuid, urlnode_uuid, ressource_hash, legitimate, malicious, details) + return redirect(url_for('hostnode_popup', tree_uuid=tree_uuid, node_uuid=hostnode_uuid)) + + # Query API @app.route('/json//redirects', methods=['GET']) diff --git a/website/web/static/tree.js b/website/web/static/tree.js index adaef832..e432c414 100644 --- a/website/web/static/tree.js +++ b/website/web/static/tree.js @@ -161,6 +161,13 @@ function UnflagAllNodes() { .on('mouseout', () => d3.select('#tooltip').style('opacity', 0)); }; +function MarkAsLegitimate(capture_uuid, hostnode_uuid=null, urlnode_uuid=null) { + let data = {}; + if (hostnode_uuid != null) { data['hostnode_uuid'] = hostnode_uuid; }; + if (urlnode_uuid != null) { data['urlnode_uuid'] = urlnode_uuid; }; + $.post(`/tree/${capture_uuid}/mark_as_legitimate`, data); +}; + function UnflagHostNode(hostnode_uuid) { d3.select(`#node_${hostnode_uuid}`).select('rect').style('fill', 'white'); d3.select(`#node_${hostnode_uuid}`).select('text').style('fill', 'black'); @@ -320,15 +327,13 @@ function update(root, computed_node_width=0) { // Set background based on the computed width and height let background = main_svg.insert('rect', ':first-child') .attr('y', 0) - // FIXME: + 200 doesn't make much sense... - .attr('width', newWidth + margin.right + margin.left + 200) + .attr('width', newWidth + (margin.right + margin.left)*2) .attr('height', newHeight + margin.top + margin.bottom) .style('fill', "url(#backstripes)"); // Update size - d3.select("body svg") - // FIXME: + 200 doesn't make much sense... - .attr("width", newWidth + margin.right + margin.left + 200) + main_svg + .attr("width", newWidth + (margin.right + margin.left)*2) .attr("height", newHeight + margin.top + margin.bottom) // Update pattern @@ -445,14 +450,14 @@ function update(root, computed_node_width=0) { }) .on('mouseout', () => d3.select('#tooltip').style('opacity', 0)); + const http_icon_size = 24; if (d.data.http_content) { - const icon_size = 24; // set lock insecure connection d3.select(this).append("svg").append('rect') .attr('x', selected_node_bbox.width - 22) .attr('y', selected_node_bbox.height - 13) - .attr('width', icon_size) - .attr('height', icon_size) + .attr('width', http_icon_size) + .attr('height', http_icon_size) .attr('fill', 'white') .attr('stroke', 'black'); @@ -460,8 +465,8 @@ function update(root, computed_node_width=0) { .attr('x', selected_node_bbox.width - 22) .attr('y', selected_node_bbox.height - 13) .attr('id', 'insecure_image') - .attr("width", icon_size) - .attr("height", icon_size) + .attr("width", http_icon_size) + .attr("height", http_icon_size) .attr("xlink:href", '/static/insecure.svg') .on('mouseover', () => { d3.select('#tooltip') @@ -472,6 +477,83 @@ function update(root, computed_node_width=0) { }) .on('mouseout', () => d3.select('#tooltip').style('opacity', 0)); }; + const context_icon_size = 24; + if (d.data.malicious) { + // set bomb + d3.select(this).append("svg").append('rect') + .attr('x', selected_node_bbox.width - 22 - http_icon_size) + .attr('y', selected_node_bbox.height - 13) + .attr('width', context_icon_size) + .attr('height', context_icon_size) + .attr('fill', 'white') + .attr('stroke', 'black'); + + d3.select(this).append('image') + .attr('x', selected_node_bbox.width - 22 - http_icon_size) + .attr('y', selected_node_bbox.height - 13) + .attr('id', 'insecure_image') + .attr("width", context_icon_size) + .attr("height", context_icon_size) + .attr("xlink:href", '/static/bomb.svg') + .on('mouseover', () => { + d3.select('#tooltip') + .style('opacity', 1) + .style('left', `${d3.event.pageX + 10}px`) + .style('top', `${d3.event.pageY + 10}px`) + .text('This node containts known malicious content'); + }) + .on('mouseout', () => d3.select('#tooltip').style('opacity', 0)); + } else if (d.data.legitimate) { + // set checkmark + d3.select(this).append("svg").append('rect') + .attr('x', selected_node_bbox.width - 22 - http_icon_size) + .attr('y', selected_node_bbox.height - 13) + .attr('width', context_icon_size) + .attr('height', context_icon_size) + .attr('fill', 'white') + .attr('stroke', 'black'); + + d3.select(this).append('image') + .attr('x', selected_node_bbox.width - 22 - http_icon_size) + .attr('y', selected_node_bbox.height - 13) + .attr('id', 'insecure_image') + .attr("width", context_icon_size) + .attr("height", context_icon_size) + .attr("xlink:href", '/static/check.svg') + .on('mouseover', () => { + d3.select('#tooltip') + .style('opacity', 1) + .style('left', `${d3.event.pageX + 10}px`) + .style('top', `${d3.event.pageY + 10}px`) + .text('This node has only legitimate content'); + }) + .on('mouseout', () => d3.select('#tooltip').style('opacity', 0)); + } else if (d.data.all_empty) { + // set empty + d3.select(this).append("svg").append('rect') + .attr('x', selected_node_bbox.width - 22 - http_icon_size) + .attr('y', selected_node_bbox.height - 13) + .attr('width', context_icon_size) + .attr('height', context_icon_size) + .attr('fill', 'white') + .attr('stroke', 'black'); + + d3.select(this).append('image') + .attr('x', selected_node_bbox.width - 22 - http_icon_size) + .attr('y', selected_node_bbox.height - 13) + .attr('id', 'insecure_image') + .attr("width", context_icon_size) + .attr("height", context_icon_size) + .attr("xlink:href", '/static/empty.svg') + .on('mouseover', () => { + d3.select('#tooltip') + .style('opacity', 1) + .style('left', `${d3.event.pageX + 10}px`) + .style('top', `${d3.event.pageY + 10}px`) + .text('This node has only empty content'); + }) + .on('mouseout', () => d3.select('#tooltip').style('opacity', 0)); + }; }); return node_group; diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html index c3e836ee..7eafcf91 100644 --- a/website/web/templates/hostname_popup.html +++ b/website/web/templates/hostname_popup.html @@ -1,5 +1,6 @@ {% extends "main.html" %} -{% from "macros.html" import sanejs_details %} +{% from "macros.html" import known_content_details %} +{% from "macros.html" import ressource_legitimacy_details %} {% from "macros.html" import indexed_hash %} {% from "macros.html" import indexed_cookies %} {% from "macros.html" import popup_icons %} @@ -61,8 +62,8 @@ {# Headers #}

{{ hostname }}

- - Download URLs as text + + Download URLs as text
{# Start list of URLs #}
    @@ -126,15 +127,16 @@

    {{ popup_icons(keys_response, url['url_object'], tree_uuid) }} +
    {% if url['url_object'].empty_response %} Empty body. {% else %} - Body size: {{ sizeof_fmt(url['url_object'].body.getbuffer().nbytes) }} + {{ ressource_legitimacy_details(url['legitimacy'], url['url_object'].body.getbuffer().nbytes) }} {%endif%} +
    - {% if url['sane_js'] %} - {# Result from SaneJS for the response #} - {{ sanejs_details(url['sane_js']) }} + {% if url['known_content'] %} + {{ known_content_details(url['known_content']) }} {% endif %} {# Everything we know about the response content #} @@ -156,6 +158,49 @@

    {% endif %} + {% if enable_context_by_users%} + +
    +
    +
    +
    +
    + + +
    +
    +
    + + +
    +
    + + +
    +
    +
    + + +
    +
    +
    + + +
    +
    + + +
    + + + +
    +
    +
    + {% endif %} {% if url['embedded_ressources'] %} {# Details on embedded resources #} @@ -168,10 +213,10 @@
    {% for hash, details in url['embedded_ressources'].items() %} - {% if details['sane_js'] %} - {# Result from SaneJS for the embedded ressources #} - {{ sanejs_details(details['sane_js']) }} + {% if details['known_content'] %} + {{ known_content_details(details['known_content']) }} {% endif %} + {{ ressource_legitimacy_details(details['legitimacy'], details['body_size']) }}
    This file ({{ details['type'] }}) can be found {{ details['hash_freq'] }} times across all the captures on this lookyloo instance, in {{ details['hash_domains_freq'] }} unique domains. diff --git a/website/web/templates/macros.html b/website/web/templates/macros.html index 3e1549cd..44c03e53 100644 --- a/website/web/templates/macros.html +++ b/website/web/templates/macros.html @@ -1,4 +1,4 @@ -{% macro sanejs_details(details) %} +{% macro known_content_details(details) %}
    {% if details is string %} {{ details }} @@ -12,6 +12,26 @@
    {% endmacro %} +{% macro ressource_legitimacy_details(details, ressource_size) %} +{% if details and details[0] == False %} + +{%endif%} +Body size: {{ sizeof_fmt(ressource_size) }} +{% if details %} + {% if details[0] %} + - This file is known legitimate. + {% elif details[0] == False %} + {% if details[1] is iterable %} +
    + The response sould be considered as phishing unless it is served by the following domain(s): {{ ', '.join(details[1]) }} +
    + {% else %} + - The response is known malicious. + {%endif%} + {%endif%} +{%endif%} +{% endmacro %} + {% macro indexed_hash(details, identifier_for_toggle) %} {% set total_captures = details['different_url']|length + details['same_url']|length %} {# Only show details if the hits are in an other capture #} diff --git a/website/web/templates/scrape.html b/website/web/templates/scrape.html index 52869dd6..ad93cef3 100644 --- a/website/web/templates/scrape.html +++ b/website/web/templates/scrape.html @@ -8,7 +8,7 @@ alt="Lookyloo" width="400">
    -
    +
    diff --git a/website/web/templates/tree.html b/website/web/templates/tree.html index 018c4501..b2b6829d 100644 --- a/website/web/templates/tree.html +++ b/website/web/templates/tree.html @@ -93,7 +93,7 @@ {% endif %}
  • - Unflag all nodes + Unflag all nodes
  • Download screenshot @@ -101,6 +101,9 @@
  • Show screenshot
  • +
  • + Mark capture as legitimate +
  • {% if enable_mail_notification %}
  • Notify by mail