diff --git a/config/generic.json.sample b/config/generic.json.sample
index 507fa011..c6fa359a 100644
--- a/config/generic.json.sample
+++ b/config/generic.json.sample
@@ -14,6 +14,7 @@
"max_depth": 1,
"use_user_agents_users": false,
"enable_default_blur_screenshot": false,
+ "enable_context_by_users": false,
"enable_mail_notification": false,
"email": {
"from": "Lookyloo ",
@@ -35,6 +36,7 @@
"max_depth": "Maximum depth for scraping. Anything > 1 will be exponentially bigger.",
"use_user_agents_users": "Only usable for medium/high use instances: use the user agents of the users of the platform",
"enable_default_blur_screenshot": "If true, blur the screenshot by default (useful on public instances)",
+ "enable_context_by_users": "Allow the users to add context to a response body",
"enable_mail_notification": "Enable email notification or not",
"email": "Configuration for sending email notifications."
}
diff --git a/known_content/generic.json b/known_content/generic.json
new file mode 100644
index 00000000..6dd96b58
--- /dev/null
+++ b/known_content/generic.json
@@ -0,0 +1,66 @@
+{
+ "1px_gif": {
+ "description": "1 pixel GIF",
+ "entries": [
+ "717ea0ff7f3f624c268eccb244e24ec1305ab21557abb3d6f1a7e183ff68a2d28f13d1d2af926c9ef6d1fb16dd8cbe34cd98cacf79091dddc7874dcee21ecfdc",
+ "e508d5d17e94d14b126164082342a9ca4774f404e87a3dd56c26812493ee18d9c3d6daacca979134a94a003066aca24116de874596d00d1e52130c1283d54209",
+ "2d073e10ae40fde434eb31cbedd581a35cd763e51fb7048b88caa5f949b1e6105e37a228c235bc8976e8db58ed22149cfccf83b40ce93a28390566a28975744a",
+ "84e24a70b78e9de9c9d0dfeb49f3f4247dbc1c715d8844471ee40669270682e199d48f5fbec62bd984c9c0270534b407c4d2561dd6c05adec3c83c1534f32d5c",
+ "d5da26b5d496edb0221df1a4057a8b0285d15592a8f8dc7016a294df37ed335f3fde6a2252962e0df38b62847f8b771463a0124ef3f84299f262ed9d9d3cee4c",
+ "f7a5f748f4c0d3096a3ca972886fe9a9dff5dce7792779ec6ffc42fa880b3815e2e4c3bdea452352f3844b81864c9bfb7861f66ac961cfa66cb9cb4febe568e8",
+ "b2ca25a3311dc42942e046eb1a27038b71d689925b7d6b3ebb4d7cd2c7b9a0c7de3d10175790ac060dc3f8acf3c1708c336626be06879097f4d0ecaa7f567041",
+ "b8d82d64ec656c63570b82215564929adad167e61643fd72283b94f3e448ef8ab0ad42202f3537a0da89960bbdc69498608fc6ec89502c6c338b6226c8bf5e14",
+ "2991c3aa1ba61a62c1cccd990c0679a1fb8dccd547d153ec0920b91a75ba20820de1d1c206f66d083bf2585d35050f0a39cd7a3e11c03882dafec907d27a0180",
+ "b1a6cfa7b21dbb0b281d241af609f3ba7f3a63e5668095bba912bf7cfd7f0320baf7c3b0bfabd0f8609448f39902baeb145ba7a2d8177fe22a6fcea03dd29be1",
+ "ebfe0c0df4bcc167d5cb6ebdd379f9083df62bef63a23818e1c6adf0f64b65467ea58b7cd4d03cf0a1b1a2b07fb7b969bf35f25f1f8538cc65cf3eebdf8a0910",
+ "1d68b92e8d822fe82dc7563edd7b37f3418a02a89f1a9f0454cca664c2fc2565235e0d85540ff9be0b20175be3f5b7b4eae1175067465d5cca13486aab4c582c",
+ "ac44da7f455bfae52b883639964276026fb259320902aa813d0333e021c356a7b3e3537b297f9a2158e588c302987ce0854866c039d1bb0ffb27f67560739db2",
+ "921944dc10fbfb6224d69f0b3ac050f4790310fd1bcac3b87c96512ad5ed9a268824f3f5180563d372642071b4704c979d209baf40bc0b1c9a714769aba7dfc7",
+ "89dfc38ec77cf258362e4db7c8203cae8a02c0fe4f99265b0539ec4f810c84f8451e22c9bef1ebc59b4089af7e93e378e053c542a5967ec4912d4c1fc5de22f0",
+ "280ea4383ee6b37051d91c5af30a5ce72aa4439340fc6d31a4fbe7ba8a8156eb7893891d5b2371b9fc4934a78f08de3d57e5b63fa9d279a317dcbefb8a07a6b0",
+ "3844065e1dd778a05e8cc39901fbf3191ded380d594359df137901ec56ca52e03d57eb60acc2421a0ee74f0733bbb5d781b7744685c26fb013a236f49b02fed3",
+ "bd9ab35dde3a5242b04c159187732e13b0a6da50ddcff7015dfb78cdd68743e191eaf5cddedd49bef7d2d5a642c217272a40e5ba603fe24ca676a53f8c417c5d",
+ "d052ecec2839340876eb57247cfc2e777dd7f2e868dc37cd3f3f740c8deb94917a0c9f2a4fc8229987a0b91b04726de2d1e9f6bcbe3f9bef0e4b7e0d7f65ea12",
+ "8717074ddf1198d27b9918132a550cb4ba343794cc3d304a793f9d78c9ff6c4929927b414141d40b6f6ad296725520f4c63edeb660ed530267766c2ab74ee4a9",
+ "6834f1548f26b94357fcc3312a3491e8c87080a84f678f990beb2c745899a01e239964521e64a534d7d5554222f728af966ec6ec8291bc64d2005861bcfd78ec",
+ "3be8176915593e79bc280d08984a16c29c495bc53be9b439276094b8dcd3764a3c72a046106a06b958e08e67451fe02743175c621a1faa261fe7a9691cc77141",
+ "826225fc21717d8861a05b9d2f959539aad2d2b131b2afed75d88fbca535e1b0d5a0da8ac69713a0876a0d467848a37a0a7f926aeafad8cf28201382d16466ab",
+ "202612457d9042fe853daab3ddcc1f0f960c5ffdbe8462fa435713e4d1d85ff0c3f197daf8dba15bda9f5266d7e1f9ecaeee045cbc156a4892d2f931fe6fa1bb",
+ "b82c6aa1ae927ade5fadbbab478cfaef26d21c1ac441f48e69cfc04cdb779b1e46d7668b4368b933213276068e52f9060228907720492a70fd9bc897191ee77c",
+ "763de1053a56a94eef4f72044adb2aa370b98ffa6e0add0b1cead7ee27da519e223921c681ae1db3311273f45d0dd3dc022d102d42ce210c90cb3e761b178438",
+ "69e2da5cdc318fc237eaa243b6ea7ecc83b68dbdea8478dc69154abdda86ecb4e16c35891cc1facb3ce7e0cf19d5abf189c50f59c769777706f4558f6442abbc",
+ "16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516",
+ "01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb",
+ "71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746",
+ "49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a",
+ "c57ebbadcf59f982ba28da35fdbd5e5369a8500a2e1edad0dc9c9174de6fd99f437953732e545b95d3de5943c61077b6b949c989f49553ff2e483f68fcc30641",
+ "c87bf81fd70cf6434ca3a6c05ad6e9bd3f1d96f77dddad8d45ee043b126b2cb07a5cf23b4137b9d8462cd8a9adf2b463ab6de2b38c93db72d2d511ca60e3b57e",
+ "fd8b021f0236e487bfee13bf8f0ae98760abc492f7ca3023e292631979e135cb4ccb0c89b6234971b060ad72c0ca4474cbb5092c6c7a3255d81a54a36277b486",
+ "235479f42cbbe0a4b0100167fece0d14c9b47d272b3ba8322bcfe8539f055bf31d500e7b2995cc968ebf73034e039f59c5f0f9410428663034bf119d74b5672c",
+ "a85e09c3b5dbb560f4e03ba880047dbc8b4999a64c1f54fbfbca17ee0bcbed3bc6708d699190b56668e464a59358d6b534c3963a1329ba01db21075ef5bedace",
+ "27656d6106a6da0c84174ba7a6307e6f1c4b3f2cc085c8466b6a25d54331035dabc7081aac208d960d8d37c5577547628c0d1c4b77bb4cf254c71859673feec1",
+ "41edf618eb0ba5158411c5ac3e900904bbf36cbb4be1347dc5281f4722244ad0b9880f0cf4fbec70089b0b7ba3b8aae6f92be7379e72db325c2802250b5e529e"
+ ]
+ },
+ "1px_png": {
+ "description": "1 pixel PNG",
+ "entries": [
+ "f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498",
+ "dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99",
+ "c2c239cb5cdd0b670780ad6414ef6be9ccd4c21ce46bb93d1fa3120ac812f1679445162978c3df05cb2e1582a1844cc4c41cf74960b8fdae3123999c5d2176cc",
+ "6ad523f5b65487369d305613366b9f68dcdeee225291766e3b25faf45439ca069f614030c08ca54c714fdbf7a944fac489b1515a8bf9e0d3191e1bcbbfe6a9df"
+ ]
+ },
+ "empty_file": {
+ "description": "empty file",
+ "entries": [
+ "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"
+ ]
+ },
+ "single_space": {
+ "description": "Empty file with a single space",
+ "entries": [
+ "f90ddd77e400dfe6a3fcf479b00b1ee29e7015c5bb8cd70f5f15b4886cc339275ff553fc8a053f8ddc7324f45168cffaf81f8c3ac93996f6536eef38e5e40768"
+ ]
+ }
+}
diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py
index 34f5b336..247455ec 100644
--- a/lookyloo/helpers.py
+++ b/lookyloo/helpers.py
@@ -183,6 +183,14 @@ def get_user_agents(directory: str='user_agents') -> Dict[str, Any]:
return json.load(f)
+def load_known_content(directory: str='known_content') -> Dict[str, Dict[str, Any]]:
+ to_return: Dict[str, Dict[str, Any]] = {}
+ for known_content_file in (get_homedir() / directory).glob('*.json'):
+ with known_content_file.open() as f:
+ to_return[known_content_file.stem] = json.load(f)
+ return to_return
+
+
def load_cookies(cookie_pseudofile: Optional[BufferedIOBase]=None) -> List[Dict[str, str]]:
if cookie_pseudofile:
cookies = json.load(cookie_pseudofile)
diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py
index 9aa7e4b8..bb3dd410 100644
--- a/lookyloo/lookyloo.py
+++ b/lookyloo/lookyloo.py
@@ -15,7 +15,7 @@ from pathlib import Path
import pickle
import smtplib
import socket
-from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set, Iterable
+from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set, Iterable, Iterator
from urllib.parse import urlsplit
from uuid import uuid4
from zipfile import ZipFile
@@ -29,10 +29,15 @@ from scrapysplashwrapper import crawl
from werkzeug.useragents import UserAgent
from .exceptions import NoValidHarFile, MissingUUID
-from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template, load_pickle_tree, remove_pickle_tree
+from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template, load_pickle_tree, remove_pickle_tree, load_known_content
from .modules import VirusTotal, SaneJavaScript, PhishingInitiative
+def dump_to_json(obj: Union[Set]) -> Union[List]:
+ if isinstance(obj, set):
+ return list(obj)
+
+
class Indexing():
def __init__(self) -> None:
@@ -114,21 +119,22 @@ class Indexing():
pipeline = self.redis.pipeline()
for urlnode in crawled_tree.root_hartree.url_tree.traverse():
- if not urlnode.empty_response:
- pipeline.zincrby('body_hashes', 1, urlnode.body_hash)
- pipeline.zincrby(f'bh|{urlnode.body_hash}', 1, urlnode.hostname)
- # set of all captures with this hash
- pipeline.sadd(f'bh|{urlnode.body_hash}|captures', crawled_tree.uuid)
- # ZSet of all urlnode_UUIDs|full_url
- pipeline.zincrby(f'bh|{urlnode.body_hash}|captures|{crawled_tree.uuid}', 1, f'{urlnode.uuid}|{urlnode.hostnode_uuid}|{urlnode.name}')
- if hasattr(urlnode, 'embedded_ressources') and urlnode.embedded_ressources:
- for mimetype, blobs in urlnode.embedded_ressources.items():
- for h, body in blobs:
- pipeline.zincrby('body_hashes', 1, h)
- pipeline.zincrby(f'bh|{h}', 1, urlnode.hostname)
- pipeline.sadd(f'bh|{h}|captures', crawled_tree.uuid)
- pipeline.zincrby(f'bh|{h}|captures|{crawled_tree.uuid}', 1,
- f'{urlnode.uuid}|{urlnode.hostnode_uuid}|{urlnode.name}')
+ if urlnode.empty_response:
+ continue
+ pipeline.zincrby('body_hashes', 1, urlnode.body_hash)
+ pipeline.zincrby(f'bh|{urlnode.body_hash}', 1, urlnode.hostname)
+ # set of all captures with this hash
+ pipeline.sadd(f'bh|{urlnode.body_hash}|captures', crawled_tree.uuid)
+ # ZSet of all urlnode_UUIDs|full_url
+ pipeline.zincrby(f'bh|{urlnode.body_hash}|captures|{crawled_tree.uuid}', 1, f'{urlnode.uuid}|{urlnode.hostnode_uuid}|{urlnode.name}')
+ if hasattr(urlnode, 'embedded_ressources') and urlnode.embedded_ressources:
+ for mimetype, blobs in urlnode.embedded_ressources.items():
+ for h, body in blobs:
+ pipeline.zincrby('body_hashes', 1, h)
+ pipeline.zincrby(f'bh|{h}', 1, urlnode.hostname)
+ pipeline.sadd(f'bh|{h}|captures', crawled_tree.uuid)
+ pipeline.zincrby(f'bh|{h}|captures|{crawled_tree.uuid}', 1,
+ f'{urlnode.uuid}|{urlnode.hostnode_uuid}|{urlnode.name}')
pipeline.execute()
@@ -147,6 +153,289 @@ class Indexing():
return self.redis.zrevrange(f'bh|{body_hash}', 0, -1, withscores=True)
+class Context():
+
+ def __init__(self, sanejs: Optional[SaneJavaScript] = None):
+ self.redis: Redis = Redis(unix_socket_path=get_socket_path('indexing'), db=1, decode_responses=True)
+ self.sanejs = sanejs
+ self._cache_known_content()
+
+ def clear_context(self):
+ self.redis.flushdb()
+
+ def _get_resources_hashes(self, har2tree_container: Union[CrawledTree, HostNode, URLNode]) -> Set[str]:
+ if isinstance(har2tree_container, CrawledTree):
+ urlnodes = har2tree_container.root_hartree.url_tree.traverse()
+ elif isinstance(har2tree_container, HostNode):
+ urlnodes = har2tree_container.urls
+ elif isinstance(har2tree_container, URLNode):
+ urlnodes = [har2tree_container]
+ else:
+ raise Exception(f'har2tree_container cannot be {type(har2tree_container)}')
+ all_ressources_hashes: Set[str] = set()
+ for urlnode in urlnodes:
+ if hasattr(urlnode, 'resources_hashes'):
+ all_ressources_hashes.update(urlnode.resources_hashes)
+ return all_ressources_hashes
+
+ def _cache_known_content(self) -> None:
+ p = self.redis.pipeline()
+ for filename, file_content in load_known_content().items():
+ if filename == 'generic':
+ for k, type_content in file_content.items():
+ p.hmset('known_content', {h: type_content['description'] for h in type_content['entries']})
+ elif filename == 'malicious':
+ for h, details in file_content.items():
+ p.sadd('bh|malicious', h)
+ elif filename == 'legitimate':
+ for h, details in file_content.items():
+ if 'domain' in details and details['domain']:
+ p.sadd(f'bh|{h}|legitimate', *details['domain'])
+ elif 'description' in details:
+ p.hset('known_content', h, details['description'])
+ else:
+ for h, details in file_content.items():
+ p.sadd(f'bh|{h}|legitimate', *details['hostnames'])
+ p.execute()
+
+ def find_known_content(self, har2tree_container: Union[CrawledTree, HostNode, URLNode]) -> Dict[str, Union[str, List[str]]]:
+ """Return a dictionary of content resources found in the local known_content database, or in SaneJS (if enabled)"""
+ all_ressources_hashes = self._get_resources_hashes(har2tree_container)
+ # Get from local cache of known content all descriptions related to the ressources.
+ if not all_ressources_hashes:
+ return {}
+ known_content_table = dict(zip(all_ressources_hashes,
+ self.redis.hmget('known_content', all_ressources_hashes)))
+
+ if self.sanejs and self.sanejs.available:
+ # Query sanejs on the remaining ones
+ to_lookup = [h for h, description in known_content_table.items() if not description]
+ for h, entry in self.sanejs.hashes_lookup(to_lookup).items():
+ libname, version, path = entry[0].split("|")
+ known_content_table[h] = (libname, version, path, len(entry))
+ return {h: details for h, details in known_content_table.items() if details}
+
+ def _filter(self, urlnodes: Union[URLNode, List[URLNode]], known_hashes: Iterable[str]) -> Iterator[Tuple[URLNode, str]]:
+ if isinstance(urlnodes, URLNode):
+ _urlnodes = [urlnodes]
+ else:
+ _urlnodes = urlnodes
+ for urlnode in _urlnodes:
+ for h in urlnode.resources_hashes:
+ if h not in known_hashes:
+ yield urlnode, h
+
+ def store_known_legitimate_tree(self, tree: CrawledTree):
+ known_content = self.find_known_content(tree)
+ urlnodes = tree.root_hartree.url_tree.traverse()
+ root_hostname = urlsplit(tree.root_url).hostname
+ known_content_file: Path = get_homedir() / 'known_content' / f'{root_hostname}.json'
+ if known_content_file.exists():
+ with open(known_content_file) as f:
+ to_store = json.load(f)
+ else:
+ to_store = {}
+ for urlnode, h in self._filter(urlnodes, known_content):
+ mimetype = ''
+ if h != urlnode.body_hash:
+ # this is the hash of an embeded content so it won't have a filename but has a different mimetype
+ # FIXME: this is ugly.
+ for ressource_mimetype, blobs in urlnode.embedded_ressources.items():
+ for ressource_h, b in blobs:
+ if ressource_h == h:
+ mimetype = ressource_mimetype.split(';')[0]
+ break
+ if mimetype:
+ break
+ else:
+ if urlnode.mimetype:
+ mimetype = urlnode.mimetype.split(';')[0]
+ if h not in to_store:
+ to_store[h] = {'filenames': set(), 'description': '', 'hostnames': set(), 'mimetype': mimetype}
+ else:
+ to_store[h]['filenames'] = set(to_store[h]['filenames'])
+ to_store[h]['hostnames'] = set(to_store[h]['hostnames'])
+
+ to_store[h]['hostnames'].add(urlnode.hostname)
+ if urlnode.url_split.path:
+ filename = Path(urlnode.url_split.path).name
+ if filename:
+ to_store[h]['filenames'].add(filename)
+
+ with open(known_content_file, 'w') as f:
+ json.dump(to_store, f, indent=2, default=dump_to_json)
+
+ def mark_as_legitimate(self, tree: CrawledTree, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> None:
+ if hostnode_uuid:
+ urlnodes = tree.root_hartree.get_host_node_by_uuid(hostnode_uuid).urls
+ elif urlnode_uuid:
+ urlnodes = [tree.root_hartree.get_url_node_by_uuid(urlnode_uuid)]
+ else:
+ urlnodes = tree.root_hartree.url_tree.traverse()
+ self.store_known_legitimate_tree(tree)
+ known_content = self.find_known_content(tree)
+ pipeline = self.redis.pipeline()
+ for urlnode, h in self._filter(urlnodes, known_content):
+ # Note: we can have multiple hahes on the same urlnode (see embedded resources).
+ # They are expected to be on the same domain as urlnode. This code work as expected.
+ pipeline.sadd(f'bh|{h}|legitimate', urlnode.hostname)
+ pipeline.execute()
+
+ def contextualize_tree(self, tree: CrawledTree) -> CrawledTree:
+ hostnodes_with_malicious_content = set()
+ known_content = self.find_known_content(tree)
+ for urlnode in tree.root_hartree.url_tree.traverse():
+ if urlnode.empty_response:
+ continue
+
+ malicious = self.is_malicious(urlnode, known_content)
+ if malicious is True:
+ urlnode.add_feature('malicious', malicious)
+ hostnodes_with_malicious_content.add(urlnode.hostnode_uuid)
+ elif malicious is False:
+ # Marked as legitimate
+ urlnode.add_feature('legitimate', True)
+ elif not urlnode.empty_response and urlnode.body_hash in known_content:
+ urlnode.add_feature('legitimate', True)
+
+ for hostnode_with_malicious_content in hostnodes_with_malicious_content:
+ hostnode = tree.root_hartree.get_host_node_by_uuid(hostnode_with_malicious_content)
+ hostnode.add_feature('malicious', malicious)
+
+ for hostnode in tree.root_hartree.hostname_tree.traverse():
+ if 'malicious' not in hostnode.features:
+ if all(urlnode.empty_response for urlnode in hostnode.urls):
+ hostnode.add_feature('all_empty', True)
+ continue
+
+ legit = [urlnode.legitimate for urlnode in hostnode.urls if hasattr(urlnode, 'legitimate')]
+ if len(legit) == len(hostnode.urls) and all(legit):
+ hostnode.add_feature('legitimate', True)
+
+ return tree
+
+ def legitimate_body(self, body_hash: str, legitimate_hostname: str) -> None:
+ self.redis.sadd(f'bh|{body_hash}|legitimate', legitimate_hostname)
+
+ def store_known_malicious_ressource(self, ressource_hash: str, details: Dict[str, str]):
+ known_malicious_ressource_file = get_homedir() / 'known_content' / 'malicious.json'
+ if known_malicious_ressource_file.exists():
+ with open(known_malicious_ressource_file) as f:
+ to_store = json.load(f)
+ else:
+ to_store = {}
+
+ if ressource_hash not in to_store:
+ to_store[ressource_hash] = {'target': set(), 'tag': set()}
+ else:
+ to_store[ressource_hash]['target'] = set(to_store[ressource_hash]['target'])
+ to_store[ressource_hash]['tag'] = set(to_store[ressource_hash]['tag'])
+
+ if 'target' in details:
+ to_store[ressource_hash]['target'].add(details['target'])
+ if 'type' in details:
+ to_store[ressource_hash]['tag'].add(details['type'])
+
+ with open(known_malicious_ressource_file, 'w') as f:
+ json.dump(to_store, f, indent=2, default=dump_to_json)
+
+ def add_malicious(self, ressource_hash: str, details: Dict[str, str]):
+ self.store_known_malicious_ressource(ressource_hash, details)
+ p = self.redis.pipeline()
+ p.sadd('bh|malicious', ressource_hash)
+ if 'target' in details:
+ p.sadd(f'{ressource_hash}|target', details['target'])
+ if 'type' in details:
+ p.sadd(f'{ressource_hash}|tag', details['type'])
+ p.execute()
+
+ def store_known_legitimate_ressource(self, ressource_hash: str, details: Dict[str, str]):
+ known_legitimate_ressource_file = get_homedir() / 'known_content' / 'legitimate.json'
+ if known_legitimate_ressource_file.exists():
+ with open(known_legitimate_ressource_file) as f:
+ to_store = json.load(f)
+ else:
+ to_store = {}
+
+ if ressource_hash not in to_store:
+ to_store[ressource_hash] = {'domain': set(), 'description': ''}
+ else:
+ to_store[ressource_hash]['domain'] = set(to_store[ressource_hash]['domain'])
+
+ if 'domain' in details:
+ to_store[ressource_hash]['domain'].add(details['domain'])
+ if 'description' in details:
+ to_store[ressource_hash]['description'] = details['description']
+
+ with open(known_legitimate_ressource_file, 'w') as f:
+ json.dump(to_store, f, indent=2, default=dump_to_json)
+
+ def add_legitimate(self, ressource_hash: str, details: Dict[str, str]):
+ self.store_known_legitimate_ressource(ressource_hash, details)
+ if 'domain' in details:
+ self.redis.sadd(f'bh|{ressource_hash}|legitimate', details['domain'])
+ elif 'description' in details:
+ # Library
+ self.redis.hset('known_content', ressource_hash, details['description'])
+
+ # Query DB
+
+ def is_legitimate(self, urlnode: URLNode, known_hashes: Iterable[str]) -> Optional[bool]:
+ """3 cases:
+ * True if *all* the contents are known legitimate
+ * False if *any* content is malicious
+ * None in all other cases
+ """
+ status: List[Optional[bool]] = []
+ for urlnode, h in self._filter(urlnode, known_hashes):
+ # Note: we can have multiple hahes on the same urlnode (see embedded resources).
+ # They are expected to be on the same domain as urlnode. This code work as expected.
+ if self.redis.sismember('bh|malicious', h):
+ # Malicious, no need to go any further
+ return False
+ hostnames = self.redis.smembers(f'bh|{h}|legitimate')
+ if hostnames:
+ if urlnode.hostname in hostnames:
+ status.append(True) # legitimate
+ continue
+ else:
+ return False # Malicious
+ else:
+ # NOTE: we do not return here, because we want to return False if *any* of the contents is malicious
+ status.append(None) # Unknown
+ if status and all(status):
+ return True # All the contents are known legitimate
+ return None
+
+ def is_malicious(self, urlnode: URLNode, known_hashes: Iterable[str]) -> Optional[bool]:
+ """3 cases:
+ * True if *any* content is malicious
+ * False if *all* the contents are known legitimate
+ * None in all other cases
+ """
+ legitimate = self.is_legitimate(urlnode, known_hashes)
+ if legitimate:
+ return False
+ elif legitimate is False:
+ return True
+ return None
+
+ def legitimacy_details(self, urlnode: URLNode, known_hashes: Iterable[str]) -> Dict[str, Tuple[bool, Optional[List[str]]]]:
+ to_return = {}
+ for urlnode, h in self._filter(urlnode, known_hashes):
+ # Note: we can have multiple hahes on the same urlnode (see embedded resources).
+ # They are expected to be on the same domain as urlnode. This code work as expected.
+ hostnames = self.redis.smembers(f'bh|{h}|legitimate')
+ if hostnames:
+ if urlnode.hostname in hostnames:
+ to_return[h] = (True, hostnames)
+ else:
+ to_return[h] = (False, hostnames)
+ elif self.redis.sismember('bh|malicious', urlnode.body_hash):
+ to_return[h] = (False, None)
+ return to_return
+
+
class Lookyloo():
def __init__(self) -> None:
@@ -184,6 +473,11 @@ class Lookyloo():
if not self.sanejs.available:
self.logger.warning('Unable to setup the SaneJS module')
+ if hasattr(self, 'sanejs') and self.sanejs.available:
+ self.context = Context(self.sanejs)
+ else:
+ self.context = Context()
+
if not self.redis.exists('cache_loaded'):
self._init_existing_dumps()
@@ -259,6 +553,16 @@ class Lookyloo():
return ct
+ def add_context(self, capture_uuid: str, urlnode_uuid: str, ressource_hash: str, legitimate: bool, malicious: bool, details: Dict[str, Dict[str, str]]):
+ if malicious:
+ self.context.add_malicious(ressource_hash, details['malicious'])
+ if legitimate:
+ self.context.add_legitimate(ressource_hash, details['legitimate'])
+
+ def add_to_legitimate(self, capture_uuid: str, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None):
+ ct = self.get_crawled_tree(capture_uuid)
+ self.context.mark_as_legitimate(ct, hostnode_uuid, urlnode_uuid)
+
def load_tree(self, capture_uuid: str) -> Tuple[str, str, str, str, Dict[str, str]]:
capture_dir = self.lookup_capture_dir(capture_uuid)
if not capture_dir:
@@ -268,6 +572,7 @@ class Lookyloo():
with open((capture_dir / 'meta'), 'r') as f:
meta = json.load(f)
ct = self.get_crawled_tree(capture_uuid)
+ ct = self.context.contextualize_tree(ct)
return ct.to_json(), ct.start_time.isoformat(), ct.user_agent, ct.root_url, meta
def remove_pickle(self, capture_uuid: str) -> None:
@@ -739,17 +1044,6 @@ class Lookyloo():
captures_list['different_url'].append((h_capture_uuid, url_uuid, cache['title'], cache['timestamp'], url_hostname))
return captures_list
- def _format_sane_js_response(self, lookup_table: Dict, h: str) -> Optional[Union[str, Tuple]]:
- if lookup_table.get(h):
- if isinstance(lookup_table[h], list):
- libname, version, path = lookup_table[h][0].split("|")
- other_files = len(lookup_table[h])
- return libname, version, path, other_files
- else:
- # Predefined generic file
- return lookup_table[h]
- return None
-
def get_hostnode_investigator(self, capture_uuid: str, node_uuid: str) -> Tuple[HostNode, List[Dict[str, Any]]]:
capture_dir = self.lookup_capture_dir(capture_uuid)
if not capture_dir:
@@ -762,10 +1056,7 @@ class Lookyloo():
if not hostnode:
raise MissingUUID(f'Unable to find UUID {node_uuid} in {capture_dir}')
- sanejs_lookups: Dict[str, List[str]] = {}
- if hasattr(self, 'sanejs') and self.sanejs.available:
- to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')]
- sanejs_lookups = self.sanejs.hashes_lookup(to_lookup)
+ known_content = self.context.find_known_content(hostnode)
urls: List[Dict[str, Any]] = []
for url in hostnode.urls:
@@ -773,10 +1064,11 @@ class Lookyloo():
# * https vs http
# * everything after the domain
# * the full URL
+ legit_details = self.context.legitimacy_details(url, known_content)
to_append: Dict[str, Any] = {
'encrypted': url.name.startswith('https'),
'url_path': url.name.split('/', 3)[-1],
- 'url_object': url
+ 'url_object': url,
}
if not url.empty_response:
@@ -797,21 +1089,16 @@ class Lookyloo():
continue
freq_embedded = self.indexing.body_hash_fequency(h)
to_append['embedded_ressources'][h] = freq_embedded
+ to_append['embedded_ressources'][h]['body_size'] = blob.getbuffer().nbytes
to_append['embedded_ressources'][h]['type'] = mimetype
if freq_embedded['hash_freq'] > 1:
to_append['embedded_ressources'][h]['other_captures'] = self.hash_lookup(h, url.name, capture_uuid)
- if hasattr(self, 'sanejs') and self.sanejs.available:
- to_lookup = list(to_append['embedded_ressources'].keys())
- sanejs_lookups_embedded = self.sanejs.hashes_lookup(to_lookup)
- for h in to_append['embedded_ressources'].keys():
- sane_js_match = self._format_sane_js_response(sanejs_lookups_embedded, h)
- if sane_js_match:
- to_append['embedded_ressources'][h]['sane_js'] = sane_js_match
+ for h in to_append['embedded_ressources'].keys():
+ to_append['embedded_ressources'][h]['known_content'] = known_content.get(h)
+ to_append['embedded_ressources'][h]['legitimacy'] = legit_details.get(h)
- # Optional: SaneJS information
- sane_js_match = self._format_sane_js_response(sanejs_lookups, url.body_hash)
- if sane_js_match:
- to_append['sane_js'] = sane_js_match
+ to_append['known_content'] = known_content.get(url.body_hash)
+ to_append['legitimacy'] = legit_details.get(url.body_hash)
# Optional: Cookies sent to server in request -> map to nodes who set the cookie in response
if hasattr(url, 'cookies_sent'):
diff --git a/lookyloo/modules.py b/lookyloo/modules.py
index a14ee57e..c70c2398 100644
--- a/lookyloo/modules.py
+++ b/lookyloo/modules.py
@@ -19,53 +19,6 @@ from pyeupi import PyEUPI
class SaneJavaScript():
- skip_lookup: Dict[str, str] = {
- "717ea0ff7f3f624c268eccb244e24ec1305ab21557abb3d6f1a7e183ff68a2d28f13d1d2af926c9ef6d1fb16dd8cbe34cd98cacf79091dddc7874dcee21ecfdc": "This is a 1*1 pixel GIF",
- "e508d5d17e94d14b126164082342a9ca4774f404e87a3dd56c26812493ee18d9c3d6daacca979134a94a003066aca24116de874596d00d1e52130c1283d54209": "This is a 1*1 pixel GIF",
- "2d073e10ae40fde434eb31cbedd581a35cd763e51fb7048b88caa5f949b1e6105e37a228c235bc8976e8db58ed22149cfccf83b40ce93a28390566a28975744a": "This is a 1*1 pixel GIF",
- "84e24a70b78e9de9c9d0dfeb49f3f4247dbc1c715d8844471ee40669270682e199d48f5fbec62bd984c9c0270534b407c4d2561dd6c05adec3c83c1534f32d5c": "This is a 1*1 pixel GIF",
- "d5da26b5d496edb0221df1a4057a8b0285d15592a8f8dc7016a294df37ed335f3fde6a2252962e0df38b62847f8b771463a0124ef3f84299f262ed9d9d3cee4c": "This is a 1*1 pixel GIF",
- "f7a5f748f4c0d3096a3ca972886fe9a9dff5dce7792779ec6ffc42fa880b3815e2e4c3bdea452352f3844b81864c9bfb7861f66ac961cfa66cb9cb4febe568e8": "This is a 1*1 pixel GIF",
- "b2ca25a3311dc42942e046eb1a27038b71d689925b7d6b3ebb4d7cd2c7b9a0c7de3d10175790ac060dc3f8acf3c1708c336626be06879097f4d0ecaa7f567041": "This is a 1*1 pixel GIF",
- "b8d82d64ec656c63570b82215564929adad167e61643fd72283b94f3e448ef8ab0ad42202f3537a0da89960bbdc69498608fc6ec89502c6c338b6226c8bf5e14": "This is a 1*1 pixel GIF",
- "2991c3aa1ba61a62c1cccd990c0679a1fb8dccd547d153ec0920b91a75ba20820de1d1c206f66d083bf2585d35050f0a39cd7a3e11c03882dafec907d27a0180": "This is a 1*1 pixel GIF",
- "b1a6cfa7b21dbb0b281d241af609f3ba7f3a63e5668095bba912bf7cfd7f0320baf7c3b0bfabd0f8609448f39902baeb145ba7a2d8177fe22a6fcea03dd29be1": "This is a 1*1 pixel GIF",
- "ebfe0c0df4bcc167d5cb6ebdd379f9083df62bef63a23818e1c6adf0f64b65467ea58b7cd4d03cf0a1b1a2b07fb7b969bf35f25f1f8538cc65cf3eebdf8a0910": "This is a 1*1 pixel GIF",
- "1d68b92e8d822fe82dc7563edd7b37f3418a02a89f1a9f0454cca664c2fc2565235e0d85540ff9be0b20175be3f5b7b4eae1175067465d5cca13486aab4c582c": "This is a 1*1 pixel GIF",
- "ac44da7f455bfae52b883639964276026fb259320902aa813d0333e021c356a7b3e3537b297f9a2158e588c302987ce0854866c039d1bb0ffb27f67560739db2": "This is a 1*1 pixel GIF",
- "921944dc10fbfb6224d69f0b3ac050f4790310fd1bcac3b87c96512ad5ed9a268824f3f5180563d372642071b4704c979d209baf40bc0b1c9a714769aba7dfc7": "This is a 1*1 pixel GIF",
- "89dfc38ec77cf258362e4db7c8203cae8a02c0fe4f99265b0539ec4f810c84f8451e22c9bef1ebc59b4089af7e93e378e053c542a5967ec4912d4c1fc5de22f0": "This is a 1*1 pixel GIF",
- "280ea4383ee6b37051d91c5af30a5ce72aa4439340fc6d31a4fbe7ba8a8156eb7893891d5b2371b9fc4934a78f08de3d57e5b63fa9d279a317dcbefb8a07a6b0": "This is a 1*1 pixel GIF",
- "3844065e1dd778a05e8cc39901fbf3191ded380d594359df137901ec56ca52e03d57eb60acc2421a0ee74f0733bbb5d781b7744685c26fb013a236f49b02fed3": "This is a 1*1 pixel GIF",
- "bd9ab35dde3a5242b04c159187732e13b0a6da50ddcff7015dfb78cdd68743e191eaf5cddedd49bef7d2d5a642c217272a40e5ba603fe24ca676a53f8c417c5d": "This is a 1*1 pixel GIF",
- "d052ecec2839340876eb57247cfc2e777dd7f2e868dc37cd3f3f740c8deb94917a0c9f2a4fc8229987a0b91b04726de2d1e9f6bcbe3f9bef0e4b7e0d7f65ea12": "This is a 1*1 pixel GIF",
- "8717074ddf1198d27b9918132a550cb4ba343794cc3d304a793f9d78c9ff6c4929927b414141d40b6f6ad296725520f4c63edeb660ed530267766c2ab74ee4a9": "This is a 1*1 pixel GIF",
- "6834f1548f26b94357fcc3312a3491e8c87080a84f678f990beb2c745899a01e239964521e64a534d7d5554222f728af966ec6ec8291bc64d2005861bcfd78ec": "This is a 1*1 pixel GIF",
- "3be8176915593e79bc280d08984a16c29c495bc53be9b439276094b8dcd3764a3c72a046106a06b958e08e67451fe02743175c621a1faa261fe7a9691cc77141": "This is a 1*1 pixel GIF",
- "826225fc21717d8861a05b9d2f959539aad2d2b131b2afed75d88fbca535e1b0d5a0da8ac69713a0876a0d467848a37a0a7f926aeafad8cf28201382d16466ab": "This is a 1*1 pixel GIF",
- "202612457d9042fe853daab3ddcc1f0f960c5ffdbe8462fa435713e4d1d85ff0c3f197daf8dba15bda9f5266d7e1f9ecaeee045cbc156a4892d2f931fe6fa1bb": "This is a 1*1 pixel GIF",
- "b82c6aa1ae927ade5fadbbab478cfaef26d21c1ac441f48e69cfc04cdb779b1e46d7668b4368b933213276068e52f9060228907720492a70fd9bc897191ee77c": "This is a 1*1 pixel GIF",
- "763de1053a56a94eef4f72044adb2aa370b98ffa6e0add0b1cead7ee27da519e223921c681ae1db3311273f45d0dd3dc022d102d42ce210c90cb3e761b178438": "This is a 1*1 pixel GIF",
- "69e2da5cdc318fc237eaa243b6ea7ecc83b68dbdea8478dc69154abdda86ecb4e16c35891cc1facb3ce7e0cf19d5abf189c50f59c769777706f4558f6442abbc": "This is a 1*1 pixel GIF",
- "16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516": "This is a 1*1 pixel GIF",
- "01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb": "This is a 1*1 pixel GIF",
- "71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746": "This is a 1*1 pixel GIF",
- "49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a": "This is a 1*1 pixel GIF",
- "c57ebbadcf59f982ba28da35fdbd5e5369a8500a2e1edad0dc9c9174de6fd99f437953732e545b95d3de5943c61077b6b949c989f49553ff2e483f68fcc30641": "This is a 1*1 pixel GIF",
- "c87bf81fd70cf6434ca3a6c05ad6e9bd3f1d96f77dddad8d45ee043b126b2cb07a5cf23b4137b9d8462cd8a9adf2b463ab6de2b38c93db72d2d511ca60e3b57e": "This is a 1*1 pixel GIF",
- "fd8b021f0236e487bfee13bf8f0ae98760abc492f7ca3023e292631979e135cb4ccb0c89b6234971b060ad72c0ca4474cbb5092c6c7a3255d81a54a36277b486": "This is a 1*1 pixel GIF",
- "235479f42cbbe0a4b0100167fece0d14c9b47d272b3ba8322bcfe8539f055bf31d500e7b2995cc968ebf73034e039f59c5f0f9410428663034bf119d74b5672c": "This is a 1*1 pixel GIF",
- "a85e09c3b5dbb560f4e03ba880047dbc8b4999a64c1f54fbfbca17ee0bcbed3bc6708d699190b56668e464a59358d6b534c3963a1329ba01db21075ef5bedace": "This is a 1*1 pixel GIF",
- "27656d6106a6da0c84174ba7a6307e6f1c4b3f2cc085c8466b6a25d54331035dabc7081aac208d960d8d37c5577547628c0d1c4b77bb4cf254c71859673feec1": "This is a 1*1 pixel GIF",
- # "": "This is a 1*1 pixel GIF",
- "f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498": "This is a 1*1 pixel PNG",
- "dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99": "This is a 1*1 pixel PNG",
- "c2c239cb5cdd0b670780ad6414ef6be9ccd4c21ce46bb93d1fa3120ac812f1679445162978c3df05cb2e1582a1844cc4c41cf74960b8fdae3123999c5d2176cc": "This is a 1*1 pixel PNG",
- "6ad523f5b65487369d305613366b9f68dcdeee225291766e3b25faf45439ca069f614030c08ca54c714fdbf7a944fac489b1515a8bf9e0d3191e1bcbbfe6a9df": "This is a 1*1 pixel PNG",
- # "": "This is a 1*1 pixel PNG",
- "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": "This is an empty file"
- }
-
def __init__(self, config: Dict[str, Any]):
if not ('enabled' in config or config['enabled']):
self.available = False
@@ -78,7 +31,7 @@ class SaneJavaScript():
self.storage_dir = get_homedir() / 'sanejs'
self.storage_dir.mkdir(parents=True, exist_ok=True)
- def hashes_lookup(self, sha512: Union[List[str], str], force: bool=False) -> Dict[str, Any]:
+ def hashes_lookup(self, sha512: Union[List[str], str], force: bool=False) -> Dict[str, List[str]]:
if isinstance(sha512, str):
hashes = [sha512]
else:
@@ -92,12 +45,13 @@ class SaneJavaScript():
with sanejs_unknowns.open() as f:
unknown_hashes = [line.strip() for line in f.readlines()]
- to_return: Dict[str, Union[str, List[str]]] = {h: details for h, details in self.skip_lookup.items() if h in sha512}
+ to_return: Dict[str, List[str]] = {}
- to_lookup = [h for h in hashes if h not in self.skip_lookup]
- if not force:
- to_lookup = [h for h in to_lookup if (h not in unknown_hashes
- and not (today_dir / h).exists())]
+ if force:
+ to_lookup = hashes
+ else:
+ to_lookup = [h for h in hashes if (h not in unknown_hashes
+ and not (today_dir / h).exists())]
for h in to_lookup:
response = self.client.sha512(h)
if 'error' in response:
diff --git a/poetry.lock b/poetry.lock
index d8f4707f..35c52787 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -56,13 +56,12 @@ description = "Classes Without Boilerplate"
name = "attrs"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-version = "19.3.0"
+version = "20.1.0"
[package.extras]
-azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"]
-dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"]
-docs = ["sphinx", "zope.interface"]
-tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
+dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"]
+docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"]
+tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
[[package]]
category = "main"
@@ -109,7 +108,7 @@ description = "Bootstrap helper for Flask/Jinja2."
name = "bootstrap-flask"
optional = false
python-versions = "*"
-version = "1.4"
+version = "1.5"
[package.dependencies]
Flask = "*"
@@ -140,7 +139,7 @@ description = "Foreign Function Interface for Python calling C code."
name = "cffi"
optional = false
python-versions = "*"
-version = "1.14.1"
+version = "1.14.2"
[package.dependencies]
pycparser = "*"
@@ -198,7 +197,7 @@ description = "cryptography is a package which provides cryptographic recipes an
name = "cryptography"
optional = false
python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
-version = "3.0"
+version = "3.1"
[package.dependencies]
cffi = ">=1.8,<1.11.3 || >1.11.3"
@@ -207,7 +206,6 @@ six = ">=1.4.1"
[package.extras]
docs = ["sphinx (>=1.6.5,<1.8.0 || >1.8.0,<3.1.0 || >3.1.0,<3.1.1 || >3.1.1)", "sphinx-rtd-theme"]
docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"]
-idna = ["idna (>=2.1)"]
pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"]
ssh = ["bcrypt (>=3.1.5)"]
test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"]
@@ -242,7 +240,15 @@ description = "A Python Environment for (phylogenetic) Tree Exploration"
name = "ete3"
optional = false
python-versions = "*"
-version = "3.1.1"
+version = "3.1.2"
+
+[[package]]
+category = "main"
+description = "Infer file type and MIME type of any file/buffer. No external dependencies."
+name = "filetype"
+optional = false
+python-versions = "*"
+version = "1.0.7"
[[package]]
category = "main"
@@ -297,12 +303,14 @@ description = "HTTP Archive (HAR) to ETE Toolkit generator"
name = "har2tree"
optional = false
python-versions = ">=3.6,<4.0"
-version = "1.2.1"
+version = "1.2.2"
[package.dependencies]
beautifulsoup4 = ">=4.8.2,<5.0.0"
ete3 = ">=3.1.1,<4.0.0"
+filetype = ">=1.0.7,<2.0.0"
lxml = ">=4.4.2,<5.0.0"
+numpy = ">=1.19.1,<2.0.0"
publicsuffix2 = ">=2.20191221,<3.0"
six = ">=1.14.0,<2.0.0"
@@ -368,7 +376,7 @@ description = "IPython: Productive Interactive Computing"
name = "ipython"
optional = false
python-versions = ">=3.7"
-version = "7.17.0"
+version = "7.18.1"
[package.dependencies]
appnope = "*"
@@ -376,7 +384,7 @@ backcall = "*"
colorama = "*"
decorator = "*"
jedi = ">=0.10"
-pexpect = "*"
+pexpect = ">4.3"
pickleshare = "*"
prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0"
pygments = "*"
@@ -467,7 +475,7 @@ description = "More routines for operating on iterables, beyond itertools"
name = "more-itertools"
optional = false
python-versions = ">=3.5"
-version = "8.4.0"
+version = "8.5.0"
[[package]]
category = "main"
@@ -501,6 +509,14 @@ optional = false
python-versions = "*"
version = "0.4.3"
+[[package]]
+category = "main"
+description = "NumPy is the fundamental package for array computing with Python."
+name = "numpy"
+optional = false
+python-versions = ">=3.6"
+version = "1.19.1"
+
[[package]]
category = "main"
description = "Core utilities for Python packages"
@@ -580,7 +596,7 @@ description = "Library for building powerful interactive command lines in Python
name = "prompt-toolkit"
optional = false
python-versions = ">=3.6.1"
-version = "3.0.5"
+version = "3.0.7"
[package.dependencies]
wcwidth = "*"
@@ -986,7 +1002,7 @@ description = "Backported and Experimental Type Hints for Python 3.5+"
name = "typing-extensions"
optional = false
python-versions = "*"
-version = "3.7.4.2"
+version = "3.7.4.3"
[[package]]
category = "main"
@@ -1130,8 +1146,8 @@ atomicwrites = [
{file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"},
]
attrs = [
- {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"},
- {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"},
+ {file = "attrs-20.1.0-py2.py3-none-any.whl", hash = "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"},
+ {file = "attrs-20.1.0.tar.gz", hash = "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a"},
]
automat = [
{file = "Automat-20.2.0-py2.py3-none-any.whl", hash = "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"},
@@ -1147,8 +1163,8 @@ beautifulsoup4 = [
{file = "beautifulsoup4-4.9.1.tar.gz", hash = "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7"},
]
bootstrap-flask = [
- {file = "Bootstrap-Flask-1.4.tar.gz", hash = "sha256:f5e75b7fb976ba2268eac88240e1a9b249c7848a0cb639f45a6be14d913c88c2"},
- {file = "Bootstrap_Flask-1.4-py2.py3-none-any.whl", hash = "sha256:a64368423512bff9e12b2d9f63be510bd3926f31e61b6be3760761d67a47a786"},
+ {file = "Bootstrap-Flask-1.5.tar.gz", hash = "sha256:94b8e67f7ba15e8e6ba83e7ca30aa784f45c8d713a18d8fbf013a59ce9370954"},
+ {file = "Bootstrap_Flask-1.5-py2.py3-none-any.whl", hash = "sha256:77f26a4ecd749063433b0e8780652c41c1c29bc91bf88756f07330fef3158cbb"},
]
cchardet = [
{file = "cchardet-2.1.6-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:2aa1b008965c703ad6597361b0f6d427c8971fe94a2c99ec3724c228ae50d6a6"},
@@ -1186,34 +1202,34 @@ certifi = [
{file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"},
]
cffi = [
- {file = "cffi-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:66dd45eb9530e3dde8f7c009f84568bc7cac489b93d04ac86e3111fb46e470c2"},
- {file = "cffi-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:4f53e4128c81ca3212ff4cf097c797ab44646a40b42ec02a891155cd7a2ba4d8"},
- {file = "cffi-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:833401b15de1bb92791d7b6fb353d4af60dc688eaa521bd97203dcd2d124a7c1"},
- {file = "cffi-1.14.1-cp27-cp27m-win32.whl", hash = "sha256:26f33e8f6a70c255767e3c3f957ccafc7f1f706b966e110b855bfe944511f1f9"},
- {file = "cffi-1.14.1-cp27-cp27m-win_amd64.whl", hash = "sha256:b87dfa9f10a470eee7f24234a37d1d5f51e5f5fa9eeffda7c282e2b8f5162eb1"},
- {file = "cffi-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:effd2ba52cee4ceff1a77f20d2a9f9bf8d50353c854a282b8760ac15b9833168"},
- {file = "cffi-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bac0d6f7728a9cc3c1e06d4fcbac12aaa70e9379b3025b27ec1226f0e2d404cf"},
- {file = "cffi-1.14.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:d6033b4ffa34ef70f0b8086fd4c3df4bf801fee485a8a7d4519399818351aa8e"},
- {file = "cffi-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:8416ed88ddc057bab0526d4e4e9f3660f614ac2394b5e019a628cdfff3733849"},
- {file = "cffi-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:892daa86384994fdf4856cb43c93f40cbe80f7f95bb5da94971b39c7f54b3a9c"},
- {file = "cffi-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:c991112622baee0ae4d55c008380c32ecfd0ad417bcd0417ba432e6ba7328caa"},
- {file = "cffi-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:fcf32bf76dc25e30ed793145a57426064520890d7c02866eb93d3e4abe516948"},
- {file = "cffi-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f960375e9823ae6a07072ff7f8a85954e5a6434f97869f50d0e41649a1c8144f"},
- {file = "cffi-1.14.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a6d28e7f14ecf3b2ad67c4f106841218c8ab12a0683b1528534a6c87d2307af3"},
- {file = "cffi-1.14.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:cda422d54ee7905bfc53ee6915ab68fe7b230cacf581110df4272ee10462aadc"},
- {file = "cffi-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:4a03416915b82b81af5502459a8a9dd62a3c299b295dcdf470877cb948d655f2"},
- {file = "cffi-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:4ce1e995aeecf7cc32380bc11598bfdfa017d592259d5da00fc7ded11e61d022"},
- {file = "cffi-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e23cb7f1d8e0f93addf0cae3c5b6f00324cccb4a7949ee558d7b6ca973ab8ae9"},
- {file = "cffi-1.14.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ddff0b2bd7edcc8c82d1adde6dbbf5e60d57ce985402541cd2985c27f7bec2a0"},
- {file = "cffi-1.14.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:f90c2267101010de42f7273c94a1f026e56cbc043f9330acd8a80e64300aba33"},
- {file = "cffi-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:3cd2c044517f38d1b577f05927fb9729d3396f1d44d0c659a445599e79519792"},
- {file = "cffi-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fa72a52a906425416f41738728268072d5acfd48cbe7796af07a923236bcf96"},
- {file = "cffi-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:267adcf6e68d77ba154334a3e4fc921b8e63cbb38ca00d33d40655d4228502bc"},
- {file = "cffi-1.14.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:d3148b6ba3923c5850ea197a91a42683f946dba7e8eb82dfa211ab7e708de939"},
- {file = "cffi-1.14.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:98be759efdb5e5fa161e46d404f4e0ce388e72fbf7d9baf010aff16689e22abe"},
- {file = "cffi-1.14.1-cp38-cp38-win32.whl", hash = "sha256:6923d077d9ae9e8bacbdb1c07ae78405a9306c8fd1af13bfa06ca891095eb995"},
- {file = "cffi-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:b1d6ebc891607e71fd9da71688fcf332a6630b7f5b7f5549e6e631821c0e5d90"},
- {file = "cffi-1.14.1.tar.gz", hash = "sha256:b2a2b0d276a136146e012154baefaea2758ef1f56ae9f4e01c612b0831e0bd2f"},
+ {file = "cffi-1.14.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:da9d3c506f43e220336433dffe643fbfa40096d408cb9b7f2477892f369d5f82"},
+ {file = "cffi-1.14.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23e44937d7695c27c66a54d793dd4b45889a81b35c0751ba91040fe825ec59c4"},
+ {file = "cffi-1.14.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:0da50dcbccd7cb7e6c741ab7912b2eff48e85af217d72b57f80ebc616257125e"},
+ {file = "cffi-1.14.2-cp27-cp27m-win32.whl", hash = "sha256:76ada88d62eb24de7051c5157a1a78fd853cca9b91c0713c2e973e4196271d0c"},
+ {file = "cffi-1.14.2-cp27-cp27m-win_amd64.whl", hash = "sha256:15a5f59a4808f82d8ec7364cbace851df591c2d43bc76bcbe5c4543a7ddd1bf1"},
+ {file = "cffi-1.14.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:e4082d832e36e7f9b2278bc774886ca8207346b99f278e54c9de4834f17232f7"},
+ {file = "cffi-1.14.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:57214fa5430399dffd54f4be37b56fe22cedb2b98862550d43cc085fb698dc2c"},
+ {file = "cffi-1.14.2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:6843db0343e12e3f52cc58430ad559d850a53684f5b352540ca3f1bc56df0731"},
+ {file = "cffi-1.14.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:577791f948d34d569acb2d1add5831731c59d5a0c50a6d9f629ae1cefd9ca4a0"},
+ {file = "cffi-1.14.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8662aabfeab00cea149a3d1c2999b0731e70c6b5bac596d95d13f643e76d3d4e"},
+ {file = "cffi-1.14.2-cp35-cp35m-win32.whl", hash = "sha256:837398c2ec00228679513802e3744d1e8e3cb1204aa6ad408b6aff081e99a487"},
+ {file = "cffi-1.14.2-cp35-cp35m-win_amd64.whl", hash = "sha256:bf44a9a0141a082e89c90e8d785b212a872db793a0080c20f6ae6e2a0ebf82ad"},
+ {file = "cffi-1.14.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:29c4688ace466a365b85a51dcc5e3c853c1d283f293dfcc12f7a77e498f160d2"},
+ {file = "cffi-1.14.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:99cc66b33c418cd579c0f03b77b94263c305c389cb0c6972dac420f24b3bf123"},
+ {file = "cffi-1.14.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:65867d63f0fd1b500fa343d7798fa64e9e681b594e0a07dc934c13e76ee28fb1"},
+ {file = "cffi-1.14.2-cp36-cp36m-win32.whl", hash = "sha256:f5033952def24172e60493b68717792e3aebb387a8d186c43c020d9363ee7281"},
+ {file = "cffi-1.14.2-cp36-cp36m-win_amd64.whl", hash = "sha256:7057613efefd36cacabbdbcef010e0a9c20a88fc07eb3e616019ea1692fa5df4"},
+ {file = "cffi-1.14.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6539314d84c4d36f28d73adc1b45e9f4ee2a89cdc7e5d2b0a6dbacba31906798"},
+ {file = "cffi-1.14.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:672b539db20fef6b03d6f7a14b5825d57c98e4026401fce838849f8de73fe4d4"},
+ {file = "cffi-1.14.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:95e9094162fa712f18b4f60896e34b621df99147c2cee216cfa8f022294e8e9f"},
+ {file = "cffi-1.14.2-cp37-cp37m-win32.whl", hash = "sha256:b9aa9d8818c2e917fa2c105ad538e222a5bce59777133840b93134022a7ce650"},
+ {file = "cffi-1.14.2-cp37-cp37m-win_amd64.whl", hash = "sha256:e4b9b7af398c32e408c00eb4e0d33ced2f9121fd9fb978e6c1b57edd014a7d15"},
+ {file = "cffi-1.14.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e613514a82539fc48291d01933951a13ae93b6b444a88782480be32245ed4afa"},
+ {file = "cffi-1.14.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9b219511d8b64d3fa14261963933be34028ea0e57455baf6781fe399c2c3206c"},
+ {file = "cffi-1.14.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:c0b48b98d79cf795b0916c57bebbc6d16bb43b9fc9b8c9f57f4cf05881904c75"},
+ {file = "cffi-1.14.2-cp38-cp38-win32.whl", hash = "sha256:15419020b0e812b40d96ec9d369b2bc8109cc3295eac6e013d3261343580cc7e"},
+ {file = "cffi-1.14.2-cp38-cp38-win_amd64.whl", hash = "sha256:12a453e03124069b6896107ee133ae3ab04c624bb10683e1ed1c1663df17c13c"},
+ {file = "cffi-1.14.2.tar.gz", hash = "sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b"},
]
chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
@@ -1236,25 +1252,28 @@ constantly = [
{file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"},
]
cryptography = [
- {file = "cryptography-3.0-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:ab49edd5bea8d8b39a44b3db618e4783ef84c19c8b47286bf05dfdb3efb01c83"},
- {file = "cryptography-3.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:124af7255ffc8e964d9ff26971b3a6153e1a8a220b9a685dc407976ecb27a06a"},
- {file = "cryptography-3.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:51e40123083d2f946794f9fe4adeeee2922b581fa3602128ce85ff813d85b81f"},
- {file = "cryptography-3.0-cp27-cp27m-win32.whl", hash = "sha256:dea0ba7fe6f9461d244679efa968d215ea1f989b9c1957d7f10c21e5c7c09ad6"},
- {file = "cryptography-3.0-cp27-cp27m-win_amd64.whl", hash = "sha256:8ecf9400d0893836ff41b6f977a33972145a855b6efeb605b49ee273c5e6469f"},
- {file = "cryptography-3.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:0c608ff4d4adad9e39b5057de43657515c7da1ccb1807c3a27d4cf31fc923b4b"},
- {file = "cryptography-3.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:bec7568c6970b865f2bcebbe84d547c52bb2abadf74cefce396ba07571109c67"},
- {file = "cryptography-3.0-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:0cbfed8ea74631fe4de00630f4bb592dad564d57f73150d6f6796a24e76c76cd"},
- {file = "cryptography-3.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:a09fd9c1cca9a46b6ad4bea0a1f86ab1de3c0c932364dbcf9a6c2a5eeb44fa77"},
- {file = "cryptography-3.0-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:ce82cc06588e5cbc2a7df3c8a9c778f2cb722f56835a23a68b5a7264726bb00c"},
- {file = "cryptography-3.0-cp35-cp35m-win32.whl", hash = "sha256:9367d00e14dee8d02134c6c9524bb4bd39d4c162456343d07191e2a0b5ec8b3b"},
- {file = "cryptography-3.0-cp35-cp35m-win_amd64.whl", hash = "sha256:384d7c681b1ab904fff3400a6909261cae1d0939cc483a68bdedab282fb89a07"},
- {file = "cryptography-3.0-cp36-cp36m-win32.whl", hash = "sha256:4d355f2aee4a29063c10164b032d9fa8a82e2c30768737a2fd56d256146ad559"},
- {file = "cryptography-3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:45741f5499150593178fc98d2c1a9c6722df88b99c821ad6ae298eff0ba1ae71"},
- {file = "cryptography-3.0-cp37-cp37m-win32.whl", hash = "sha256:8ecef21ac982aa78309bb6f092d1677812927e8b5ef204a10c326fc29f1367e2"},
- {file = "cryptography-3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4b9303507254ccb1181d1803a2080a798910ba89b1a3c9f53639885c90f7a756"},
- {file = "cryptography-3.0-cp38-cp38-win32.whl", hash = "sha256:8713ddb888119b0d2a1462357d5946b8911be01ddbf31451e1d07eaa5077a261"},
- {file = "cryptography-3.0-cp38-cp38-win_amd64.whl", hash = "sha256:bea0b0468f89cdea625bb3f692cd7a4222d80a6bdafd6fb923963f2b9da0e15f"},
- {file = "cryptography-3.0.tar.gz", hash = "sha256:8e924dbc025206e97756e8903039662aa58aa9ba357d8e1d8fc29e3092322053"},
+ {file = "cryptography-3.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:969ae512a250f869c1738ca63be843488ff5cc031987d302c1f59c7dbe1b225f"},
+ {file = "cryptography-3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:b45ab1c6ece7c471f01c56f5d19818ca797c34541f0b2351635a5c9fe09ac2e0"},
+ {file = "cryptography-3.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:247df238bc05c7d2e934a761243bfdc67db03f339948b1e2e80c75d41fc7cc36"},
+ {file = "cryptography-3.1-cp27-cp27m-win32.whl", hash = "sha256:10c9775a3f31610cf6b694d1fe598f2183441de81cedcf1814451ae53d71b13a"},
+ {file = "cryptography-3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:9f734423eb9c2ea85000aa2476e0d7a58e021bc34f0a373ac52a5454cd52f791"},
+ {file = "cryptography-3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e7563eb7bc5c7e75a213281715155248cceba88b11cb4b22957ad45b85903761"},
+ {file = "cryptography-3.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:94191501e4b4009642be21dde2a78bd3c2701a81ee57d3d3d02f1d99f8b64a9e"},
+ {file = "cryptography-3.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:dc3f437ca6353979aace181f1b790f0fc79e446235b14306241633ab7d61b8f8"},
+ {file = "cryptography-3.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:725875681afe50b41aee7fdd629cedbc4720bab350142b12c55c0a4d17c7416c"},
+ {file = "cryptography-3.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:321761d55fb7cb256b771ee4ed78e69486a7336be9143b90c52be59d7657f50f"},
+ {file = "cryptography-3.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:2a27615c965173c4c88f2961cf18115c08fedfb8bdc121347f26e8458dc6d237"},
+ {file = "cryptography-3.1-cp35-cp35m-win32.whl", hash = "sha256:e7dad66a9e5684a40f270bd4aee1906878193ae50a4831922e454a2a457f1716"},
+ {file = "cryptography-3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4005b38cd86fc51c955db40b0f0e52ff65340874495af72efabb1bb8ca881695"},
+ {file = "cryptography-3.1-cp36-abi3-win32.whl", hash = "sha256:cc6096c86ec0de26e2263c228fb25ee01c3ff1346d3cfc219d67d49f303585af"},
+ {file = "cryptography-3.1-cp36-abi3-win_amd64.whl", hash = "sha256:2e26223ac636ca216e855748e7d435a1bf846809ed12ed898179587d0cf74618"},
+ {file = "cryptography-3.1-cp36-cp36m-win32.whl", hash = "sha256:7a63e97355f3cd77c94bd98c59cb85fe0efd76ea7ef904c9b0316b5bbfde6ed1"},
+ {file = "cryptography-3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:4b9e96543d0784acebb70991ebc2dbd99aa287f6217546bb993df22dd361d41c"},
+ {file = "cryptography-3.1-cp37-cp37m-win32.whl", hash = "sha256:eb80a288e3cfc08f679f95da72d2ef90cb74f6d8a8ba69d2f215c5e110b2ca32"},
+ {file = "cryptography-3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:180c9f855a8ea280e72a5d61cf05681b230c2dce804c48e9b2983f491ecc44ed"},
+ {file = "cryptography-3.1-cp38-cp38-win32.whl", hash = "sha256:fa7fbcc40e2210aca26c7ac8a39467eae444d90a2c346cbcffd9133a166bcc67"},
+ {file = "cryptography-3.1-cp38-cp38-win_amd64.whl", hash = "sha256:548b0818e88792318dc137d8b1ec82a0ab0af96c7f0603a00bb94f896fbf5e10"},
+ {file = "cryptography-3.1.tar.gz", hash = "sha256:26409a473cc6278e4c90f782cd5968ebad04d3911ed1c402fc86908c17633e08"},
]
cssselect = [
{file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"},
@@ -1268,7 +1287,11 @@ defang = [
{file = "defang-0.5.3.tar.gz", hash = "sha256:86aeff658d7cd4c3b61d16089872e1c1f0a1b7b3c64d4ca9525c017caeb284d7"},
]
ete3 = [
- {file = "ete3-3.1.1.tar.gz", hash = "sha256:870a3d4b496a36fbda4b13c7c6b9dfa7638384539ae93551ec7acb377fb9c385"},
+ {file = "ete3-3.1.2.tar.gz", hash = "sha256:4fc987b8c529889d6608fab1101f1455cb5cbd42722788de6aea9c7d0a8e59e9"},
+]
+filetype = [
+ {file = "filetype-1.0.7-py2.py3-none-any.whl", hash = "sha256:353369948bb1c09b8b3ea3d78390b5586e9399bff9aab894a1dff954e31a66f6"},
+ {file = "filetype-1.0.7.tar.gz", hash = "sha256:da393ece8d98b47edf2dd5a85a2c8733e44b769e32c71af4cd96ed8d38d96aa7"},
]
flask = [
{file = "Flask-1.1.2-py2.py3-none-any.whl", hash = "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557"},
@@ -1283,8 +1306,8 @@ gunicorn = [
{file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"},
]
har2tree = [
- {file = "har2tree-1.2.1-py3-none-any.whl", hash = "sha256:180d090a455c435f5bd8dd20e8e66239722177e60e99d1e3dc89a21796aad11c"},
- {file = "har2tree-1.2.1.tar.gz", hash = "sha256:7201a1b9764127b4ef1d0ea7f541eeac3a617eb61c156d26e17878b77395ac2b"},
+ {file = "har2tree-1.2.2-py3-none-any.whl", hash = "sha256:0cdd46181dad9ae738eeedba054584299e3a60233cc15d22f163039974ca1daf"},
+ {file = "har2tree-1.2.2.tar.gz", hash = "sha256:5ae6ccad4b93bcd53b57cc4c371cc8ae1a1d614240e3228d24ca36aea5927291"},
]
hyperlink = [
{file = "hyperlink-20.0.1-py2.py3-none-any.whl", hash = "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"},
@@ -1307,8 +1330,8 @@ iniconfig = [
{file = "iniconfig-1.0.1.tar.gz", hash = "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"},
]
ipython = [
- {file = "ipython-7.17.0-py3-none-any.whl", hash = "sha256:5a8f159ca8b22b9a0a1f2a28befe5ad2b703339afb58c2ffe0d7c8d7a3af5999"},
- {file = "ipython-7.17.0.tar.gz", hash = "sha256:b70974aaa2674b05eb86a910c02ed09956a33f2dd6c71afc60f0b128a77e7f28"},
+ {file = "ipython-7.18.1-py3-none-any.whl", hash = "sha256:2e22c1f74477b5106a6fb301c342ab8c64bb75d702e350f05a649e8cb40a0fb8"},
+ {file = "ipython-7.18.1.tar.gz", hash = "sha256:a331e78086001931de9424940699691ad49dfb457cea31f5471eae7b78222d5e"},
]
ipython-genutils = [
{file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"},
@@ -1386,8 +1409,8 @@ markupsafe = [
{file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
]
more-itertools = [
- {file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"},
- {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"},
+ {file = "more-itertools-8.5.0.tar.gz", hash = "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20"},
+ {file = "more_itertools-8.5.0-py3-none-any.whl", hash = "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"},
]
multidict = [
{file = "multidict-4.7.6-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000"},
@@ -1428,6 +1451,34 @@ mypy-extensions = [
{file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
{file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
]
+numpy = [
+ {file = "numpy-1.19.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b1cca51512299841bf69add3b75361779962f9cee7d9ee3bb446d5982e925b69"},
+ {file = "numpy-1.19.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c9591886fc9cbe5532d5df85cb8e0cc3b44ba8ce4367bd4cf1b93dc19713da72"},
+ {file = "numpy-1.19.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:cf1347450c0b7644ea142712619533553f02ef23f92f781312f6a3553d031fc7"},
+ {file = "numpy-1.19.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:ed8a311493cf5480a2ebc597d1e177231984c818a86875126cfd004241a73c3e"},
+ {file = "numpy-1.19.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3673c8b2b29077f1b7b3a848794f8e11f401ba0b71c49fbd26fb40b71788b132"},
+ {file = "numpy-1.19.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:56ef7f56470c24bb67fb43dae442e946a6ce172f97c69f8d067ff8550cf782ff"},
+ {file = "numpy-1.19.1-cp36-cp36m-win32.whl", hash = "sha256:aaf42a04b472d12515debc621c31cf16c215e332242e7a9f56403d814c744624"},
+ {file = "numpy-1.19.1-cp36-cp36m-win_amd64.whl", hash = "sha256:082f8d4dd69b6b688f64f509b91d482362124986d98dc7dc5f5e9f9b9c3bb983"},
+ {file = "numpy-1.19.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e4f6d3c53911a9d103d8ec9518190e52a8b945bab021745af4939cfc7c0d4a9e"},
+ {file = "numpy-1.19.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:5b6885c12784a27e957294b60f97e8b5b4174c7504665333c5e94fbf41ae5d6a"},
+ {file = "numpy-1.19.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1bc0145999e8cb8aed9d4e65dd8b139adf1919e521177f198529687dbf613065"},
+ {file = "numpy-1.19.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:5a936fd51049541d86ccdeef2833cc89a18e4d3808fe58a8abeb802665c5af93"},
+ {file = "numpy-1.19.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:ef71a1d4fd4858596ae80ad1ec76404ad29701f8ca7cdcebc50300178db14dfc"},
+ {file = "numpy-1.19.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b9792b0ac0130b277536ab8944e7b754c69560dac0415dd4b2dbd16b902c8954"},
+ {file = "numpy-1.19.1-cp37-cp37m-win32.whl", hash = "sha256:b12e639378c741add21fbffd16ba5ad25c0a1a17cf2b6fe4288feeb65144f35b"},
+ {file = "numpy-1.19.1-cp37-cp37m-win_amd64.whl", hash = "sha256:8343bf67c72e09cfabfab55ad4a43ce3f6bf6e6ced7acf70f45ded9ebb425055"},
+ {file = "numpy-1.19.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e45f8e981a0ab47103181773cc0a54e650b2aef8c7b6cd07405d0fa8d869444a"},
+ {file = "numpy-1.19.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:667c07063940e934287993366ad5f56766bc009017b4a0fe91dbd07960d0aba7"},
+ {file = "numpy-1.19.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:480fdd4dbda4dd6b638d3863da3be82873bba6d32d1fc12ea1b8486ac7b8d129"},
+ {file = "numpy-1.19.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:935c27ae2760c21cd7354402546f6be21d3d0c806fffe967f745d5f2de5005a7"},
+ {file = "numpy-1.19.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:309cbcfaa103fc9a33ec16d2d62569d541b79f828c382556ff072442226d1968"},
+ {file = "numpy-1.19.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7ed448ff4eaffeb01094959b19cbaf998ecdee9ef9932381420d514e446601cd"},
+ {file = "numpy-1.19.1-cp38-cp38-win32.whl", hash = "sha256:de8b4a9b56255797cbddb93281ed92acbc510fb7b15df3f01bd28f46ebc4edae"},
+ {file = "numpy-1.19.1-cp38-cp38-win_amd64.whl", hash = "sha256:92feb989b47f83ebef246adabc7ff3b9a59ac30601c3f6819f8913458610bdcc"},
+ {file = "numpy-1.19.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:e1b1dc0372f530f26a03578ac75d5e51b3868b9b76cd2facba4c9ee0eb252ab1"},
+ {file = "numpy-1.19.1.zip", hash = "sha256:b8456987b637232602ceb4d663cb34106f7eb780e247d51a260b84760fd8f491"},
+]
packaging = [
{file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"},
{file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"},
@@ -1453,8 +1504,8 @@ pluggy = [
{file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
]
prompt-toolkit = [
- {file = "prompt_toolkit-3.0.5-py3-none-any.whl", hash = "sha256:df7e9e63aea609b1da3a65641ceaf5bc7d05e0a04de5bd45d05dbeffbabf9e04"},
- {file = "prompt_toolkit-3.0.5.tar.gz", hash = "sha256:563d1a4140b63ff9dd587bda9557cffb2fe73650205ab6f4383092fb882e7dc8"},
+ {file = "prompt_toolkit-3.0.7-py3-none-any.whl", hash = "sha256:83074ee28ad4ba6af190593d4d4c607ff525272a504eb159199b6dd9f950c950"},
+ {file = "prompt_toolkit-3.0.7.tar.gz", hash = "sha256:822f4605f28f7d2ba6b0b09a31e25e140871e96364d1d377667b547bb3bf4489"},
]
protego = [
{file = "Protego-0.1.16.tar.gz", hash = "sha256:a682771bc7b51b2ff41466460896c1a5a653f9a1e71639ef365a72e66d8734b4"},
@@ -1642,9 +1693,9 @@ typed-ast = [
{file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"},
]
typing-extensions = [
- {file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"},
- {file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"},
- {file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"},
+ {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"},
+ {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"},
+ {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"},
]
urllib3 = [
{file = "urllib3-1.25.10-py2.py3-none-any.whl", hash = "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"},
diff --git a/website/web/__init__.py b/website/web/__init__.py
index 7a8a5556..06c07224 100644
--- a/website/web/__init__.py
+++ b/website/web/__init__.py
@@ -191,16 +191,21 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
keys_request = {
'request_cookie': "/static/cookie_read.png",
}
+ if lookyloo.get_config('enable_context_by_users'):
+ enable_context_by_users = True
+ else:
+ enable_context_by_users = False
hostnode, urls = lookyloo.get_hostnode_investigator(tree_uuid, node_uuid)
return render_template('hostname_popup.html',
tree_uuid=tree_uuid,
- hostname_uuid=node_uuid,
+ hostnode_uuid=node_uuid,
hostname=hostnode.name,
urls=urls,
keys_response=keys_response,
- keys_request=keys_request)
+ keys_request=keys_request,
+ enable_context_by_users=enable_context_by_users)
@app.route('/tree//url//request_cookies', methods=['GET'])
@@ -492,6 +497,43 @@ def body_hash_details(body_hash: str):
return render_template('body_hash.html', body_hash=body_hash, domains=domains, captures=captures)
+@app.route('/tree//mark_as_legitimate', methods=['POST'])
+def mark_as_legitimate(tree_uuid: str):
+ if request.data:
+ legitimate_entries = request.get_json(force=True)
+ lookyloo.add_to_legitimate(tree_uuid, **legitimate_entries)
+ else:
+ lookyloo.add_to_legitimate(tree_uuid)
+ return jsonify({'message': 'Legitimate entry added.'})
+
+
+@app.route('/tree//add_context/', methods=['POST'])
+@auth.login_required
+def add_context(tree_uuid: str, urlnode_uuid: str):
+ context_data = request.form
+ ressource_hash: str = context_data.get('hash_to_contextualize') # type: ignore
+ hostnode_uuid: str = context_data.get('hostnode_uuid') # type: ignore
+ legitimate: bool = True if context_data.get('legitimate') else False
+ malicious: bool = True if context_data.get('malicious') else False
+ details: Dict[str, Dict] = {'malicious': {}, 'legitimate': {}}
+ if malicious:
+ malicious_details = {}
+ if context_data.get('malicious_type'):
+ malicious_details['type'] = context_data['malicious_type']
+ if context_data.get('malicious_target'):
+ malicious_details['target'] = context_data['malicious_target']
+ details['malicious'] = malicious_details
+ if legitimate:
+ legitimate_details = {}
+ if context_data.get('legitimate_domain'):
+ legitimate_details['domain'] = context_data['legitimate_domain']
+ if context_data.get('legitimate_description'):
+ legitimate_details['description'] = context_data['legitimate_description']
+ details['legitimate'] = legitimate_details
+ lookyloo.add_context(tree_uuid, urlnode_uuid, ressource_hash, legitimate, malicious, details)
+ return redirect(url_for('hostnode_popup', tree_uuid=tree_uuid, node_uuid=hostnode_uuid))
+
+
# Query API
@app.route('/json//redirects', methods=['GET'])
diff --git a/website/web/static/tree.js b/website/web/static/tree.js
index adaef832..e432c414 100644
--- a/website/web/static/tree.js
+++ b/website/web/static/tree.js
@@ -161,6 +161,13 @@ function UnflagAllNodes() {
.on('mouseout', () => d3.select('#tooltip').style('opacity', 0));
};
+function MarkAsLegitimate(capture_uuid, hostnode_uuid=null, urlnode_uuid=null) {
+ let data = {};
+ if (hostnode_uuid != null) { data['hostnode_uuid'] = hostnode_uuid; };
+ if (urlnode_uuid != null) { data['urlnode_uuid'] = urlnode_uuid; };
+ $.post(`/tree/${capture_uuid}/mark_as_legitimate`, data);
+};
+
function UnflagHostNode(hostnode_uuid) {
d3.select(`#node_${hostnode_uuid}`).select('rect').style('fill', 'white');
d3.select(`#node_${hostnode_uuid}`).select('text').style('fill', 'black');
@@ -320,15 +327,13 @@ function update(root, computed_node_width=0) {
// Set background based on the computed width and height
let background = main_svg.insert('rect', ':first-child')
.attr('y', 0)
- // FIXME: + 200 doesn't make much sense...
- .attr('width', newWidth + margin.right + margin.left + 200)
+ .attr('width', newWidth + (margin.right + margin.left)*2)
.attr('height', newHeight + margin.top + margin.bottom)
.style('fill', "url(#backstripes)");
// Update size
- d3.select("body svg")
- // FIXME: + 200 doesn't make much sense...
- .attr("width", newWidth + margin.right + margin.left + 200)
+ main_svg
+ .attr("width", newWidth + (margin.right + margin.left)*2)
.attr("height", newHeight + margin.top + margin.bottom)
// Update pattern
@@ -445,14 +450,14 @@ function update(root, computed_node_width=0) {
})
.on('mouseout', () => d3.select('#tooltip').style('opacity', 0));
+ const http_icon_size = 24;
if (d.data.http_content) {
- const icon_size = 24;
// set lock insecure connection
d3.select(this).append("svg").append('rect')
.attr('x', selected_node_bbox.width - 22)
.attr('y', selected_node_bbox.height - 13)
- .attr('width', icon_size)
- .attr('height', icon_size)
+ .attr('width', http_icon_size)
+ .attr('height', http_icon_size)
.attr('fill', 'white')
.attr('stroke', 'black');
@@ -460,8 +465,8 @@ function update(root, computed_node_width=0) {
.attr('x', selected_node_bbox.width - 22)
.attr('y', selected_node_bbox.height - 13)
.attr('id', 'insecure_image')
- .attr("width", icon_size)
- .attr("height", icon_size)
+ .attr("width", http_icon_size)
+ .attr("height", http_icon_size)
.attr("xlink:href", '/static/insecure.svg')
.on('mouseover', () => {
d3.select('#tooltip')
@@ -472,6 +477,83 @@ function update(root, computed_node_width=0) {
})
.on('mouseout', () => d3.select('#tooltip').style('opacity', 0));
};
+ const context_icon_size = 24;
+ if (d.data.malicious) {
+ // set bomb
+ d3.select(this).append("svg").append('rect')
+ .attr('x', selected_node_bbox.width - 22 - http_icon_size)
+ .attr('y', selected_node_bbox.height - 13)
+ .attr('width', context_icon_size)
+ .attr('height', context_icon_size)
+ .attr('fill', 'white')
+ .attr('stroke', 'black');
+
+ d3.select(this).append('image')
+ .attr('x', selected_node_bbox.width - 22 - http_icon_size)
+ .attr('y', selected_node_bbox.height - 13)
+ .attr('id', 'insecure_image')
+ .attr("width", context_icon_size)
+ .attr("height", context_icon_size)
+ .attr("xlink:href", '/static/bomb.svg')
+ .on('mouseover', () => {
+ d3.select('#tooltip')
+ .style('opacity', 1)
+ .style('left', `${d3.event.pageX + 10}px`)
+ .style('top', `${d3.event.pageY + 10}px`)
+ .text('This node containts known malicious content');
+ })
+ .on('mouseout', () => d3.select('#tooltip').style('opacity', 0));
+ } else if (d.data.legitimate) {
+ // set checkmark
+ d3.select(this).append("svg").append('rect')
+ .attr('x', selected_node_bbox.width - 22 - http_icon_size)
+ .attr('y', selected_node_bbox.height - 13)
+ .attr('width', context_icon_size)
+ .attr('height', context_icon_size)
+ .attr('fill', 'white')
+ .attr('stroke', 'black');
+
+ d3.select(this).append('image')
+ .attr('x', selected_node_bbox.width - 22 - http_icon_size)
+ .attr('y', selected_node_bbox.height - 13)
+ .attr('id', 'insecure_image')
+ .attr("width", context_icon_size)
+ .attr("height", context_icon_size)
+ .attr("xlink:href", '/static/check.svg')
+ .on('mouseover', () => {
+ d3.select('#tooltip')
+ .style('opacity', 1)
+ .style('left', `${d3.event.pageX + 10}px`)
+ .style('top', `${d3.event.pageY + 10}px`)
+ .text('This node has only legitimate content');
+ })
+ .on('mouseout', () => d3.select('#tooltip').style('opacity', 0));
+ } else if (d.data.all_empty) {
+ // set empty
+ d3.select(this).append("svg").append('rect')
+ .attr('x', selected_node_bbox.width - 22 - http_icon_size)
+ .attr('y', selected_node_bbox.height - 13)
+ .attr('width', context_icon_size)
+ .attr('height', context_icon_size)
+ .attr('fill', 'white')
+ .attr('stroke', 'black');
+
+ d3.select(this).append('image')
+ .attr('x', selected_node_bbox.width - 22 - http_icon_size)
+ .attr('y', selected_node_bbox.height - 13)
+ .attr('id', 'insecure_image')
+ .attr("width", context_icon_size)
+ .attr("height", context_icon_size)
+ .attr("xlink:href", '/static/empty.svg')
+ .on('mouseover', () => {
+ d3.select('#tooltip')
+ .style('opacity', 1)
+ .style('left', `${d3.event.pageX + 10}px`)
+ .style('top', `${d3.event.pageY + 10}px`)
+ .text('This node has only empty content');
+ })
+ .on('mouseout', () => d3.select('#tooltip').style('opacity', 0));
+ };
});
return node_group;
diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html
index c3e836ee..7eafcf91 100644
--- a/website/web/templates/hostname_popup.html
+++ b/website/web/templates/hostname_popup.html
@@ -1,5 +1,6 @@
{% extends "main.html" %}
-{% from "macros.html" import sanejs_details %}
+{% from "macros.html" import known_content_details %}
+{% from "macros.html" import ressource_legitimacy_details %}
{% from "macros.html" import indexed_hash %}
{% from "macros.html" import indexed_cookies %}
{% from "macros.html" import popup_icons %}
@@ -61,8 +62,8 @@
{# Headers #}
{{ hostname }}
- Locate in tree
- Download URLs as text
+ Locate in tree
+ Download URLs as text
{# Start list of URLs #}
{{ popup_icons(keys_response, url['url_object'], tree_uuid) }}
+
{% if url['url_object'].empty_response %}
Empty body.
{% else %}
- Body size: {{ sizeof_fmt(url['url_object'].body.getbuffer().nbytes) }}
+ {{ ressource_legitimacy_details(url['legitimacy'], url['url_object'].body.getbuffer().nbytes) }}
{%endif%}
+
- {% if url['sane_js'] %}
- {# Result from SaneJS for the response #}
- {{ sanejs_details(url['sane_js']) }}
+ {% if url['known_content'] %}
+ {{ known_content_details(url['known_content']) }}
{% endif %}
{# Everything we know about the response content #}
@@ -156,6 +158,49 @@
{% endif %}
+ {% if enable_context_by_users%}
+
+ Add context
+ Hide context form
+
+
+ {% endif %}
{% if url['embedded_ressources'] %}
{# Details on embedded resources #}
@@ -168,10 +213,10 @@
{% for hash, details in url['embedded_ressources'].items() %}
- {% if details['sane_js'] %}
- {# Result from SaneJS for the embedded ressources #}
- {{ sanejs_details(details['sane_js']) }}
+ {% if details['known_content'] %}
+ {{ known_content_details(details['known_content']) }}
{% endif %}
+ {{ ressource_legitimacy_details(details['legitimacy'], details['body_size']) }}
This file (
{{ details['type'] }} ) can be found
{{ details['hash_freq'] }} times
across all the captures on this lookyloo instance, in
{{ details['hash_domains_freq'] }} unique domains.
diff --git a/website/web/templates/macros.html b/website/web/templates/macros.html
index 3e1549cd..44c03e53 100644
--- a/website/web/templates/macros.html
+++ b/website/web/templates/macros.html
@@ -1,4 +1,4 @@
-{% macro sanejs_details(details) %}
+{% macro known_content_details(details) %}
{% if details is string %}
{{ details }}
@@ -12,6 +12,26 @@
{% endmacro %}
+{% macro ressource_legitimacy_details(details, ressource_size) %}
+{% if details and details[0] == False %}
+
+{%endif%}
+Body size: {{ sizeof_fmt(ressource_size) }}
+{% if details %}
+ {% if details[0] %}
+ - This file is known
legitimate .
+ {% elif details[0] == False %}
+ {% if details[1] is iterable %}
+
+ The response sould be considered as
phishing unless it is served by
the following domain(s) : {{ ', '.join(details[1]) }}
+
+ {% else %}
+ - The response is known
malicious .
+ {%endif%}
+ {%endif%}
+{%endif%}
+{% endmacro %}
+
{% macro indexed_hash(details, identifier_for_toggle) %}
{% set total_captures = details['different_url']|length + details['same_url']|length %}
{# Only show details if the hits are in an other capture #}
diff --git a/website/web/templates/scrape.html b/website/web/templates/scrape.html
index 52869dd6..ad93cef3 100644
--- a/website/web/templates/scrape.html
+++ b/website/web/templates/scrape.html
@@ -8,7 +8,7 @@
alt="Lookyloo" width="400">
-