mirror of https://github.com/CIRCL/lookyloo
chg: Remove known content from SaneJS module
Raphaël Vinot
parent
de0be61e68
commit
c75b9e65fe
|
|
@ -0,0 +1,59 @@
|
|||
{
|
||||
"1px_gif": {
|
||||
"description": "1 pixel GIF",
|
||||
"entries": [
|
||||
"717ea0ff7f3f624c268eccb244e24ec1305ab21557abb3d6f1a7e183ff68a2d28f13d1d2af926c9ef6d1fb16dd8cbe34cd98cacf79091dddc7874dcee21ecfdc",
|
||||
"e508d5d17e94d14b126164082342a9ca4774f404e87a3dd56c26812493ee18d9c3d6daacca979134a94a003066aca24116de874596d00d1e52130c1283d54209",
|
||||
"2d073e10ae40fde434eb31cbedd581a35cd763e51fb7048b88caa5f949b1e6105e37a228c235bc8976e8db58ed22149cfccf83b40ce93a28390566a28975744a",
|
||||
"84e24a70b78e9de9c9d0dfeb49f3f4247dbc1c715d8844471ee40669270682e199d48f5fbec62bd984c9c0270534b407c4d2561dd6c05adec3c83c1534f32d5c",
|
||||
"d5da26b5d496edb0221df1a4057a8b0285d15592a8f8dc7016a294df37ed335f3fde6a2252962e0df38b62847f8b771463a0124ef3f84299f262ed9d9d3cee4c",
|
||||
"f7a5f748f4c0d3096a3ca972886fe9a9dff5dce7792779ec6ffc42fa880b3815e2e4c3bdea452352f3844b81864c9bfb7861f66ac961cfa66cb9cb4febe568e8",
|
||||
"b2ca25a3311dc42942e046eb1a27038b71d689925b7d6b3ebb4d7cd2c7b9a0c7de3d10175790ac060dc3f8acf3c1708c336626be06879097f4d0ecaa7f567041",
|
||||
"b8d82d64ec656c63570b82215564929adad167e61643fd72283b94f3e448ef8ab0ad42202f3537a0da89960bbdc69498608fc6ec89502c6c338b6226c8bf5e14",
|
||||
"2991c3aa1ba61a62c1cccd990c0679a1fb8dccd547d153ec0920b91a75ba20820de1d1c206f66d083bf2585d35050f0a39cd7a3e11c03882dafec907d27a0180",
|
||||
"b1a6cfa7b21dbb0b281d241af609f3ba7f3a63e5668095bba912bf7cfd7f0320baf7c3b0bfabd0f8609448f39902baeb145ba7a2d8177fe22a6fcea03dd29be1",
|
||||
"ebfe0c0df4bcc167d5cb6ebdd379f9083df62bef63a23818e1c6adf0f64b65467ea58b7cd4d03cf0a1b1a2b07fb7b969bf35f25f1f8538cc65cf3eebdf8a0910",
|
||||
"1d68b92e8d822fe82dc7563edd7b37f3418a02a89f1a9f0454cca664c2fc2565235e0d85540ff9be0b20175be3f5b7b4eae1175067465d5cca13486aab4c582c",
|
||||
"ac44da7f455bfae52b883639964276026fb259320902aa813d0333e021c356a7b3e3537b297f9a2158e588c302987ce0854866c039d1bb0ffb27f67560739db2",
|
||||
"921944dc10fbfb6224d69f0b3ac050f4790310fd1bcac3b87c96512ad5ed9a268824f3f5180563d372642071b4704c979d209baf40bc0b1c9a714769aba7dfc7",
|
||||
"89dfc38ec77cf258362e4db7c8203cae8a02c0fe4f99265b0539ec4f810c84f8451e22c9bef1ebc59b4089af7e93e378e053c542a5967ec4912d4c1fc5de22f0",
|
||||
"280ea4383ee6b37051d91c5af30a5ce72aa4439340fc6d31a4fbe7ba8a8156eb7893891d5b2371b9fc4934a78f08de3d57e5b63fa9d279a317dcbefb8a07a6b0",
|
||||
"3844065e1dd778a05e8cc39901fbf3191ded380d594359df137901ec56ca52e03d57eb60acc2421a0ee74f0733bbb5d781b7744685c26fb013a236f49b02fed3",
|
||||
"bd9ab35dde3a5242b04c159187732e13b0a6da50ddcff7015dfb78cdd68743e191eaf5cddedd49bef7d2d5a642c217272a40e5ba603fe24ca676a53f8c417c5d",
|
||||
"d052ecec2839340876eb57247cfc2e777dd7f2e868dc37cd3f3f740c8deb94917a0c9f2a4fc8229987a0b91b04726de2d1e9f6bcbe3f9bef0e4b7e0d7f65ea12",
|
||||
"8717074ddf1198d27b9918132a550cb4ba343794cc3d304a793f9d78c9ff6c4929927b414141d40b6f6ad296725520f4c63edeb660ed530267766c2ab74ee4a9",
|
||||
"6834f1548f26b94357fcc3312a3491e8c87080a84f678f990beb2c745899a01e239964521e64a534d7d5554222f728af966ec6ec8291bc64d2005861bcfd78ec",
|
||||
"3be8176915593e79bc280d08984a16c29c495bc53be9b439276094b8dcd3764a3c72a046106a06b958e08e67451fe02743175c621a1faa261fe7a9691cc77141",
|
||||
"826225fc21717d8861a05b9d2f959539aad2d2b131b2afed75d88fbca535e1b0d5a0da8ac69713a0876a0d467848a37a0a7f926aeafad8cf28201382d16466ab",
|
||||
"202612457d9042fe853daab3ddcc1f0f960c5ffdbe8462fa435713e4d1d85ff0c3f197daf8dba15bda9f5266d7e1f9ecaeee045cbc156a4892d2f931fe6fa1bb",
|
||||
"b82c6aa1ae927ade5fadbbab478cfaef26d21c1ac441f48e69cfc04cdb779b1e46d7668b4368b933213276068e52f9060228907720492a70fd9bc897191ee77c",
|
||||
"763de1053a56a94eef4f72044adb2aa370b98ffa6e0add0b1cead7ee27da519e223921c681ae1db3311273f45d0dd3dc022d102d42ce210c90cb3e761b178438",
|
||||
"69e2da5cdc318fc237eaa243b6ea7ecc83b68dbdea8478dc69154abdda86ecb4e16c35891cc1facb3ce7e0cf19d5abf189c50f59c769777706f4558f6442abbc",
|
||||
"16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516",
|
||||
"01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb",
|
||||
"71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746",
|
||||
"49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a",
|
||||
"c57ebbadcf59f982ba28da35fdbd5e5369a8500a2e1edad0dc9c9174de6fd99f437953732e545b95d3de5943c61077b6b949c989f49553ff2e483f68fcc30641",
|
||||
"c87bf81fd70cf6434ca3a6c05ad6e9bd3f1d96f77dddad8d45ee043b126b2cb07a5cf23b4137b9d8462cd8a9adf2b463ab6de2b38c93db72d2d511ca60e3b57e",
|
||||
"fd8b021f0236e487bfee13bf8f0ae98760abc492f7ca3023e292631979e135cb4ccb0c89b6234971b060ad72c0ca4474cbb5092c6c7a3255d81a54a36277b486",
|
||||
"235479f42cbbe0a4b0100167fece0d14c9b47d272b3ba8322bcfe8539f055bf31d500e7b2995cc968ebf73034e039f59c5f0f9410428663034bf119d74b5672c",
|
||||
"a85e09c3b5dbb560f4e03ba880047dbc8b4999a64c1f54fbfbca17ee0bcbed3bc6708d699190b56668e464a59358d6b534c3963a1329ba01db21075ef5bedace",
|
||||
"27656d6106a6da0c84174ba7a6307e6f1c4b3f2cc085c8466b6a25d54331035dabc7081aac208d960d8d37c5577547628c0d1c4b77bb4cf254c71859673feec1"
|
||||
]
|
||||
},
|
||||
"1px_png": {
|
||||
"description": "1 pixel PNG",
|
||||
"entries": [
|
||||
"f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498",
|
||||
"dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99",
|
||||
"c2c239cb5cdd0b670780ad6414ef6be9ccd4c21ce46bb93d1fa3120ac812f1679445162978c3df05cb2e1582a1844cc4c41cf74960b8fdae3123999c5d2176cc",
|
||||
"6ad523f5b65487369d305613366b9f68dcdeee225291766e3b25faf45439ca069f614030c08ca54c714fdbf7a944fac489b1515a8bf9e0d3191e1bcbbfe6a9df"
|
||||
]
|
||||
},
|
||||
"empty_file": {
|
||||
"description": "empty file",
|
||||
"entries": [
|
||||
"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -183,6 +183,14 @@ def get_user_agents(directory: str='user_agents') -> Dict[str, Any]:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def load_known_content(directory: str='known_content') -> Dict[str, Dict[str, Any]]:
|
||||
to_return: Dict[str, Dict[str, Any]] = {}
|
||||
for known_content_file in (get_homedir() / directory).glob('*.json'):
|
||||
with known_content_file.open() as f:
|
||||
to_return[known_content_file.stem] = json.load(f)
|
||||
return to_return
|
||||
|
||||
|
||||
def load_cookies(cookie_pseudofile: Optional[BufferedIOBase]=None) -> List[Dict[str, str]]:
|
||||
if cookie_pseudofile:
|
||||
cookies = json.load(cookie_pseudofile)
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from scrapysplashwrapper import crawl
|
|||
from werkzeug.useragents import UserAgent
|
||||
|
||||
from .exceptions import NoValidHarFile, MissingUUID
|
||||
from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template, load_pickle_tree, remove_pickle_tree
|
||||
from .helpers import get_homedir, get_socket_path, load_cookies, load_configs, safe_create_dir, get_email_template, load_pickle_tree, remove_pickle_tree, load_known_content
|
||||
from .modules import VirusTotal, SaneJavaScript, PhishingInitiative
|
||||
|
||||
|
||||
|
|
@ -168,6 +168,9 @@ class Indexing():
|
|||
return
|
||||
self.redis.sadd(f'bh|{urlnode.body_hash}|legitimate', urlnode.hostname)
|
||||
|
||||
def legitimate_body(self, body_hash: str, legitimate_hostname: str) -> None:
|
||||
self.redis.sadd(f'bh|{body_hash}|legitimate', legitimate_hostname)
|
||||
|
||||
def malicious_node(self, urlnode: URLNode) -> None:
|
||||
if urlnode.empty_response:
|
||||
return
|
||||
|
|
@ -250,9 +253,18 @@ class Lookyloo():
|
|||
if not self.sanejs.available:
|
||||
self.logger.warning('Unable to setup the SaneJS module')
|
||||
|
||||
# TODO: reorganize startup cache.
|
||||
self.cache_known_content()
|
||||
if not self.redis.exists('cache_loaded'):
|
||||
self._init_existing_dumps()
|
||||
|
||||
def cache_known_content(self) -> None:
|
||||
p = self.redis.pipeline()
|
||||
for filename, file_content in load_known_content().items():
|
||||
for k, type_content in file_content.items():
|
||||
p.hmset('known_content', {h: type_content['description'] for h in type_content['entries']})
|
||||
p.execute()
|
||||
|
||||
def cache_user_agents(self, user_agent: str, remote_ip: str) -> None:
|
||||
today = date.today().isoformat()
|
||||
self.redis.zincrby(f'user_agents|{today}', 1, f'{remote_ip}|{user_agent}')
|
||||
|
|
@ -833,13 +845,9 @@ class Lookyloo():
|
|||
|
||||
def _format_sane_js_response(self, lookup_table: Dict, h: str) -> Optional[Union[str, Tuple]]:
|
||||
if lookup_table.get(h):
|
||||
if isinstance(lookup_table[h], list):
|
||||
libname, version, path = lookup_table[h][0].split("|")
|
||||
other_files = len(lookup_table[h])
|
||||
return libname, version, path, other_files
|
||||
else:
|
||||
# Predefined generic file
|
||||
return lookup_table[h]
|
||||
libname, version, path = lookup_table[h][0].split("|")
|
||||
other_files = len(lookup_table[h])
|
||||
return libname, version, path, other_files
|
||||
return None
|
||||
|
||||
def get_hostnode_investigator(self, capture_uuid: str, node_uuid: str) -> Tuple[HostNode, List[Dict[str, Any]]]:
|
||||
|
|
@ -854,10 +862,17 @@ class Lookyloo():
|
|||
if not hostnode:
|
||||
raise MissingUUID(f'Unable to find UUID {node_uuid} in {capture_dir}')
|
||||
|
||||
sanejs_lookups: Dict[str, List[str]] = {}
|
||||
# search in locally defined known content
|
||||
# 1. get from cache all descriptions related to a body hash
|
||||
to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')]
|
||||
known_content_table = dict(zip(to_lookup, self.redis.hmget('known_content', to_lookup)))
|
||||
|
||||
# 2. query sanejs if enabled
|
||||
if hasattr(self, 'sanejs') and self.sanejs.available:
|
||||
to_lookup = [url.body_hash for url in hostnode.urls if hasattr(url, 'body_hash')]
|
||||
sanejs_lookups = self.sanejs.hashes_lookup(to_lookup)
|
||||
to_lookup = [h for h, description in known_content_table.items() if not description]
|
||||
for h, entry in self.sanejs.hashes_lookup(to_lookup).items():
|
||||
libname, version, path = entry[0].split("|")
|
||||
known_content_table[h] = (libname, version, path, len(entry))
|
||||
|
||||
urls: List[Dict[str, Any]] = []
|
||||
for url in hostnode.urls:
|
||||
|
|
@ -902,9 +917,8 @@ class Lookyloo():
|
|||
to_append['embedded_ressources'][h]['sane_js'] = sane_js_match
|
||||
|
||||
# Optional: SaneJS information
|
||||
sane_js_match = self._format_sane_js_response(sanejs_lookups, url.body_hash)
|
||||
if sane_js_match:
|
||||
to_append['sane_js'] = sane_js_match
|
||||
if url.body_hash in known_content_table:
|
||||
to_append['sane_js'] = known_content_table[url.body_hash]
|
||||
|
||||
# Optional: Cookies sent to server in request -> map to nodes who set the cookie in response
|
||||
if hasattr(url, 'cookies_sent'):
|
||||
|
|
|
|||
|
|
@ -19,53 +19,6 @@ from pyeupi import PyEUPI
|
|||
|
||||
class SaneJavaScript():
|
||||
|
||||
skip_lookup: Dict[str, str] = {
|
||||
"717ea0ff7f3f624c268eccb244e24ec1305ab21557abb3d6f1a7e183ff68a2d28f13d1d2af926c9ef6d1fb16dd8cbe34cd98cacf79091dddc7874dcee21ecfdc": "This is a 1*1 pixel GIF",
|
||||
"e508d5d17e94d14b126164082342a9ca4774f404e87a3dd56c26812493ee18d9c3d6daacca979134a94a003066aca24116de874596d00d1e52130c1283d54209": "This is a 1*1 pixel GIF",
|
||||
"2d073e10ae40fde434eb31cbedd581a35cd763e51fb7048b88caa5f949b1e6105e37a228c235bc8976e8db58ed22149cfccf83b40ce93a28390566a28975744a": "This is a 1*1 pixel GIF",
|
||||
"84e24a70b78e9de9c9d0dfeb49f3f4247dbc1c715d8844471ee40669270682e199d48f5fbec62bd984c9c0270534b407c4d2561dd6c05adec3c83c1534f32d5c": "This is a 1*1 pixel GIF",
|
||||
"d5da26b5d496edb0221df1a4057a8b0285d15592a8f8dc7016a294df37ed335f3fde6a2252962e0df38b62847f8b771463a0124ef3f84299f262ed9d9d3cee4c": "This is a 1*1 pixel GIF",
|
||||
"f7a5f748f4c0d3096a3ca972886fe9a9dff5dce7792779ec6ffc42fa880b3815e2e4c3bdea452352f3844b81864c9bfb7861f66ac961cfa66cb9cb4febe568e8": "This is a 1*1 pixel GIF",
|
||||
"b2ca25a3311dc42942e046eb1a27038b71d689925b7d6b3ebb4d7cd2c7b9a0c7de3d10175790ac060dc3f8acf3c1708c336626be06879097f4d0ecaa7f567041": "This is a 1*1 pixel GIF",
|
||||
"b8d82d64ec656c63570b82215564929adad167e61643fd72283b94f3e448ef8ab0ad42202f3537a0da89960bbdc69498608fc6ec89502c6c338b6226c8bf5e14": "This is a 1*1 pixel GIF",
|
||||
"2991c3aa1ba61a62c1cccd990c0679a1fb8dccd547d153ec0920b91a75ba20820de1d1c206f66d083bf2585d35050f0a39cd7a3e11c03882dafec907d27a0180": "This is a 1*1 pixel GIF",
|
||||
"b1a6cfa7b21dbb0b281d241af609f3ba7f3a63e5668095bba912bf7cfd7f0320baf7c3b0bfabd0f8609448f39902baeb145ba7a2d8177fe22a6fcea03dd29be1": "This is a 1*1 pixel GIF",
|
||||
"ebfe0c0df4bcc167d5cb6ebdd379f9083df62bef63a23818e1c6adf0f64b65467ea58b7cd4d03cf0a1b1a2b07fb7b969bf35f25f1f8538cc65cf3eebdf8a0910": "This is a 1*1 pixel GIF",
|
||||
"1d68b92e8d822fe82dc7563edd7b37f3418a02a89f1a9f0454cca664c2fc2565235e0d85540ff9be0b20175be3f5b7b4eae1175067465d5cca13486aab4c582c": "This is a 1*1 pixel GIF",
|
||||
"ac44da7f455bfae52b883639964276026fb259320902aa813d0333e021c356a7b3e3537b297f9a2158e588c302987ce0854866c039d1bb0ffb27f67560739db2": "This is a 1*1 pixel GIF",
|
||||
"921944dc10fbfb6224d69f0b3ac050f4790310fd1bcac3b87c96512ad5ed9a268824f3f5180563d372642071b4704c979d209baf40bc0b1c9a714769aba7dfc7": "This is a 1*1 pixel GIF",
|
||||
"89dfc38ec77cf258362e4db7c8203cae8a02c0fe4f99265b0539ec4f810c84f8451e22c9bef1ebc59b4089af7e93e378e053c542a5967ec4912d4c1fc5de22f0": "This is a 1*1 pixel GIF",
|
||||
"280ea4383ee6b37051d91c5af30a5ce72aa4439340fc6d31a4fbe7ba8a8156eb7893891d5b2371b9fc4934a78f08de3d57e5b63fa9d279a317dcbefb8a07a6b0": "This is a 1*1 pixel GIF",
|
||||
"3844065e1dd778a05e8cc39901fbf3191ded380d594359df137901ec56ca52e03d57eb60acc2421a0ee74f0733bbb5d781b7744685c26fb013a236f49b02fed3": "This is a 1*1 pixel GIF",
|
||||
"bd9ab35dde3a5242b04c159187732e13b0a6da50ddcff7015dfb78cdd68743e191eaf5cddedd49bef7d2d5a642c217272a40e5ba603fe24ca676a53f8c417c5d": "This is a 1*1 pixel GIF",
|
||||
"d052ecec2839340876eb57247cfc2e777dd7f2e868dc37cd3f3f740c8deb94917a0c9f2a4fc8229987a0b91b04726de2d1e9f6bcbe3f9bef0e4b7e0d7f65ea12": "This is a 1*1 pixel GIF",
|
||||
"8717074ddf1198d27b9918132a550cb4ba343794cc3d304a793f9d78c9ff6c4929927b414141d40b6f6ad296725520f4c63edeb660ed530267766c2ab74ee4a9": "This is a 1*1 pixel GIF",
|
||||
"6834f1548f26b94357fcc3312a3491e8c87080a84f678f990beb2c745899a01e239964521e64a534d7d5554222f728af966ec6ec8291bc64d2005861bcfd78ec": "This is a 1*1 pixel GIF",
|
||||
"3be8176915593e79bc280d08984a16c29c495bc53be9b439276094b8dcd3764a3c72a046106a06b958e08e67451fe02743175c621a1faa261fe7a9691cc77141": "This is a 1*1 pixel GIF",
|
||||
"826225fc21717d8861a05b9d2f959539aad2d2b131b2afed75d88fbca535e1b0d5a0da8ac69713a0876a0d467848a37a0a7f926aeafad8cf28201382d16466ab": "This is a 1*1 pixel GIF",
|
||||
"202612457d9042fe853daab3ddcc1f0f960c5ffdbe8462fa435713e4d1d85ff0c3f197daf8dba15bda9f5266d7e1f9ecaeee045cbc156a4892d2f931fe6fa1bb": "This is a 1*1 pixel GIF",
|
||||
"b82c6aa1ae927ade5fadbbab478cfaef26d21c1ac441f48e69cfc04cdb779b1e46d7668b4368b933213276068e52f9060228907720492a70fd9bc897191ee77c": "This is a 1*1 pixel GIF",
|
||||
"763de1053a56a94eef4f72044adb2aa370b98ffa6e0add0b1cead7ee27da519e223921c681ae1db3311273f45d0dd3dc022d102d42ce210c90cb3e761b178438": "This is a 1*1 pixel GIF",
|
||||
"69e2da5cdc318fc237eaa243b6ea7ecc83b68dbdea8478dc69154abdda86ecb4e16c35891cc1facb3ce7e0cf19d5abf189c50f59c769777706f4558f6442abbc": "This is a 1*1 pixel GIF",
|
||||
"16dd1560fdd43c3eee7bcf622d940be93e7e74dee90286da37992d69cea844130911b97f41c71f8287b54f00bd3a388191112f490470cf27c374d524f49ba516": "This is a 1*1 pixel GIF",
|
||||
"01211111688dc2007519ff56603fbe345d057337b911c829aaee97b8d02e7d885e7a2c2d51730f54a04aebc1821897c8041f15e216f1c973ed313087fa91a3fb": "This is a 1*1 pixel GIF",
|
||||
"71db01662075fac031dea18b2c766826c77dbab01400a8642cdc7059394841d5df9020076554c3beca6f808187d42e1a1acc98fad9a0e1ad32ae869145f53746": "This is a 1*1 pixel GIF",
|
||||
"49b8daf1f5ba868bc8c6b224c787a75025ca36513ef8633d1d8f34e48ee0b578f466fcc104a7bed553404ddc5f9faff3fef5f894b31cd57f32245e550fad656a": "This is a 1*1 pixel GIF",
|
||||
"c57ebbadcf59f982ba28da35fdbd5e5369a8500a2e1edad0dc9c9174de6fd99f437953732e545b95d3de5943c61077b6b949c989f49553ff2e483f68fcc30641": "This is a 1*1 pixel GIF",
|
||||
"c87bf81fd70cf6434ca3a6c05ad6e9bd3f1d96f77dddad8d45ee043b126b2cb07a5cf23b4137b9d8462cd8a9adf2b463ab6de2b38c93db72d2d511ca60e3b57e": "This is a 1*1 pixel GIF",
|
||||
"fd8b021f0236e487bfee13bf8f0ae98760abc492f7ca3023e292631979e135cb4ccb0c89b6234971b060ad72c0ca4474cbb5092c6c7a3255d81a54a36277b486": "This is a 1*1 pixel GIF",
|
||||
"235479f42cbbe0a4b0100167fece0d14c9b47d272b3ba8322bcfe8539f055bf31d500e7b2995cc968ebf73034e039f59c5f0f9410428663034bf119d74b5672c": "This is a 1*1 pixel GIF",
|
||||
"a85e09c3b5dbb560f4e03ba880047dbc8b4999a64c1f54fbfbca17ee0bcbed3bc6708d699190b56668e464a59358d6b534c3963a1329ba01db21075ef5bedace": "This is a 1*1 pixel GIF",
|
||||
"27656d6106a6da0c84174ba7a6307e6f1c4b3f2cc085c8466b6a25d54331035dabc7081aac208d960d8d37c5577547628c0d1c4b77bb4cf254c71859673feec1": "This is a 1*1 pixel GIF",
|
||||
# "": "This is a 1*1 pixel GIF",
|
||||
"f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498": "This is a 1*1 pixel PNG",
|
||||
"dc7c40381b3d22919e32c1b700ccb77b1b0aea2690642d01c1ac802561e135c01d5a4d2a0ea18efc0ec3362e8c549814a10a23563f1f56bd62aee0ced7e2bd99": "This is a 1*1 pixel PNG",
|
||||
"c2c239cb5cdd0b670780ad6414ef6be9ccd4c21ce46bb93d1fa3120ac812f1679445162978c3df05cb2e1582a1844cc4c41cf74960b8fdae3123999c5d2176cc": "This is a 1*1 pixel PNG",
|
||||
"6ad523f5b65487369d305613366b9f68dcdeee225291766e3b25faf45439ca069f614030c08ca54c714fdbf7a944fac489b1515a8bf9e0d3191e1bcbbfe6a9df": "This is a 1*1 pixel PNG",
|
||||
# "": "This is a 1*1 pixel PNG",
|
||||
"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": "This is an empty file"
|
||||
}
|
||||
|
||||
def __init__(self, config: Dict[str, Any]):
|
||||
if not ('enabled' in config or config['enabled']):
|
||||
self.available = False
|
||||
|
|
@ -78,7 +31,7 @@ class SaneJavaScript():
|
|||
self.storage_dir = get_homedir() / 'sanejs'
|
||||
self.storage_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def hashes_lookup(self, sha512: Union[List[str], str], force: bool=False) -> Dict[str, Any]:
|
||||
def hashes_lookup(self, sha512: Union[List[str], str], force: bool=False) -> Dict[str, List[str]]:
|
||||
if isinstance(sha512, str):
|
||||
hashes = [sha512]
|
||||
else:
|
||||
|
|
@ -92,12 +45,13 @@ class SaneJavaScript():
|
|||
with sanejs_unknowns.open() as f:
|
||||
unknown_hashes = [line.strip() for line in f.readlines()]
|
||||
|
||||
to_return: Dict[str, Union[str, List[str]]] = {h: details for h, details in self.skip_lookup.items() if h in sha512}
|
||||
to_return: Dict[str, List[str]] = {}
|
||||
|
||||
to_lookup = [h for h in hashes if h not in self.skip_lookup]
|
||||
if not force:
|
||||
to_lookup = [h for h in to_lookup if (h not in unknown_hashes
|
||||
and not (today_dir / h).exists())]
|
||||
if force:
|
||||
to_lookup = hashes
|
||||
else:
|
||||
to_lookup = [h for h in hashes if (h not in unknown_hashes
|
||||
and not (today_dir / h).exists())]
|
||||
for h in to_lookup:
|
||||
response = self.client.sha512(h)
|
||||
if 'error' in response:
|
||||
|
|
|
|||
|
|
@ -502,6 +502,15 @@ def mark_as_legitimate(tree_uuid: str):
|
|||
return jsonify({'message': 'Legitimate entry added.'})
|
||||
|
||||
|
||||
@app.route('/tree/<string:tree_uuid>/add_context/<string:urlnode_uuid>', methods=['POST'])
|
||||
@auth.login_required
|
||||
def add_context(tree_uuid: str, urlnode_uuid: str):
|
||||
context_data = request.form
|
||||
legitimate: bool = context_data.get('legitimate') if context_data.get('legitimate') else False # type: ignore
|
||||
malicious: bool = context_data.get('malicious') if context_data.get('malicious') else False # type: ignore
|
||||
|
||||
|
||||
|
||||
# Query API
|
||||
|
||||
@app.route('/json/<string:tree_uuid>/redirects', methods=['GET'])
|
||||
|
|
|
|||
|
|
@ -180,35 +180,36 @@
|
|||
</button>
|
||||
<div class="collapse" id="context_response_{{ url['url_object'].uuid }}">
|
||||
<div class="card card-body">
|
||||
<form role="form" action="add_context" method=post enctype=multipart/form-data>
|
||||
<div class="form-group row">
|
||||
<div class="col-sm-10">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input"type="checkbox" name="legitimate"></input>
|
||||
<label for="legitimate" class="form-check-label">Legitimate</label>
|
||||
</div>
|
||||
<form role="form" action="{{ url_for('add_context', tree_uuid=tree_uuid, urlnode_uuid=url['url_object'].uuid) }}" method=post enctype=multipart/form-data>
|
||||
<div class="form-group">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" name="legitimate" id="legitimate">
|
||||
<label for="legitimate" class="form-check-label">Legitimate</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group row">
|
||||
<label for="legitimate_domain" class="col-sm-2 col-form-label">Domain serving the file when considered legitimate (Optional):</label>
|
||||
<div class="col-sm-10">
|
||||
<input type="text" class="form-control" name="legitimate_domain" id=legitimate_domain placeholder="Domain name">
|
||||
<div class="form-group">
|
||||
<label for="legitimate_domain">Domain serving the file when considered legitimate:</label>
|
||||
<input type="text" class="form-control" name="legitimate_domain" id=legitimate_domain placeholder="Domain name">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="extra_context">Other context for this content (library name, owner, ...):</label>
|
||||
<input type="text" class="form-control" name="extra_context" id=extra_context placeholder="Context">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" name="malicious"></input>
|
||||
<label for="malicious" class="form-check-label">Malicious</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group row">
|
||||
<label for="extra_context" class="col-sm-2 col-form-label">Other context for this content (library name for example):</label>
|
||||
<div class="col-sm-10">
|
||||
<input type="text" class="form-control" name="extra_context" id=extra_context placeholder="Context">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="malicious_type">Type of malicious content (phishing, malware, ...):</label>
|
||||
<input type="text" class="form-control" name="malicious_type" id=malicious_type placeholder="Type of malicious content">
|
||||
</div>
|
||||
<div class="form-group row">
|
||||
<div class="col-sm-10">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input"type="checkbox" name="malicious"></input>
|
||||
<label for="malicious" class="form-check-label">Malicious</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="malicious_target">Legitimate target of the malicious content (expecially for phishing):</label>
|
||||
<input type="text" class="form-control" name="legitimate_domain" id=legitimate_domain placeholder="Target">
|
||||
</div>
|
||||
<button type="submit" class="btn btn-primary" id="btn-looking">Submit context</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
alt="Lookyloo" width="400">
|
||||
</center>
|
||||
</br>
|
||||
<form role="form" action="scrape" method=post enctype=multipart/form-data>
|
||||
<form role="form" action="{{ url_for('scrape_web') }}" method=post enctype=multipart/form-data>
|
||||
<div class="form-group row">
|
||||
<div class="col-sm-10">
|
||||
<div class="form-check">
|
||||
|
|
|
|||
Loading…
Reference in New Issue