mirror of https://github.com/CIRCL/lookyloo
chg: Add basic pre-hook config
parent
ee69e66750
commit
cf46dde1ed
|
@ -38,4 +38,3 @@ jobs:
|
|||
- name: Run MyPy
|
||||
run: |
|
||||
poetry run mypy .
|
||||
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
# See https://pre-commit.com for more information
|
||||
# See https://pre-commit.com/hooks.html for more hooks
|
||||
exclude: "user_agents|website/web/sri.txt"
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.1.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v2.31.1
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
args: [--py38-plus]
|
|
@ -85,4 +85,3 @@ Thank you very much [Tech Blog @ willshouse.com](https://techblog.willshouse.com
|
|||
|
||||
### License
|
||||
See our [LICENSE](LICENSE).
|
||||
|
||||
|
|
|
@ -2,14 +2,14 @@
|
|||
|
||||
## Supported Versions
|
||||
|
||||
At any point in time, we only support the latest version of Lookyloo.
|
||||
At any point in time, we only support the latest version of Lookyloo.
|
||||
There will be no security patches for other releases (tagged or not).
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
In the case of a security vulnerability report, we ask the reporter to send it directly to
|
||||
[CIRCL](https://www.circl.lu/contact/), if possible encrypted with the following GnuPG key:
|
||||
In the case of a security vulnerability report, we ask the reporter to send it directly to
|
||||
[CIRCL](https://www.circl.lu/contact/), if possible encrypted with the following GnuPG key:
|
||||
**CA57 2205 C002 4E06 BA70 BE89 EAAD CFFC 22BD 4CD5**.
|
||||
|
||||
If you report security vulnerabilities, do not forget to **tell us if and how you want to
|
||||
If you report security vulnerabilities, do not forget to **tell us if and how you want to
|
||||
be acknowledged** and if you already requested CVE(s). Otherwise, we will request the CVE(s) directly.
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import csv
|
||||
import logging
|
||||
|
@ -72,12 +71,12 @@ class Archiver(AbstractManager):
|
|||
def _update_all_capture_indexes(self):
|
||||
'''Run that after the captures are in the proper directories'''
|
||||
# Recent captures
|
||||
directories_to_index = set(capture_dir.parent.parent for capture_dir in get_captures_dir().glob('**/uuid'))
|
||||
directories_to_index = {capture_dir.parent.parent for capture_dir in get_captures_dir().glob('**/uuid')}
|
||||
for directory_to_index in directories_to_index:
|
||||
self._update_index(directory_to_index)
|
||||
|
||||
# Archived captures
|
||||
directories_to_index = set(capture_dir.parent.parent for capture_dir in self.archived_captures_dir.glob('**/uuid'))
|
||||
directories_to_index = {capture_dir.parent.parent for capture_dir in self.archived_captures_dir.glob('**/uuid')}
|
||||
for directory_to_index in directories_to_index:
|
||||
self._update_index(directory_to_index)
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import base64
|
||||
import ipaddress
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from subprocess import Popen, run
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
from subprocess import Popen
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from subprocess import Popen, run
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
|
|
|
@ -1373,4 +1373,3 @@ rdb-save-incremental-fsync yes
|
|||
# Maximum number of set/hash/zset/list fields that will be processed from
|
||||
# the main dictionary scan
|
||||
# active-defrag-max-scan-fields 1000
|
||||
|
||||
|
|
|
@ -127,4 +127,3 @@ enforcement ladder](https://github.com/mozilla/diversity).
|
|||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
https://www.contributor-covenant.org/faq. Translations are available at
|
||||
https://www.contributor-covenant.org/translations.
|
||||
|
||||
|
|
|
@ -45,4 +45,4 @@ services:
|
|||
links:
|
||||
- "redis-cache"
|
||||
- "redis-indexing"
|
||||
- "splash"
|
||||
- "splash"
|
||||
|
|
|
@ -16,4 +16,3 @@ StandardError=append:/var/log/lookyloo_error.log
|
|||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import signal
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
class LookylooException(Exception):
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from .misp import MISP # noqa
|
||||
from .pi import PhishingInitiative # noqa
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
@ -47,7 +46,7 @@ class Phishtank():
|
|||
with (crawled_tree.root_hartree.har.path.parent / 'ips.json').open() as f:
|
||||
ips_dump = json.load(f)
|
||||
to_return: Dict[str, List[Dict[str, Any]]] = {}
|
||||
for ip in set(ip for ips_list in ips_dump.values() for ip in ips_list):
|
||||
for ip in {ip for ips_list in ips_dump.values() for ip in ips_list}:
|
||||
entry = self.get_ip_lookup(ip)
|
||||
if not entry:
|
||||
continue
|
||||
|
@ -90,7 +89,7 @@ class Phishtank():
|
|||
# Check all the IPs in the ips file of the capture
|
||||
with (crawled_tree.root_hartree.har.path.parent / 'ips.json').open() as f:
|
||||
ips_dump = json.load(f)
|
||||
for ip in set(ip for ips_list in ips_dump.values() for ip in ips_list):
|
||||
for ip in {ip for ips_list in ips_dump.values() for ip in ips_list}:
|
||||
self.ip_lookup(ip)
|
||||
return {'success': 'Module triggered'}
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import time
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
@ -43,7 +42,7 @@ class SaneJavaScript():
|
|||
unknown_hashes = set()
|
||||
if sanejs_unknowns.exists():
|
||||
with sanejs_unknowns.open() as f:
|
||||
unknown_hashes = set(line.strip() for line in f.readlines())
|
||||
unknown_hashes = {line.strip() for line in f.readlines()}
|
||||
|
||||
to_return: Dict[str, List[str]] = {}
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import socket
|
||||
|
@ -65,7 +64,7 @@ class UniversalWhois():
|
|||
bytes_whois = b''
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.connect((self.server, self.port))
|
||||
sock.sendall('{}\n'.format(query).encode())
|
||||
sock.sendall(f'{query}\n'.encode())
|
||||
while True:
|
||||
data = sock.recv(2048)
|
||||
if not data:
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import time
|
||||
|
|
3
setup.py
3
setup.py
|
@ -1,6 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
from setuptools import setup # type: ignore
|
||||
from setuptools import setup
|
||||
|
||||
|
||||
setup(
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import requests
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import traceback
|
||||
|
@ -7,7 +6,7 @@ from datetime import datetime
|
|||
from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
|
||||
from bs4 import BeautifulSoup # type: ignore
|
||||
from bs4 import BeautifulSoup
|
||||
try:
|
||||
import cloudscraper # type: ignore
|
||||
HAS_CF = True
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import calendar
|
||||
import http
|
||||
|
@ -127,9 +126,9 @@ logging.basicConfig(level=get_config('generic', 'loglevel'))
|
|||
def sizeof_fmt(num, suffix='B'):
|
||||
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
|
||||
if abs(num) < 1024.0:
|
||||
return "%3.1f%s%s" % (num, unit, suffix)
|
||||
return f"{num:3.1f}{unit}{suffix}"
|
||||
num /= 1024.0
|
||||
return ("%.1f%s%s" % (num, 'Yi', suffix)).strip()
|
||||
return ("{:.1f}{}{}".format(num, 'Yi', suffix)).strip()
|
||||
|
||||
|
||||
app.jinja_env.globals.update(sizeof_fmt=sizeof_fmt)
|
||||
|
@ -520,7 +519,7 @@ def hashlookup(tree_uuid: str):
|
|||
merged, total_ressources = lookyloo.merge_hashlookup_tree(tree_uuid)
|
||||
# We only want unique URLs for the template
|
||||
for sha1, entries in merged.items():
|
||||
entries['nodes'] = set(node.name for node in entries['nodes'])
|
||||
entries['nodes'] = {node.name for node in entries['nodes']}
|
||||
return render_template('hashlookup.html', base_tree_uuid=tree_uuid, merged=merged, total_ressources=total_ressources)
|
||||
|
||||
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from typing import Any, MutableMapping
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue