chg: Add basic pre-hook config

.github/workflows/mypy.yml

@@ -38,4 +38,3 @@ jobs:
     - name: Run MyPy
       run: |
         poetry run mypy .
-

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+exclude: "user_agents|website/web/sri.txt"
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.31.1
+    hooks:
+    -   id: pyupgrade
+        args: [--py38-plus]

README.md

@@ -85,4 +85,3 @@ Thank you very much [Tech Blog @ willshouse.com](https://techblog.willshouse.com
 
 ### License
 See our [LICENSE](LICENSE).
-

SECURITY.md

@@ -2,14 +2,14 @@
 
 ## Supported Versions
 
-At any point in time, we only support the latest version of Lookyloo. 
+At any point in time, we only support the latest version of Lookyloo.
 There will be no security patches for other releases (tagged or not).
 
 ## Reporting a Vulnerability
 
-In the case of a security vulnerability report, we ask the reporter to send it directly to 
+In the case of a security vulnerability report, we ask the reporter to send it directly to
-[CIRCL](https://www.circl.lu/contact/), if possible encrypted with the following GnuPG key: 
+[CIRCL](https://www.circl.lu/contact/), if possible encrypted with the following GnuPG key:
 **CA57 2205 C002 4E06 BA70 BE89 EAAD CFFC 22BD 4CD5**.
 
-If you report security vulnerabilities, do not forget to **tell us if and how you want to 
+If you report security vulnerabilities, do not forget to **tell us if and how you want to
 be acknowledged** and if you already requested CVE(s). Otherwise, we will request the CVE(s) directly.

bin/archiver.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import csv
 import logging
@@ -72,12 +71,12 @@ class Archiver(AbstractManager):
     def _update_all_capture_indexes(self):
         '''Run that after the captures are in the proper directories'''
         # Recent captures
-        directories_to_index = set(capture_dir.parent.parent for capture_dir in get_captures_dir().glob('**/uuid'))
+        directories_to_index = {capture_dir.parent.parent for capture_dir in get_captures_dir().glob('**/uuid')}
         for directory_to_index in directories_to_index:
             self._update_index(directory_to_index)
 
         # Archived captures
-        directories_to_index = set(capture_dir.parent.parent for capture_dir in self.archived_captures_dir.glob('**/uuid'))
+        directories_to_index = {capture_dir.parent.parent for capture_dir in self.archived_captures_dir.glob('**/uuid')}
         for directory_to_index in directories_to_index:
             self._update_index(directory_to_index)
 

bin/async_capture.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import base64
 import ipaddress

bin/background_indexer.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import logging
 import os

bin/background_processing.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 import logging

bin/run_backend.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import argparse
 import os

bin/shutdown.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import time
 

bin/start.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 from subprocess import Popen, run
 

bin/start_website.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import logging
 from subprocess import Popen

bin/stop.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 from subprocess import Popen, run
 

bin/update.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import argparse
 import hashlib

cache/cache.conf

@@ -1373,4 +1373,3 @@ rdb-save-incremental-fsync yes
 # Maximum number of set/hash/zset/list fields that will be processed from
 # the main dictionary scan
 # active-defrag-max-scan-fields 1000
-

code_of_conduct.md

@@ -127,4 +127,3 @@ enforcement ladder](https://github.com/mozilla/diversity).
 For answers to common questions about this code of conduct, see the FAQ at
 https://www.contributor-covenant.org/faq. Translations are available at
 https://www.contributor-covenant.org/translations.
-

docker-compose.yml

@@ -45,4 +45,4 @@ services:
     links:
         - "redis-cache"
         - "redis-indexing"
-        - "splash"
+        - "splash"

etc/systemd/system/lookyloo.service.sample

@@ -16,4 +16,3 @@ StandardError=append:/var/log/lookyloo_error.log
 
 [Install]
 WantedBy=multi-user.target
-

lookyloo/default/abstractmanager.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import logging
 import signal

lookyloo/default/exceptions.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 
 class LookylooException(Exception):

lookyloo/default/helpers.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 import json
 import logging
 import os

lookyloo/modules/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 from .misp import MISP  # noqa
 from .pi import PhishingInitiative  # noqa

lookyloo/modules/hashlookup.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 from typing import Any, Dict, List

lookyloo/modules/misp.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import logging
 import re

lookyloo/modules/phishtank.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 from datetime import date, datetime, timedelta, timezone
@@ -47,7 +46,7 @@ class Phishtank():
         with (crawled_tree.root_hartree.har.path.parent / 'ips.json').open() as f:
             ips_dump = json.load(f)
         to_return: Dict[str, List[Dict[str, Any]]] = {}
-        for ip in set(ip for ips_list in ips_dump.values() for ip in ips_list):
+        for ip in {ip for ips_list in ips_dump.values() for ip in ips_list}:
             entry = self.get_ip_lookup(ip)
             if not entry:
                 continue
@@ -90,7 +89,7 @@ class Phishtank():
         # Check all the IPs in the ips file of the capture
         with (crawled_tree.root_hartree.har.path.parent / 'ips.json').open() as f:
             ips_dump = json.load(f)
-        for ip in set(ip for ips_list in ips_dump.values() for ip in ips_list):
+        for ip in {ip for ips_list in ips_dump.values() for ip in ips_list}:
             self.ip_lookup(ip)
         return {'success': 'Module triggered'}
 

lookyloo/modules/pi.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 import time

lookyloo/modules/sanejs.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 import logging
@@ -43,7 +42,7 @@ class SaneJavaScript():
         unknown_hashes = set()
         if sanejs_unknowns.exists():
             with sanejs_unknowns.open() as f:
-                unknown_hashes = set(line.strip() for line in f.readlines())
+                unknown_hashes = {line.strip() for line in f.readlines()}
 
         to_return: Dict[str, List[str]] = {}
 

lookyloo/modules/urlscan.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 import logging

lookyloo/modules/uwhois.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import logging
 import socket
@@ -65,7 +64,7 @@ class UniversalWhois():
         bytes_whois = b''
         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
             sock.connect((self.server, self.port))
-            sock.sendall('{}\n'.format(query).encode())
+            sock.sendall(f'{query}\n'.encode())
             while True:
                 data = sock.recv(2048)
                 if not data:

lookyloo/modules/vt.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 import time

setup.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
+from setuptools import setup
-from setuptools import setup  # type: ignore
 
 
 setup(

tools/3rdparty.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import requests
 

tools/change_captures_dir.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 from datetime import datetime
 from pathlib import Path

tools/generate_sri.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import base64
 import hashlib

tools/manual_parse_ua_list.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 import traceback
@@ -7,7 +6,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Dict, Any
 
-from bs4 import BeautifulSoup  # type: ignore
+from bs4 import BeautifulSoup
 try:
     import cloudscraper  # type: ignore
     HAS_CF = True

tools/monitoring.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import os
 import sys

tools/rebuild_caches.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import argparse
 import logging

tools/validate_config_files.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import json
 import logging

website/web/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import calendar
 import http
@@ -127,9 +126,9 @@ logging.basicConfig(level=get_config('generic', 'loglevel'))
 def sizeof_fmt(num, suffix='B'):
     for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
         if abs(num) < 1024.0:
-            return "%3.1f%s%s" % (num, unit, suffix)
+            return f"{num:3.1f}{unit}{suffix}"
         num /= 1024.0
-    return ("%.1f%s%s" % (num, 'Yi', suffix)).strip()
+    return ("{:.1f}{}{}".format(num, 'Yi', suffix)).strip()
 
 
 app.jinja_env.globals.update(sizeof_fmt=sizeof_fmt)
@@ -520,7 +519,7 @@ def hashlookup(tree_uuid: str):
     merged, total_ressources = lookyloo.merge_hashlookup_tree(tree_uuid)
     # We only want unique URLs for the template
     for sha1, entries in merged.items():
-        entries['nodes'] = set(node.name for node in entries['nodes'])
+        entries['nodes'] = {node.name for node in entries['nodes']}
     return render_template('hashlookup.html', base_tree_uuid=tree_uuid, merged=merged, total_ressources=total_ressources)
 
 

website/web/genericapi.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import base64
 import hashlib

website/web/helpers.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 import hashlib
 import json

website/web/proxied.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 from typing import Any, MutableMapping