chg: Add basic pre-hook config

pull/382/head
Raphaël Vinot 2022-03-31 11:30:53 +02:00
parent ee69e66750
commit cf46dde1ed
42 changed files with 32 additions and 55 deletions

View File

@ -38,4 +38,3 @@ jobs:
- name: Run MyPy - name: Run MyPy
run: | run: |
poetry run mypy . poetry run mypy .

16
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,16 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
exclude: "user_agents|website/web/sri.txt"
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.1.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- repo: https://github.com/asottile/pyupgrade
rev: v2.31.1
hooks:
- id: pyupgrade
args: [--py38-plus]

View File

@ -85,4 +85,3 @@ Thank you very much [Tech Blog @ willshouse.com](https://techblog.willshouse.com
### License ### License
See our [LICENSE](LICENSE). See our [LICENSE](LICENSE).

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import csv import csv
import logging import logging
@ -72,12 +71,12 @@ class Archiver(AbstractManager):
def _update_all_capture_indexes(self): def _update_all_capture_indexes(self):
'''Run that after the captures are in the proper directories''' '''Run that after the captures are in the proper directories'''
# Recent captures # Recent captures
directories_to_index = set(capture_dir.parent.parent for capture_dir in get_captures_dir().glob('**/uuid')) directories_to_index = {capture_dir.parent.parent for capture_dir in get_captures_dir().glob('**/uuid')}
for directory_to_index in directories_to_index: for directory_to_index in directories_to_index:
self._update_index(directory_to_index) self._update_index(directory_to_index)
# Archived captures # Archived captures
directories_to_index = set(capture_dir.parent.parent for capture_dir in self.archived_captures_dir.glob('**/uuid')) directories_to_index = {capture_dir.parent.parent for capture_dir in self.archived_captures_dir.glob('**/uuid')}
for directory_to_index in directories_to_index: for directory_to_index in directories_to_index:
self._update_index(directory_to_index) self._update_index(directory_to_index)

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import base64 import base64
import ipaddress import ipaddress

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging import logging
import os import os

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
import logging import logging

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse import argparse
import os import os

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import time import time

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
from subprocess import Popen, run from subprocess import Popen, run

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging import logging
from subprocess import Popen from subprocess import Popen

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
from subprocess import Popen, run from subprocess import Popen, run

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse import argparse
import hashlib import hashlib

1
cache/cache.conf vendored
View File

@ -1373,4 +1373,3 @@ rdb-save-incremental-fsync yes
# Maximum number of set/hash/zset/list fields that will be processed from # Maximum number of set/hash/zset/list fields that will be processed from
# the main dictionary scan # the main dictionary scan
# active-defrag-max-scan-fields 1000 # active-defrag-max-scan-fields 1000

View File

@ -127,4 +127,3 @@ enforcement ladder](https://github.com/mozilla/diversity).
For answers to common questions about this code of conduct, see the FAQ at For answers to common questions about this code of conduct, see the FAQ at
https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/faq. Translations are available at
https://www.contributor-covenant.org/translations. https://www.contributor-covenant.org/translations.

View File

@ -16,4 +16,3 @@ StandardError=append:/var/log/lookyloo_error.log
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging import logging
import signal import signal

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
class LookylooException(Exception): class LookylooException(Exception):

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
import logging import logging
import os import os

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
from .misp import MISP # noqa from .misp import MISP # noqa
from .pi import PhishingInitiative # noqa from .pi import PhishingInitiative # noqa

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
from typing import Any, Dict, List from typing import Any, Dict, List

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging import logging
import re import re

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
from datetime import date, datetime, timedelta, timezone from datetime import date, datetime, timedelta, timezone
@ -47,7 +46,7 @@ class Phishtank():
with (crawled_tree.root_hartree.har.path.parent / 'ips.json').open() as f: with (crawled_tree.root_hartree.har.path.parent / 'ips.json').open() as f:
ips_dump = json.load(f) ips_dump = json.load(f)
to_return: Dict[str, List[Dict[str, Any]]] = {} to_return: Dict[str, List[Dict[str, Any]]] = {}
for ip in set(ip for ips_list in ips_dump.values() for ip in ips_list): for ip in {ip for ips_list in ips_dump.values() for ip in ips_list}:
entry = self.get_ip_lookup(ip) entry = self.get_ip_lookup(ip)
if not entry: if not entry:
continue continue
@ -90,7 +89,7 @@ class Phishtank():
# Check all the IPs in the ips file of the capture # Check all the IPs in the ips file of the capture
with (crawled_tree.root_hartree.har.path.parent / 'ips.json').open() as f: with (crawled_tree.root_hartree.har.path.parent / 'ips.json').open() as f:
ips_dump = json.load(f) ips_dump = json.load(f)
for ip in set(ip for ips_list in ips_dump.values() for ip in ips_list): for ip in {ip for ips_list in ips_dump.values() for ip in ips_list}:
self.ip_lookup(ip) self.ip_lookup(ip)
return {'success': 'Module triggered'} return {'success': 'Module triggered'}

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
import time import time

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
import logging import logging
@ -43,7 +42,7 @@ class SaneJavaScript():
unknown_hashes = set() unknown_hashes = set()
if sanejs_unknowns.exists(): if sanejs_unknowns.exists():
with sanejs_unknowns.open() as f: with sanejs_unknowns.open() as f:
unknown_hashes = set(line.strip() for line in f.readlines()) unknown_hashes = {line.strip() for line in f.readlines()}
to_return: Dict[str, List[str]] = {} to_return: Dict[str, List[str]] = {}

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
import logging import logging

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging import logging
import socket import socket
@ -65,7 +64,7 @@ class UniversalWhois():
bytes_whois = b'' bytes_whois = b''
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((self.server, self.port)) sock.connect((self.server, self.port))
sock.sendall('{}\n'.format(query).encode()) sock.sendall(f'{query}\n'.encode())
while True: while True:
data = sock.recv(2048) data = sock.recv(2048)
if not data: if not data:

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
import time import time

View File

@ -1,6 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- from setuptools import setup
from setuptools import setup # type: ignore
setup( setup(

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import requests import requests

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import base64 import base64
import hashlib import hashlib

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
import traceback import traceback
@ -7,7 +6,7 @@ from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Dict, Any from typing import Dict, Any
from bs4 import BeautifulSoup # type: ignore from bs4 import BeautifulSoup
try: try:
import cloudscraper # type: ignore import cloudscraper # type: ignore
HAS_CF = True HAS_CF = True

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os import os
import sys import sys

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse import argparse
import logging import logging

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json import json
import logging import logging

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import calendar import calendar
import http import http
@ -127,9 +126,9 @@ logging.basicConfig(level=get_config('generic', 'loglevel'))
def sizeof_fmt(num, suffix='B'): def sizeof_fmt(num, suffix='B'):
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0: if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix) return f"{num:3.1f}{unit}{suffix}"
num /= 1024.0 num /= 1024.0
return ("%.1f%s%s" % (num, 'Yi', suffix)).strip() return ("{:.1f}{}{}".format(num, 'Yi', suffix)).strip()
app.jinja_env.globals.update(sizeof_fmt=sizeof_fmt) app.jinja_env.globals.update(sizeof_fmt=sizeof_fmt)
@ -520,7 +519,7 @@ def hashlookup(tree_uuid: str):
merged, total_ressources = lookyloo.merge_hashlookup_tree(tree_uuid) merged, total_ressources = lookyloo.merge_hashlookup_tree(tree_uuid)
# We only want unique URLs for the template # We only want unique URLs for the template
for sha1, entries in merged.items(): for sha1, entries in merged.items():
entries['nodes'] = set(node.name for node in entries['nodes']) entries['nodes'] = {node.name for node in entries['nodes']}
return render_template('hashlookup.html', base_tree_uuid=tree_uuid, merged=merged, total_ressources=total_ressources) return render_template('hashlookup.html', base_tree_uuid=tree_uuid, merged=merged, total_ressources=total_ressources)

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import base64 import base64
import hashlib import hashlib

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import hashlib import hashlib
import json import json

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*-
from typing import Any, MutableMapping from typing import Any, MutableMapping