diff --git a/bin/archiver.py b/bin/archiver.py index 6a88ac4..1987788 100755 --- a/bin/archiver.py +++ b/bin/archiver.py @@ -1,17 +1,18 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from collections import defaultdict import csv -from datetime import datetime, timedelta import logging -from typing import Dict, List +from collections import defaultdict +from datetime import datetime, timedelta from pathlib import Path +from typing import Dict, List from redis import Redis from lookyloo.abstractmanager import AbstractManager -from lookyloo.helpers import get_config, get_homedir, get_socket_path, get_captures_dir +from lookyloo.helpers import (get_captures_dir, get_config, get_homedir, + get_socket_path) logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/bin/async_capture.py b/bin/async_capture.py index b896994..188577d 100755 --- a/bin/async_capture.py +++ b/bin/async_capture.py @@ -6,11 +6,10 @@ import ipaddress import json import logging import socket - -from io import BufferedIOBase from datetime import datetime +from io import BufferedIOBase from pathlib import Path -from typing import Union, Dict, Optional, Tuple, List +from typing import Dict, List, Optional, Tuple, Union from urllib.parse import urlsplit from defang import refang # type: ignore @@ -18,9 +17,9 @@ from redis import Redis from scrapysplashwrapper import crawl from lookyloo.abstractmanager import AbstractManager -from lookyloo.helpers import (splash_status, get_socket_path, - load_cookies, safe_create_dir, get_config, get_splash_url, - get_captures_dir) +from lookyloo.helpers import (get_captures_dir, get_config, get_socket_path, + get_splash_url, load_cookies, safe_create_dir, + splash_status) from lookyloo.lookyloo import Lookyloo logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', diff --git a/bin/background_indexer.py b/bin/background_indexer.py index bf35540..bdc6677 100755 --- a/bin/background_indexer.py +++ b/bin/background_indexer.py @@ -5,8 +5,8 @@ import logging from datetime import datetime, timedelta from lookyloo.abstractmanager import AbstractManager +from lookyloo.exceptions import MissingUUID, NoValidHarFile from lookyloo.lookyloo import Lookyloo -from lookyloo.exceptions import NoValidHarFile, MissingUUID logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/bin/background_processing.py b/bin/background_processing.py index e3966e5..11f0fb6 100755 --- a/bin/background_processing.py +++ b/bin/background_processing.py @@ -4,8 +4,8 @@ import json import logging from collections import Counter -from datetime import timedelta, date -from typing import Dict, Any +from datetime import date, timedelta +from typing import Any, Dict from redis import Redis from werkzeug.useragents import UserAgent diff --git a/bin/rebuild_caches.py b/bin/rebuild_caches.py index 96f55dc..71d1ea6 100755 --- a/bin/rebuild_caches.py +++ b/bin/rebuild_caches.py @@ -4,7 +4,7 @@ import argparse import logging -from lookyloo.lookyloo import Lookyloo, Indexing +from lookyloo.lookyloo import Indexing, Lookyloo logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/bin/run_backend.py b/bin/run_backend.py index f45bf76..b6ed280 100755 --- a/bin/run_backend.py +++ b/bin/run_backend.py @@ -4,13 +4,13 @@ import argparse import os import time - from pathlib import Path from subprocess import Popen -from typing import Optional, List, Union +from typing import List, Optional, Union from redis import Redis from redis.exceptions import ConnectionError + from lookyloo.helpers import get_homedir, get_socket_path diff --git a/bin/start.py b/bin/start.py index f0135be..fb8bab1 100755 --- a/bin/start.py +++ b/bin/start.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from subprocess import run, Popen -from lookyloo.helpers import get_homedir, get_config +from subprocess import Popen, run + +from lookyloo.helpers import get_config, get_homedir def main(): diff --git a/bin/start_website.py b/bin/start_website.py index 317d3c8..3962716 100755 --- a/bin/start_website.py +++ b/bin/start_website.py @@ -2,11 +2,10 @@ # -*- coding: utf-8 -*- import logging - from subprocess import Popen from lookyloo.abstractmanager import AbstractManager -from lookyloo.helpers import get_homedir, get_config +from lookyloo.helpers import get_config, get_homedir logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s', level=logging.INFO) diff --git a/bin/stop.py b/bin/stop.py index 1336521..68588d4 100755 --- a/bin/stop.py +++ b/bin/stop.py @@ -2,9 +2,11 @@ # -*- coding: utf-8 -*- from subprocess import Popen -from lookyloo.helpers import get_homedir, get_socket_path + from redis import Redis +from lookyloo.helpers import get_homedir, get_socket_path + def main(): get_homedir() diff --git a/bin/update.py b/bin/update.py index 9e5c3c8..6f3c232 100755 --- a/bin/update.py +++ b/bin/update.py @@ -2,12 +2,12 @@ # -*- coding: utf-8 -*- import argparse -import logging -import subprocess -import shlex -import sys import hashlib +import logging import platform +import shlex +import subprocess +import sys from pathlib import Path from lookyloo.helpers import get_homedir diff --git a/lookyloo/__init__.py b/lookyloo/__init__.py index 26346d7..967b94b 100644 --- a/lookyloo/__init__.py +++ b/lookyloo/__init__.py @@ -1,2 +1,3 @@ import logging + logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/lookyloo/abstractmanager.py b/lookyloo/abstractmanager.py index f52e65c..cfbd4b5 100644 --- a/lookyloo/abstractmanager.py +++ b/lookyloo/abstractmanager.py @@ -4,11 +4,10 @@ import logging import signal import time - from abc import ABC from datetime import datetime, timedelta -from typing import Optional, List, Tuple from subprocess import Popen +from typing import List, Optional, Tuple from redis import Redis from redis.exceptions import ConnectionError diff --git a/lookyloo/capturecache.py b/lookyloo/capturecache.py index d64ba13..c001ce2 100644 --- a/lookyloo/capturecache.py +++ b/lookyloo/capturecache.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from datetime import datetime import json +from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Tuple diff --git a/lookyloo/context.py b/lookyloo/context.py index d0c872d..9221a1a 100644 --- a/lookyloo/context.py +++ b/lookyloo/context.py @@ -4,14 +4,14 @@ import json import logging from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Union from urllib.parse import urlsplit -from typing import Optional, Union, Set, Any, Dict, List -from redis import Redis from har2tree import CrawledTree, HostNode, URLNode +from redis import Redis -from .helpers import (get_config, get_socket_path, load_known_content, get_resources_hashes, - get_homedir, serialize_to_json) +from .helpers import (get_config, get_homedir, get_resources_hashes, + get_socket_path, load_known_content, serialize_to_json) from .modules import SaneJavaScript diff --git a/lookyloo/helpers.py b/lookyloo/helpers.py index 5c270c5..e8eb2e3 100644 --- a/lookyloo/helpers.py +++ b/lookyloo/helpers.py @@ -1,27 +1,26 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import os -import logging import json +import logging +import os import pickle -import pkg_resources -from typing import List, Optional, Dict, Union, Any, Set, Tuple -from urllib.parse import urljoin +from datetime import datetime, timedelta +from enum import IntEnum, unique +from functools import lru_cache +from glob import glob from io import BufferedIOBase from pathlib import Path -from datetime import datetime, timedelta -from glob import glob -from urllib.parse import urlparse -from functools import lru_cache -from enum import IntEnum, unique +from typing import Any, Dict, List, Optional, Set, Tuple, Union +from urllib.parse import urljoin, urlparse -from har2tree import CrawledTree, HostNode, URLNode +import pkg_resources import requests -from requests.exceptions import HTTPError +from har2tree import CrawledTree, HostNode, URLNode from publicsuffix2 import PublicSuffixList, fetch # type: ignore from pytaxonomies import Taxonomies +from requests.exceptions import HTTPError -from .exceptions import MissingEnv, CreateDirectoryException, ConfigError +from .exceptions import ConfigError, CreateDirectoryException, MissingEnv configs: Dict[str, Dict[str, Any]] = {} logger = logging.getLogger('Lookyloo - Helpers') diff --git a/lookyloo/indexing.py b/lookyloo/indexing.py index 032c0ef..954c8b6 100644 --- a/lookyloo/indexing.py +++ b/lookyloo/indexing.py @@ -2,16 +2,16 @@ # -*- coding: utf-8 -*- import hashlib -from urllib.parse import urlsplit -from typing import List, Tuple, Set, Dict, Optional, Iterable -from collections import defaultdict import re +from collections import defaultdict +from typing import Dict, Iterable, List, Optional, Set, Tuple +from urllib.parse import urlsplit -from redis import Redis, ConnectionPool -from redis.connection import UnixDomainSocketConnection from har2tree import CrawledTree +from redis import ConnectionPool, Redis +from redis.connection import UnixDomainSocketConnection -from .helpers import get_socket_path, get_public_suffix_list +from .helpers import get_public_suffix_list, get_socket_path class Indexing(): diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 25ddaa0..c26e0b7 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -2,40 +2,45 @@ # -*- coding: utf-8 -*- import base64 -from collections import defaultdict -from datetime import datetime, date -from email.message import EmailMessage -from io import BytesIO import json import logging -from pathlib import Path +import operator import pickle import smtplib import sys -from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set, Iterable +import time +from collections import defaultdict +from datetime import date, datetime +from email.message import EmailMessage +from io import BytesIO +from pathlib import Path +from typing import (Any, Dict, Iterable, List, MutableMapping, Optional, Set, + Tuple, Union) from uuid import uuid4 from zipfile import ZipFile -import operator -import time -import dns.resolver import dns.rdatatype +import dns.resolver from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode from PIL import Image # type: ignore -from pymisp import MISPEvent, MISPAttribute, MISPObject -from pymisp.tools import URLObject, FileObject -from redis import Redis, ConnectionPool +from pymisp import MISPAttribute, MISPEvent, MISPObject +from pymisp.tools import FileObject, URLObject +from redis import ConnectionPool, Redis from redis.connection import UnixDomainSocketConnection from werkzeug.useragents import UserAgent -from .exceptions import NoValidHarFile, MissingUUID, LookylooException, MissingCaptureDirectory -from .helpers import (get_homedir, get_socket_path, get_config, get_email_template, load_pickle_tree, - remove_pickle_tree, get_resources_hashes, get_taxonomies, uniq_domains, - try_make_file, get_captures_dir, get_splash_url, CaptureStatus) -from .modules import VirusTotal, SaneJavaScript, PhishingInitiative, MISP, UniversalWhois, UrlScan from .capturecache import CaptureCache from .context import Context +from .exceptions import (LookylooException, MissingCaptureDirectory, + MissingUUID, NoValidHarFile) +from .helpers import (CaptureStatus, get_captures_dir, get_config, + get_email_template, get_homedir, get_resources_hashes, + get_socket_path, get_splash_url, get_taxonomies, + load_pickle_tree, remove_pickle_tree, try_make_file, + uniq_domains) from .indexing import Indexing +from .modules import (MISP, PhishingInitiative, SaneJavaScript, UniversalWhois, + UrlScan, VirusTotal) class Lookyloo(): diff --git a/lookyloo/modules.py b/lookyloo/modules.py index e1dd4f2..1bd75c3 100644 --- a/lookyloo/modules.py +++ b/lookyloo/modules.py @@ -1,28 +1,28 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from typing import Dict, Any, Optional, List, Union, Iterable, Set -from datetime import date -from collections import defaultdict import hashlib import json -from pathlib import Path -import time import logging -import socket import re +import socket +import time +from collections import defaultdict +from datetime import date +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Set, Union -from .helpers import get_homedir, get_config, get_public_suffix_list, get_useragent_for_requests -from .exceptions import ConfigError - -import vt # type: ignore -from vt.error import APIError # type: ignore -from pysanejs import SaneJS -from pyeupi import PyEUPI -from pymisp import PyMISP, MISPEvent, MISPAttribute import requests +import vt # type: ignore +from har2tree import CrawledTree, Har2TreeError, HostNode, URLNode +from pyeupi import PyEUPI +from pymisp import MISPAttribute, MISPEvent, PyMISP +from pysanejs import SaneJS +from vt.error import APIError # type: ignore -from har2tree import CrawledTree, HostNode, URLNode, Har2TreeError +from .exceptions import ConfigError +from .helpers import (get_config, get_homedir, get_public_suffix_list, + get_useragent_for_requests) class MISP(): diff --git a/website/web/__init__.py b/website/web/__init__.py index 5be94e5..4595e29 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1,34 +1,34 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from io import BytesIO, StringIO -from datetime import datetime, timedelta, timezone, date -import json -import http import calendar -from typing import Optional, Dict, Any, Union, List +import http +import json import logging -from urllib.parse import quote_plus, unquote_plus, urlparse import time -import pkg_resources +from datetime import date, datetime, timedelta, timezone +from io import BytesIO, StringIO +from typing import Any, Dict, List, Optional, Union +from urllib.parse import quote_plus, unquote_plus, urlparse -from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash, jsonify -from flask_bootstrap import Bootstrap # type: ignore import flask_login # type: ignore +import pkg_resources +from flask import (Flask, Response, flash, jsonify, redirect, render_template, + request, send_file, url_for) +from flask_bootstrap import Bootstrap # type: ignore from flask_restx import Api # type: ignore -from .genericapi import api as generic_api +from pymisp import MISPEvent, MISPServerError from werkzeug.security import check_password_hash -from pymisp import MISPEvent, MISPServerError - -from lookyloo.helpers import (get_user_agents, get_config, get_taxonomies, load_cookies, - CaptureStatus, splash_status) -from lookyloo.lookyloo import Lookyloo, Indexing -from lookyloo.exceptions import NoValidHarFile, MissingUUID +from lookyloo.exceptions import MissingUUID, NoValidHarFile +from lookyloo.helpers import (CaptureStatus, get_config, get_taxonomies, + get_user_agents, load_cookies, splash_status) +from lookyloo.lookyloo import Indexing, Lookyloo +from .genericapi import api as generic_api +from .helpers import (User, build_users_table, get_secret_key, + load_user_from_request, src_request_ip, sri_load) from .proxied import ReverseProxied -from .helpers import (src_request_ip, User, load_user_from_request, build_users_table, - get_secret_key, sri_load) app: Flask = Flask(__name__) app.wsgi_app = ReverseProxied(app.wsgi_app) # type: ignore diff --git a/website/web/genericapi.py b/website/web/genericapi.py index 51d4cf1..6ee13ff 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -1,19 +1,19 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import json import base64 -from typing import Dict, Any +import json +from typing import Any, Dict -from flask import request, send_file import flask_login # type: ignore -from flask_restx import Namespace, Resource, fields, abort # type: ignore +from flask import request, send_file +from flask_restx import Namespace, Resource, abort, fields # type: ignore from werkzeug.security import check_password_hash +from lookyloo.helpers import splash_status from lookyloo.lookyloo import Lookyloo -from .helpers import (src_request_ip, load_user_from_request, build_users_table) -from lookyloo.helpers import splash_status +from .helpers import build_users_table, load_user_from_request, src_request_ip api = Namespace('GenericAPI', description='Generic Lookyloo API', path='/') diff --git a/website/web/helpers.py b/website/web/helpers.py index 5338b45..a2243c9 100644 --- a/website/web/helpers.py +++ b/website/web/helpers.py @@ -4,16 +4,14 @@ import hashlib import json import os - from functools import lru_cache from pathlib import Path from typing import Dict, List, Union import flask_login # type: ignore - from werkzeug.security import generate_password_hash -from lookyloo.helpers import get_homedir, get_config +from lookyloo.helpers import get_config, get_homedir def src_request_ip(request) -> str: diff --git a/website/web/proxied.py b/website/web/proxied.py index 507a949..1175294 100644 --- a/website/web/proxied.py +++ b/website/web/proxied.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import MutableMapping, Any +from typing import Any, MutableMapping class ReverseProxied():