chg: reorder imports

bin/archiver.py

@@ -1,17 +1,18 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from collections import defaultdict
 import csv
-from datetime import datetime, timedelta
 import logging
-from typing import Dict, List
+from collections import defaultdict
+from datetime import datetime, timedelta
 from pathlib import Path
+from typing import Dict, List
 
 from redis import Redis
 
 from lookyloo.abstractmanager import AbstractManager
-from lookyloo.helpers import get_config, get_homedir, get_socket_path, get_captures_dir
+from lookyloo.helpers import (get_captures_dir, get_config, get_homedir,
+                              get_socket_path)
 
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
                     level=logging.INFO)

bin/async_capture.py

@@ -6,11 +6,10 @@ import ipaddress
 import json
 import logging
 import socket
-
-from io import BufferedIOBase
 from datetime import datetime
+from io import BufferedIOBase
 from pathlib import Path
-from typing import Union, Dict, Optional, Tuple, List
+from typing import Dict, List, Optional, Tuple, Union
 from urllib.parse import urlsplit
 
 from defang import refang  # type: ignore
@@ -18,9 +17,9 @@ from redis import Redis
 from scrapysplashwrapper import crawl
 
 from lookyloo.abstractmanager import AbstractManager
-from lookyloo.helpers import (splash_status, get_socket_path,
+from lookyloo.helpers import (get_captures_dir, get_config, get_socket_path,
-                              load_cookies, safe_create_dir, get_config, get_splash_url,
+                              get_splash_url, load_cookies, safe_create_dir,
-                              get_captures_dir)
+                              splash_status)
 from lookyloo.lookyloo import Lookyloo
 
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',

bin/background_indexer.py

@@ -5,8 +5,8 @@ import logging
 from datetime import datetime, timedelta
 
 from lookyloo.abstractmanager import AbstractManager
+from lookyloo.exceptions import MissingUUID, NoValidHarFile
 from lookyloo.lookyloo import Lookyloo
-from lookyloo.exceptions import NoValidHarFile, MissingUUID
 
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
                     level=logging.INFO)

bin/background_processing.py

@@ -4,8 +4,8 @@
 import json
 import logging
 from collections import Counter
-from datetime import timedelta, date
+from datetime import date, timedelta
-from typing import Dict, Any
+from typing import Any, Dict
 
 from redis import Redis
 from werkzeug.useragents import UserAgent

bin/rebuild_caches.py

@@ -4,7 +4,7 @@
 import argparse
 import logging
 
-from lookyloo.lookyloo import Lookyloo, Indexing
+from lookyloo.lookyloo import Indexing, Lookyloo
 
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
                     level=logging.INFO)

bin/run_backend.py

@@ -4,13 +4,13 @@
 import argparse
 import os
 import time
-
 from pathlib import Path
 from subprocess import Popen
-from typing import Optional, List, Union
+from typing import List, Optional, Union
 
 from redis import Redis
 from redis.exceptions import ConnectionError
+
 from lookyloo.helpers import get_homedir, get_socket_path
 
 

bin/start.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from subprocess import run, Popen
+from subprocess import Popen, run
-from lookyloo.helpers import get_homedir, get_config
+
+from lookyloo.helpers import get_config, get_homedir
 
 
 def main():

bin/start_website.py

@@ -2,11 +2,10 @@
 # -*- coding: utf-8 -*-
 
 import logging
-
 from subprocess import Popen
 
 from lookyloo.abstractmanager import AbstractManager
-from lookyloo.helpers import get_homedir, get_config
+from lookyloo.helpers import get_config, get_homedir
 
 logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s:%(message)s',
                     level=logging.INFO)

bin/stop.py

@@ -2,9 +2,11 @@
 # -*- coding: utf-8 -*-
 
 from subprocess import Popen
-from lookyloo.helpers import get_homedir, get_socket_path
+
 from redis import Redis
 
+from lookyloo.helpers import get_homedir, get_socket_path
+
 
 def main():
     get_homedir()

bin/update.py

@@ -2,12 +2,12 @@
 # -*- coding: utf-8 -*-
 
 import argparse
-import logging
-import subprocess
-import shlex
-import sys
 import hashlib
+import logging
 import platform
+import shlex
+import subprocess
+import sys
 from pathlib import Path
 
 from lookyloo.helpers import get_homedir

lookyloo/__init__.py

@@ -1,2 +1,3 @@
 import logging
+
 logging.getLogger(__name__).addHandler(logging.NullHandler())

lookyloo/abstractmanager.py

@@ -4,11 +4,10 @@
 import logging
 import signal
 import time
-
 from abc import ABC
 from datetime import datetime, timedelta
-from typing import Optional, List, Tuple
 from subprocess import Popen
+from typing import List, Optional, Tuple
 
 from redis import Redis
 from redis.exceptions import ConnectionError

lookyloo/capturecache.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from datetime import datetime
 import json
+from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 

lookyloo/context.py

@@ -4,14 +4,14 @@
 import json
 import logging
 from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Union
 from urllib.parse import urlsplit
-from typing import Optional, Union, Set, Any, Dict, List
 
-from redis import Redis
 from har2tree import CrawledTree, HostNode, URLNode
+from redis import Redis
 
-from .helpers import (get_config, get_socket_path, load_known_content, get_resources_hashes,
+from .helpers import (get_config, get_homedir, get_resources_hashes,
-                      get_homedir, serialize_to_json)
+                      get_socket_path, load_known_content, serialize_to_json)
 from .modules import SaneJavaScript
 
 

lookyloo/helpers.py

@@ -1,27 +1,26 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-import os
-import logging
 import json
+import logging
+import os
 import pickle
-import pkg_resources
+from datetime import datetime, timedelta
-from typing import List, Optional, Dict, Union, Any, Set, Tuple
+from enum import IntEnum, unique
-from urllib.parse import urljoin
+from functools import lru_cache
+from glob import glob
 from io import BufferedIOBase
 from pathlib import Path
-from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
-from glob import glob
+from urllib.parse import urljoin, urlparse
-from urllib.parse import urlparse
-from functools import lru_cache
-from enum import IntEnum, unique
 
-from har2tree import CrawledTree, HostNode, URLNode
+import pkg_resources
 import requests
-from requests.exceptions import HTTPError
+from har2tree import CrawledTree, HostNode, URLNode
 from publicsuffix2 import PublicSuffixList, fetch  # type: ignore
 from pytaxonomies import Taxonomies
+from requests.exceptions import HTTPError
 
-from .exceptions import MissingEnv, CreateDirectoryException, ConfigError
+from .exceptions import ConfigError, CreateDirectoryException, MissingEnv
 
 configs: Dict[str, Dict[str, Any]] = {}
 logger = logging.getLogger('Lookyloo - Helpers')

lookyloo/indexing.py

@@ -2,16 +2,16 @@
 # -*- coding: utf-8 -*-
 
 import hashlib
-from urllib.parse import urlsplit
-from typing import List, Tuple, Set, Dict, Optional, Iterable
-from collections import defaultdict
 import re
+from collections import defaultdict
+from typing import Dict, Iterable, List, Optional, Set, Tuple
+from urllib.parse import urlsplit
 
-from redis import Redis, ConnectionPool
-from redis.connection import UnixDomainSocketConnection
 from har2tree import CrawledTree
+from redis import ConnectionPool, Redis
+from redis.connection import UnixDomainSocketConnection
 
-from .helpers import get_socket_path, get_public_suffix_list
+from .helpers import get_public_suffix_list, get_socket_path
 
 
 class Indexing():

lookyloo/lookyloo.py

@@ -2,40 +2,45 @@
 # -*- coding: utf-8 -*-
 
 import base64
-from collections import defaultdict
-from datetime import datetime, date
-from email.message import EmailMessage
-from io import BytesIO
 import json
 import logging
-from pathlib import Path
+import operator
 import pickle
 import smtplib
 import sys
-from typing import Union, Dict, List, Tuple, Optional, Any, MutableMapping, Set, Iterable
+import time
+from collections import defaultdict
+from datetime import date, datetime
+from email.message import EmailMessage
+from io import BytesIO
+from pathlib import Path
+from typing import (Any, Dict, Iterable, List, MutableMapping, Optional, Set,
+                    Tuple, Union)
 from uuid import uuid4
 from zipfile import ZipFile
-import operator
-import time
 
-import dns.resolver
 import dns.rdatatype
+import dns.resolver
 from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode
 from PIL import Image  # type: ignore
-from pymisp import MISPEvent, MISPAttribute, MISPObject
+from pymisp import MISPAttribute, MISPEvent, MISPObject
-from pymisp.tools import URLObject, FileObject
+from pymisp.tools import FileObject, URLObject
-from redis import Redis, ConnectionPool
+from redis import ConnectionPool, Redis
 from redis.connection import UnixDomainSocketConnection
 from werkzeug.useragents import UserAgent
 
-from .exceptions import NoValidHarFile, MissingUUID, LookylooException, MissingCaptureDirectory
-from .helpers import (get_homedir, get_socket_path, get_config, get_email_template, load_pickle_tree,
-                      remove_pickle_tree, get_resources_hashes, get_taxonomies, uniq_domains,
-                      try_make_file, get_captures_dir, get_splash_url, CaptureStatus)
-from .modules import VirusTotal, SaneJavaScript, PhishingInitiative, MISP, UniversalWhois, UrlScan
 from .capturecache import CaptureCache
 from .context import Context
+from .exceptions import (LookylooException, MissingCaptureDirectory,
+                         MissingUUID, NoValidHarFile)
+from .helpers import (CaptureStatus, get_captures_dir, get_config,
+                      get_email_template, get_homedir, get_resources_hashes,
+                      get_socket_path, get_splash_url, get_taxonomies,
+                      load_pickle_tree, remove_pickle_tree, try_make_file,
+                      uniq_domains)
 from .indexing import Indexing
+from .modules import (MISP, PhishingInitiative, SaneJavaScript, UniversalWhois,
+                      UrlScan, VirusTotal)
 
 
 class Lookyloo():

lookyloo/modules.py

@@ -1,28 +1,28 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from typing import Dict, Any, Optional, List, Union, Iterable, Set
-from datetime import date
-from collections import defaultdict
 import hashlib
 import json
-from pathlib import Path
-import time
 import logging
-import socket
 import re
+import socket
+import time
+from collections import defaultdict
+from datetime import date
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Set, Union
 
-from .helpers import get_homedir, get_config, get_public_suffix_list, get_useragent_for_requests
-from .exceptions import ConfigError
-
-import vt  # type: ignore
-from vt.error import APIError  # type: ignore
-from pysanejs import SaneJS
-from pyeupi import PyEUPI
-from pymisp import PyMISP, MISPEvent, MISPAttribute
 import requests
+import vt  # type: ignore
+from har2tree import CrawledTree, Har2TreeError, HostNode, URLNode
+from pyeupi import PyEUPI
+from pymisp import MISPAttribute, MISPEvent, PyMISP
+from pysanejs import SaneJS
+from vt.error import APIError  # type: ignore
 
-from har2tree import CrawledTree, HostNode, URLNode, Har2TreeError
+from .exceptions import ConfigError
+from .helpers import (get_config, get_homedir, get_public_suffix_list,
+                      get_useragent_for_requests)
 
 
 class MISP():

website/web/__init__.py

@@ -1,34 +1,34 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from io import BytesIO, StringIO
-from datetime import datetime, timedelta, timezone, date
-import json
-import http
 import calendar
-from typing import Optional, Dict, Any, Union, List
+import http
+import json
 import logging
-from urllib.parse import quote_plus, unquote_plus, urlparse
 import time
-import pkg_resources
+from datetime import date, datetime, timedelta, timezone
+from io import BytesIO, StringIO
+from typing import Any, Dict, List, Optional, Union
+from urllib.parse import quote_plus, unquote_plus, urlparse
 
-from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash, jsonify
-from flask_bootstrap import Bootstrap  # type: ignore
 import flask_login  # type: ignore
+import pkg_resources
+from flask import (Flask, Response, flash, jsonify, redirect, render_template,
+                   request, send_file, url_for)
+from flask_bootstrap import Bootstrap  # type: ignore
 from flask_restx import Api  # type: ignore
-from .genericapi import api as generic_api
+from pymisp import MISPEvent, MISPServerError
 from werkzeug.security import check_password_hash
 
-from pymisp import MISPEvent, MISPServerError
+from lookyloo.exceptions import MissingUUID, NoValidHarFile
-
+from lookyloo.helpers import (CaptureStatus, get_config, get_taxonomies,
-from lookyloo.helpers import (get_user_agents, get_config, get_taxonomies, load_cookies,
+                              get_user_agents, load_cookies, splash_status)
-                              CaptureStatus, splash_status)
+from lookyloo.lookyloo import Indexing, Lookyloo
-from lookyloo.lookyloo import Lookyloo, Indexing
-from lookyloo.exceptions import NoValidHarFile, MissingUUID
 
+from .genericapi import api as generic_api
+from .helpers import (User, build_users_table, get_secret_key,
+                      load_user_from_request, src_request_ip, sri_load)
 from .proxied import ReverseProxied
-from .helpers import (src_request_ip, User, load_user_from_request, build_users_table,
-                      get_secret_key, sri_load)
 
 app: Flask = Flask(__name__)
 app.wsgi_app = ReverseProxied(app.wsgi_app)  # type: ignore

website/web/genericapi.py

@@ -1,19 +1,19 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import json
 import base64
-from typing import Dict, Any
+import json
+from typing import Any, Dict
 
-from flask import request, send_file
 import flask_login  # type: ignore
-from flask_restx import Namespace, Resource, fields, abort  # type: ignore
+from flask import request, send_file
+from flask_restx import Namespace, Resource, abort, fields  # type: ignore
 from werkzeug.security import check_password_hash
 
+from lookyloo.helpers import splash_status
 from lookyloo.lookyloo import Lookyloo
 
-from .helpers import (src_request_ip, load_user_from_request, build_users_table)
+from .helpers import build_users_table, load_user_from_request, src_request_ip
-from lookyloo.helpers import splash_status
 
 api = Namespace('GenericAPI', description='Generic Lookyloo API', path='/')
 

website/web/helpers.py

@@ -4,16 +4,14 @@
 import hashlib
 import json
 import os
-
 from functools import lru_cache
 from pathlib import Path
 from typing import Dict, List, Union
 
 import flask_login  # type: ignore
-
 from werkzeug.security import generate_password_hash
 
-from lookyloo.helpers import get_homedir, get_config
+from lookyloo.helpers import get_config, get_homedir
 
 
 def src_request_ip(request) -> str:

website/web/proxied.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-from typing import MutableMapping, Any
+from typing import Any, MutableMapping
 
 
 class ReverseProxied():