mirror of https://github.com/CIRCL/lookyloo
fix: Use publicsuffixlist instead of publicsuffix2
parent
64007fd5bc
commit
f335b4a58d
|
@ -14,7 +14,7 @@ from urllib.parse import urlparse
|
||||||
|
|
||||||
from har2tree import CrawledTree, HostNode, URLNode
|
from har2tree import CrawledTree, HostNode, URLNode
|
||||||
from playwrightcapture import get_devices
|
from playwrightcapture import get_devices
|
||||||
from publicsuffix2 import PublicSuffixList, fetch # type: ignore
|
from publicsuffixlist import PublicSuffixList # type: ignore
|
||||||
from pytaxonomies import Taxonomies
|
from pytaxonomies import Taxonomies
|
||||||
from ua_parser import user_agent_parser # type: ignore
|
from ua_parser import user_agent_parser # type: ignore
|
||||||
from werkzeug.user_agent import UserAgent
|
from werkzeug.user_agent import UserAgent
|
||||||
|
@ -56,12 +56,8 @@ def get_taxonomies():
|
||||||
@lru_cache(64)
|
@lru_cache(64)
|
||||||
def get_public_suffix_list():
|
def get_public_suffix_list():
|
||||||
"""Initialize Public Suffix List"""
|
"""Initialize Public Suffix List"""
|
||||||
try:
|
# TODO (?): fetch the list
|
||||||
psl_file = fetch()
|
return PublicSuffixList()
|
||||||
psl = PublicSuffixList(psl_file=psl_file)
|
|
||||||
except Exception:
|
|
||||||
psl = PublicSuffixList()
|
|
||||||
return psl
|
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(64)
|
@lru_cache(64)
|
||||||
|
|
|
@ -3045,4 +3045,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
|
||||||
[metadata]
|
[metadata]
|
||||||
lock-version = "2.0"
|
lock-version = "2.0"
|
||||||
python-versions = ">=3.8,<3.12"
|
python-versions = ">=3.8,<3.12"
|
||||||
content-hash = "f805ed5fcd825fc58d1cabf78b21468ce3cbf51583464412c3c70c3d3651674d"
|
content-hash = "88242fcbcceeffb0a38b63a8724bc222ec56045f57f5b94614f772dbcd717f3f"
|
||||||
|
|
|
@ -68,6 +68,7 @@ pypandora = "^1.3.0"
|
||||||
lacuscore = "^1.2.3"
|
lacuscore = "^1.2.3"
|
||||||
pylacus = "^1.2.0"
|
pylacus = "^1.2.0"
|
||||||
pyipasnhistory = "^2.1.2"
|
pyipasnhistory = "^2.1.2"
|
||||||
|
publicsuffixlist = "^0.9.2"
|
||||||
|
|
||||||
[tool.poetry.group.dev.dependencies]
|
[tool.poetry.group.dev.dependencies]
|
||||||
mypy = "^0.991"
|
mypy = "^0.991"
|
||||||
|
|
Loading…
Reference in New Issue