diff --git a/.github/workflows/nosetests.yml b/.github/workflows/nosetests.yml new file mode 100644 index 00000000..5b1b635e --- /dev/null +++ b/.github/workflows/nosetests.yml @@ -0,0 +1,22 @@ +name: Python application + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip poetry + poetry install + - name: Test with nosetests + run: | + poetry run mypy . diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 6d786ffc..72d71b50 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -118,7 +118,7 @@ class Lookyloo(): for redirect in ct.redirects: self.vt.url_lookup(redirect, force) else: - self.vt.url_lookup(ct.root_hartree.har.first_url, force) + self.vt.url_lookup(ct.root_hartree.har.root_url, force) def get_modules_responses(self, capture_dir: Path) -> Optional[Dict[str, Any]]: ct = self._load_pickle(capture_dir / 'tree.pickle') @@ -132,7 +132,7 @@ class Lookyloo(): for redirect in ct.redirects: to_return['vt'][redirect] = self.vt.get_url_lookup(redirect) else: - to_return['vt'][ct.root_hartree.har.first_url] = self.vt.get_url_lookup(ct.root_hartree.har.first_url) + to_return['vt'][ct.root_hartree.har.root_url] = self.vt.get_url_lookup(ct.root_hartree.har.root_url) return to_return def _set_capture_cache(self, capture_dir: Path, force: bool=False) -> None: @@ -177,7 +177,7 @@ class Lookyloo(): cache: Dict[str, Union[str, int]] = {'uuid': uuid, 'title': har.initial_title, 'timestamp': har.initial_start_time, - 'url': har.first_url, + 'url': har.root_url, 'redirects': json.dumps(redirects), 'incomplete_redirects': 1 if incomplete_redirects else 0} if (capture_dir / 'no_index').exists(): # If the folders claims anonymity diff --git a/poetry.lock b/poetry.lock index 1f6a8893..68a9b78c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -132,7 +132,7 @@ description = "Python package for providing Mozilla's CA Bundle." name = "certifi" optional = false python-versions = "*" -version = "2020.4.5.1" +version = "2020.4.5.2" [[package]] category = "main" @@ -306,7 +306,7 @@ publicsuffix2 = "^2.20191221" six = "^1.14.0" [package.source] -reference = "6ee33972962351214413e07c33559691f1ae9b9d" +reference = "93d301b19b78b81d690dba5123021cc990b6c961" type = "git" url = "https://github.com/viper-framework/har2tree.git" [[package]] @@ -336,14 +336,14 @@ marker = "python_version < \"3.8\"" name = "importlib-metadata" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "1.6.0" +version = "1.6.1" [package.dependencies] zipp = ">=0.5" [package.extras] docs = ["sphinx", "rst.linker"] -testing = ["packaging", "importlib-resources"] +testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] [[package]] category = "main" @@ -870,7 +870,7 @@ scrapy = "^1.8.0" scrapy-splash = "^0.7.2" [package.source] -reference = "3b56f1d4160383982c3316d4ba9ee479783a57a5" +reference = "ae2cc94c77fba5fb40603c9714082fcd3ab9998f" type = "git" url = "https://github.com/viper-framework/ScrapySplashWrapper.git" [[package]] @@ -1015,7 +1015,7 @@ description = "Measures the displayed width of unicode strings in a terminal" name = "wcwidth" optional = false python-versions = "*" -version = "0.2.3" +version = "0.2.4" [[package]] category = "main" @@ -1160,8 +1160,8 @@ cchardet = [ {file = "cchardet-2.1.6.tar.gz", hash = "sha256:b76afb2059ad69eab576949980a17413c1e9e5a5624abf9e43542d8853f146b3"}, ] certifi = [ - {file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"}, - {file = "certifi-2020.4.5.1.tar.gz", hash = "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"}, + {file = "certifi-2020.4.5.2-py2.py3-none-any.whl", hash = "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc"}, + {file = "certifi-2020.4.5.2.tar.gz", hash = "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1"}, ] cffi = [ {file = "cffi-1.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384"}, @@ -1270,8 +1270,8 @@ idna = [ {file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"}, ] importlib-metadata = [ - {file = "importlib_metadata-1.6.0-py2.py3-none-any.whl", hash = "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f"}, - {file = "importlib_metadata-1.6.0.tar.gz", hash = "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"}, + {file = "importlib_metadata-1.6.1-py2.py3-none-any.whl", hash = "sha256:15ec6c0fd909e893e3a08b3a7c76ecb149122fb14b7efe1199ddd4c7c57ea958"}, + {file = "importlib_metadata-1.6.1.tar.gz", hash = "sha256:0505dd08068cfec00f53a74a0ad927676d7757da81b7436a6eefe4c7cf75c545"}, ] incremental = [ {file = "incremental-17.5.0-py2.py3-none-any.whl", hash = "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f"}, @@ -1615,8 +1615,8 @@ w3lib = [ {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"}, ] wcwidth = [ - {file = "wcwidth-0.2.3-py2.py3-none-any.whl", hash = "sha256:980fbf4f3c196c0f329cdcd1e84c554d6a211f18e252e525a0cf4223154a41d6"}, - {file = "wcwidth-0.2.3.tar.gz", hash = "sha256:edbc2b718b4db6cdf393eefe3a420183947d6aa312505ce6754516f458ff8830"}, + {file = "wcwidth-0.2.4-py2.py3-none-any.whl", hash = "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f"}, + {file = "wcwidth-0.2.4.tar.gz", hash = "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f"}, ] werkzeug = [ {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"}, diff --git a/pyproject.toml b/pyproject.toml index 264f0c83..5632f312 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "1.0" description = "Web interface to track the trackers." authors = ["Raphaël Vinot "] license = "BSD-3-Clause" -repository = "https://github.com/CIRCL/lookyloo" +repository = "https://github.com/Lookyloo/lookyloo" readme = "README.md" @@ -30,8 +30,8 @@ stop = "bin/stop.py" [tool.poetry.dependencies] python = "^3.7" -scrapysplashwrapper = {git = "https://github.com/viper-framework/ScrapySplashWrapper.git"} -har2tree = {git = "https://github.com/viper-framework/har2tree.git"} +scrapysplashwrapper = {git = "https://github.com/Lookyloo/ScrapySplashWrapper.git"} +har2tree = {git = "https://github.com/Lookyloo/har2tree.git"} pysanejs = {git = "https://github.com/CIRCL/PySaneJS.git"} requests = "^2.22.0" flask = "^1.1.1"