chg: Cleanup, move to Lookyloo organization

.github/workflows/nosetests.yml

@@ -0,0 +1,22 @@
+name: Python application
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip poetry
+        poetry install
+    - name: Test with nosetests
+      run: |
+        poetry run mypy .

lookyloo/lookyloo.py

@@ -118,7 +118,7 @@ class Lookyloo():
                 for redirect in ct.redirects:
                     self.vt.url_lookup(redirect, force)
             else:
-                self.vt.url_lookup(ct.root_hartree.har.first_url, force)
+                self.vt.url_lookup(ct.root_hartree.har.root_url, force)
 
     def get_modules_responses(self, capture_dir: Path) -> Optional[Dict[str, Any]]:
         ct = self._load_pickle(capture_dir / 'tree.pickle')
@@ -132,7 +132,7 @@ class Lookyloo():
                 for redirect in ct.redirects:
                     to_return['vt'][redirect] = self.vt.get_url_lookup(redirect)
             else:
-                to_return['vt'][ct.root_hartree.har.first_url] = self.vt.get_url_lookup(ct.root_hartree.har.first_url)
+                to_return['vt'][ct.root_hartree.har.root_url] = self.vt.get_url_lookup(ct.root_hartree.har.root_url)
         return to_return
 
     def _set_capture_cache(self, capture_dir: Path, force: bool=False) -> None:
@@ -177,7 +177,7 @@ class Lookyloo():
         cache: Dict[str, Union[str, int]] = {'uuid': uuid,
                                              'title': har.initial_title,
                                              'timestamp': har.initial_start_time,
-                                             'url': har.first_url,
+                                             'url': har.root_url,
                                              'redirects': json.dumps(redirects),
                                              'incomplete_redirects': 1 if incomplete_redirects else 0}
         if (capture_dir / 'no_index').exists():  # If the folders claims anonymity

poetry.lock

@@ -132,7 +132,7 @@ description = "Python package for providing Mozilla's CA Bundle."
 name = "certifi"
 optional = false
 python-versions = "*"
-version = "2020.4.5.1"
+version = "2020.4.5.2"
 
 [[package]]
 category = "main"
@@ -306,7 +306,7 @@ publicsuffix2 = "^2.20191221"
 six = "^1.14.0"
 
 [package.source]
-reference = "6ee33972962351214413e07c33559691f1ae9b9d"
+reference = "93d301b19b78b81d690dba5123021cc990b6c961"
 type = "git"
 url = "https://github.com/viper-framework/har2tree.git"
 [[package]]
@@ -336,14 +336,14 @@ marker = "python_version < \"3.8\""
 name = "importlib-metadata"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
-version = "1.6.0"
+version = "1.6.1"
 
 [package.dependencies]
 zipp = ">=0.5"
 
 [package.extras]
 docs = ["sphinx", "rst.linker"]
-testing = ["packaging", "importlib-resources"]
+testing = ["packaging", "pep517", "importlib-resources (>=1.3)"]
 
 [[package]]
 category = "main"
@@ -870,7 +870,7 @@ scrapy = "^1.8.0"
 scrapy-splash = "^0.7.2"
 
 [package.source]
-reference = "3b56f1d4160383982c3316d4ba9ee479783a57a5"
+reference = "ae2cc94c77fba5fb40603c9714082fcd3ab9998f"
 type = "git"
 url = "https://github.com/viper-framework/ScrapySplashWrapper.git"
 [[package]]
@@ -1015,7 +1015,7 @@ description = "Measures the displayed width of unicode strings in a terminal"
 name = "wcwidth"
 optional = false
 python-versions = "*"
-version = "0.2.3"
+version = "0.2.4"
 
 [[package]]
 category = "main"
@@ -1160,8 +1160,8 @@ cchardet = [
     {file = "cchardet-2.1.6.tar.gz", hash = "sha256:b76afb2059ad69eab576949980a17413c1e9e5a5624abf9e43542d8853f146b3"},
 ]
 certifi = [
-    {file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"},
-    {file = "certifi-2020.4.5.1.tar.gz", hash = "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"},
+    {file = "certifi-2020.4.5.2-py2.py3-none-any.whl", hash = "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc"},
+    {file = "certifi-2020.4.5.2.tar.gz", hash = "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1"},
 ]
 cffi = [
     {file = "cffi-1.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384"},
@@ -1270,8 +1270,8 @@ idna = [
     {file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"},
 ]
 importlib-metadata = [
-    {file = "importlib_metadata-1.6.0-py2.py3-none-any.whl", hash = "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f"},
-    {file = "importlib_metadata-1.6.0.tar.gz", hash = "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"},
+    {file = "importlib_metadata-1.6.1-py2.py3-none-any.whl", hash = "sha256:15ec6c0fd909e893e3a08b3a7c76ecb149122fb14b7efe1199ddd4c7c57ea958"},
+    {file = "importlib_metadata-1.6.1.tar.gz", hash = "sha256:0505dd08068cfec00f53a74a0ad927676d7757da81b7436a6eefe4c7cf75c545"},
 ]
 incremental = [
     {file = "incremental-17.5.0-py2.py3-none-any.whl", hash = "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f"},
@@ -1615,8 +1615,8 @@ w3lib = [
     {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"},
 ]
 wcwidth = [
-    {file = "wcwidth-0.2.3-py2.py3-none-any.whl", hash = "sha256:980fbf4f3c196c0f329cdcd1e84c554d6a211f18e252e525a0cf4223154a41d6"},
-    {file = "wcwidth-0.2.3.tar.gz", hash = "sha256:edbc2b718b4db6cdf393eefe3a420183947d6aa312505ce6754516f458ff8830"},
+    {file = "wcwidth-0.2.4-py2.py3-none-any.whl", hash = "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f"},
+    {file = "wcwidth-0.2.4.tar.gz", hash = "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f"},
 ]
 werkzeug = [
     {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"},

pyproject.toml

@@ -4,7 +4,7 @@ version = "1.0"
 description = "Web interface to track the trackers."
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
-repository = "https://github.com/CIRCL/lookyloo"
+repository = "https://github.com/Lookyloo/lookyloo"
 
 readme = "README.md"
 
@@ -30,8 +30,8 @@ stop = "bin/stop.py"
 
 [tool.poetry.dependencies]
 python = "^3.7"
-scrapysplashwrapper = {git = "https://github.com/viper-framework/ScrapySplashWrapper.git"}
-har2tree = {git = "https://github.com/viper-framework/har2tree.git"}
+scrapysplashwrapper = {git = "https://github.com/Lookyloo/ScrapySplashWrapper.git"}
+har2tree = {git = "https://github.com/Lookyloo/har2tree.git"}
 pysanejs = {git = "https://github.com/CIRCL/PySaneJS.git"}
 requests = "^2.22.0"
 flask = "^1.1.1"