new: prepare s3fs support for archives

.github/workflows/instance_test.yml

@@ -13,7 +13,7 @@ jobs:
      strategy:
        fail-fast: false
        matrix:
-         python-version: ["3.8", "3.9", "3.10", "3.11"]
+         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
      steps:
       - uses: actions/checkout@v4

.github/workflows/mypy.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4

poetry.lock

@@ -1,5 +1,26 @@
 # This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
 
+[[package]]
+name = "aiobotocore"
+version = "2.7.0"
+description = "Async client for aws services using botocore and aiohttp"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "aiobotocore-2.7.0-py3-none-any.whl", hash = "sha256:aec605df77ce4635a0479b50fd849aa6b640900f7b295021ecca192e1140e551"},
+    {file = "aiobotocore-2.7.0.tar.gz", hash = "sha256:506591374cc0aee1bdf0ebe290560424a24af176dfe2ea7057fe1df97c4f0467"},
+]
+
+[package.dependencies]
+aiohttp = ">=3.7.4.post0,<4.0.0"
+aioitertools = ">=0.5.1,<1.0.0"
+botocore = ">=1.31.16,<1.31.65"
+wrapt = ">=1.10.10,<2.0.0"
+
+[package.extras]
+awscli = ["awscli (>=1.29.16,<1.29.65)"]
+boto3 = ["boto3 (>=1.28.16,<1.28.65)"]
+
 [[package]]
 name = "aiohttp"
 version = "3.8.6"
@@ -108,6 +129,20 @@ yarl = ">=1.0,<2.0"
 [package.extras]
 speedups = ["Brotli", "aiodns", "cchardet"]
 
+[[package]]
+name = "aioitertools"
+version = "0.11.0"
+description = "itertools and builtins for AsyncIO and mixed iterables"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"},
+    {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"},
+]
+
+[package.dependencies]
+typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""}
+
 [[package]]
 name = "aiosignal"
 version = "1.3.1"
@@ -277,6 +312,28 @@ files = [
 Flask = "*"
 WTForms = "*"
 
+[[package]]
+name = "botocore"
+version = "1.31.64"
+description = "Low-level, data-driven core of boto 3."
+optional = false
+python-versions = ">= 3.7"
+files = [
+    {file = "botocore-1.31.64-py3-none-any.whl", hash = "sha256:7b709310343a5b430ec9025b2e17c0bac6b16c05f1ac1d9521dece3f10c71bac"},
+    {file = "botocore-1.31.64.tar.gz", hash = "sha256:d8eb4b724ac437343359b318d73de0cfae0fecb24095827e56135b0ad6b44caf"},
+]
+
+[package.dependencies]
+jmespath = ">=0.7.1,<2.0.0"
+python-dateutil = ">=2.1,<3.0.0"
+urllib3 = [
+    {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""},
+    {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""},
+]
+
+[package.extras]
+crt = ["awscrt (==0.16.26)"]
+
 [[package]]
 name = "certifi"
 version = "2023.7.22"
@@ -821,6 +878,41 @@ files = [
     {file = "frozenlist-1.4.0.tar.gz", hash = "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251"},
 ]
 
+[[package]]
+name = "fsspec"
+version = "2023.10.0"
+description = "File-system specification"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "fsspec-2023.10.0-py3-none-any.whl", hash = "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"},
+    {file = "fsspec-2023.10.0.tar.gz", hash = "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5"},
+]
+
+[package.extras]
+abfs = ["adlfs"]
+adl = ["adlfs"]
+arrow = ["pyarrow (>=1)"]
+dask = ["dask", "distributed"]
+devel = ["pytest", "pytest-cov"]
+dropbox = ["dropbox", "dropboxdrivefs", "requests"]
+full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"]
+fuse = ["fusepy"]
+gcs = ["gcsfs"]
+git = ["pygit2"]
+github = ["requests"]
+gs = ["gcsfs"]
+gui = ["panel"]
+hdfs = ["pyarrow (>=1)"]
+http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"]
+libarchive = ["libarchive-c"]
+oci = ["ocifs"]
+s3 = ["s3fs"]
+sftp = ["paramiko"]
+smb = ["smbprotocol"]
+ssh = ["paramiko"]
+tqdm = ["tqdm"]
+
 [[package]]
 name = "future"
 version = "0.18.3"
@@ -1224,6 +1316,17 @@ MarkupSafe = ">=2.0"
 [package.extras]
 i18n = ["Babel (>=2.7)"]
 
+[[package]]
+name = "jmespath"
+version = "1.0.1"
+description = "JSON Matching Expressions"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
+    {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
+]
+
 [[package]]
 name = "jsonschema"
 version = "4.19.1"
@@ -2143,13 +2246,13 @@ docs = ["Sphinx (<7.2)", "Sphinx (>=7.2,<8.0)"]
 
 [[package]]
 name = "pylookyloo"
-version = "1.22.0"
+version = "1.22.1"
 description = "Python CLI and module for Lookyloo"
 optional = false
 python-versions = ">=3.8,<4.0"
 files = [
-    {file = "pylookyloo-1.22.0-py3-none-any.whl", hash = "sha256:c82708aaad65fd978ff2d4b924ecf26203f9f6605d8926742ce8c289f228150b"},
+    {file = "pylookyloo-1.22.1-py3-none-any.whl", hash = "sha256:abbe50476391873fc70c1c7621118a3d47379617dd7a9f5aecdd599f5b220806"},
-    {file = "pylookyloo-1.22.0.tar.gz", hash = "sha256:60b69a5a3c9b4edab98698a8fbe25e214d7ac19ece7ce3c856e4d8effa54829d"},
+    {file = "pylookyloo-1.22.1.tar.gz", hash = "sha256:e4159bd0d08ca4a6e97f03d9c0836c4cadd6ef40cfc9d8566821dc69eff959df"},
 ]
 
 [package.dependencies]
@@ -2630,6 +2733,26 @@ files = [
     {file = "rpds_py-0.10.6.tar.gz", hash = "sha256:4ce5a708d65a8dbf3748d2474b580d606b1b9f91b5c6ab2a316e0b0cf7a4ba50"},
 ]
 
+[[package]]
+name = "s3fs"
+version = "2023.10.0"
+description = "Convenient Filesystem interface over S3"
+optional = false
+python-versions = ">= 3.8"
+files = [
+    {file = "s3fs-2023.10.0-py3-none-any.whl", hash = "sha256:3df68ff4f5f70c3338219a66df92e91fd15c6b78d0f559e57f617dfdd49feb41"},
+    {file = "s3fs-2023.10.0.tar.gz", hash = "sha256:c40f238ccc9fefff3f6d09d4b5762abd6c913ba42e1a328976b54d038901b835"},
+]
+
+[package.dependencies]
+aiobotocore = ">=2.7.0,<2.8.0"
+aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1"
+fsspec = "2023.10.0"
+
+[package.extras]
+awscli = ["aiobotocore[awscli] (>=2.7.0,<2.8.0)"]
+boto3 = ["aiobotocore[boto3] (>=2.7.0,<2.8.0)"]
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -2840,6 +2963,20 @@ files = [
 cryptography = ">=35.0.0"
 types-pyOpenSSL = "*"
 
+[[package]]
+name = "types-requests"
+version = "2.30.0.0"
+description = "Typing stubs for requests"
+optional = false
+python-versions = "*"
+files = [
+    {file = "types-requests-2.30.0.0.tar.gz", hash = "sha256:dec781054324a70ba64430ae9e62e7e9c8e4618c185a5cb3f87a6738251b5a31"},
+    {file = "types_requests-2.30.0.0-py3-none-any.whl", hash = "sha256:c6cf08e120ca9f0dc4fa4e32c3f953c3fba222bcc1db6b97695bce8da1ba9864"},
+]
+
+[package.dependencies]
+types-urllib3 = "*"
+
 [[package]]
 name = "types-requests"
 version = "2.31.0.10"
@@ -2854,6 +2991,17 @@ files = [
 [package.dependencies]
 urllib3 = ">=2"
 
+[[package]]
+name = "types-urllib3"
+version = "1.26.25.14"
+description = "Typing stubs for urllib3"
+optional = false
+python-versions = "*"
+files = [
+    {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"},
+    {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"},
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.8.0"
@@ -2905,6 +3053,22 @@ files = [
     {file = "ua_parser-0.18.0-py2.py3-none-any.whl", hash = "sha256:9d94ac3a80bcb0166823956a779186c746b50ea4c9fd9bf30fdb758553c38950"},
 ]
 
+[[package]]
+name = "urllib3"
+version = "1.26.18"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+files = [
+    {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"},
+    {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"},
+]
+
+[package.extras]
+brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
+secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
 [[package]]
 name = "urllib3"
 version = "2.0.7"
@@ -3183,5 +3347,5 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 
 [metadata]
 lock-version = "2.0"
-python-versions = ">=3.8,<3.12"
+python-versions = ">=3.8,<3.13"
-content-hash = "9400db3325e19346e80949f35913552685dd24288dc88d23a249d1b24c1dab75"
+content-hash = "ad66d5e2270a0f9634e6f4091a9bdb976241325e9ac42f9b0f36f3753dced049"

pyproject.toml

@@ -35,7 +35,7 @@ start_website = "bin.start_website:main"
 
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.12"
+python = ">=3.8,<3.13"
 requests = "^2.31.0"
 flask = "^2.3.3"
 gunicorn = "^21.2.0"
@@ -47,7 +47,7 @@ defang = "^0.5.3"
 vt-py = "^0.17.5"
 pyeupi = "^1.3.0"
 pysanejs = "^2.0.2"
-pylookyloo = "^1.22.0"
+pylookyloo = "^1.22.1"
 dnspython = "^2.4.2"
 pytaxonomies = "^1.5.0"
 pymisp = {version = "^2.4.176", extras = ["url", "fileobjects"]}
@@ -74,6 +74,11 @@ chardet = "^5.2.0"
 pysecuritytxt = "^1.2.0"
 pylookyloomonitoring = "^1.1.2"
 pytz = {"version" = "^2023.3.post1", python = "<3.9"}
+s3fs = "^2023.10.0"
+urllib3 = [
+    {version = "<2", python = "<3.10"},
+    {version = "^2.0.7", python = ">=3.10"}
+]
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.6.1"
@@ -81,8 +86,11 @@ ipython = [
     {version = "<8.13.0", python = "<3.9"},
     {version = "^8.13.0", python = ">=3.9"}
 ]
+types-requests = [
+    {version = "<2.31", python = "<3.10"},
+    {version = "^2.31.0.10", python = ">=3.10"}
+]
 types-redis = {version = "^4.6.0.7"}
-types-requests = "^2.31.0.10"
 types-pkg-resources = "^0.1.3"
 types-Deprecated = "^1.2.9.3"
 types-python-dateutil = "^2.8.19.14"
@@ -95,7 +103,7 @@ requires = ["poetry_core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.mypy]
-python_version = 3.8
+python_version = "3.10"
 check_untyped_defs = true
 ignore_errors = false
 ignore_missing_imports = false