From bf9ff87dacb48633477dbfeff76e56d58308cdd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 22 Oct 2023 00:57:46 +0200 Subject: [PATCH] new: prepare s3fs support for archives --- .github/workflows/instance_test.yml | 2 +- .github/workflows/mypy.yml | 2 +- poetry.lock | 174 +++++++++++++++++++++++++++- pyproject.toml | 16 ++- 4 files changed, 183 insertions(+), 11 deletions(-) diff --git a/.github/workflows/instance_test.yml b/.github/workflows/instance_test.yml index 6324bdaa..8be9dfe0 100644 --- a/.github/workflows/instance_test.yml +++ b/.github/workflows/instance_test.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index c7c7e286..f3bedfe2 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/poetry.lock b/poetry.lock index f00693b3..6ad1db23 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,26 @@ # This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +[[package]] +name = "aiobotocore" +version = "2.7.0" +description = "Async client for aws services using botocore and aiohttp" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiobotocore-2.7.0-py3-none-any.whl", hash = "sha256:aec605df77ce4635a0479b50fd849aa6b640900f7b295021ecca192e1140e551"}, + {file = "aiobotocore-2.7.0.tar.gz", hash = "sha256:506591374cc0aee1bdf0ebe290560424a24af176dfe2ea7057fe1df97c4f0467"}, +] + +[package.dependencies] +aiohttp = ">=3.7.4.post0,<4.0.0" +aioitertools = ">=0.5.1,<1.0.0" +botocore = ">=1.31.16,<1.31.65" +wrapt = ">=1.10.10,<2.0.0" + +[package.extras] +awscli = ["awscli (>=1.29.16,<1.29.65)"] +boto3 = ["boto3 (>=1.28.16,<1.28.65)"] + [[package]] name = "aiohttp" version = "3.8.6" @@ -108,6 +129,20 @@ yarl = ">=1.0,<2.0" [package.extras] speedups = ["Brotli", "aiodns", "cchardet"] +[[package]] +name = "aioitertools" +version = "0.11.0" +description = "itertools and builtins for AsyncIO and mixed iterables" +optional = false +python-versions = ">=3.6" +files = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] + +[package.dependencies] +typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} + [[package]] name = "aiosignal" version = "1.3.1" @@ -277,6 +312,28 @@ files = [ Flask = "*" WTForms = "*" +[[package]] +name = "botocore" +version = "1.31.64" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">= 3.7" +files = [ + {file = "botocore-1.31.64-py3-none-any.whl", hash = "sha256:7b709310343a5b430ec9025b2e17c0bac6b16c05f1ac1d9521dece3f10c71bac"}, + {file = "botocore-1.31.64.tar.gz", hash = "sha256:d8eb4b724ac437343359b318d73de0cfae0fecb24095827e56135b0ad6b44caf"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = [ + {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, + {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}, +] + +[package.extras] +crt = ["awscrt (==0.16.26)"] + [[package]] name = "certifi" version = "2023.7.22" @@ -821,6 +878,41 @@ files = [ {file = "frozenlist-1.4.0.tar.gz", hash = "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251"}, ] +[[package]] +name = "fsspec" +version = "2023.10.0" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2023.10.0-py3-none-any.whl", hash = "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"}, + {file = "fsspec-2023.10.0.tar.gz", hash = "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + [[package]] name = "future" version = "0.18.3" @@ -1224,6 +1316,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "jsonschema" version = "4.19.1" @@ -2143,13 +2246,13 @@ docs = ["Sphinx (<7.2)", "Sphinx (>=7.2,<8.0)"] [[package]] name = "pylookyloo" -version = "1.22.0" +version = "1.22.1" description = "Python CLI and module for Lookyloo" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "pylookyloo-1.22.0-py3-none-any.whl", hash = "sha256:c82708aaad65fd978ff2d4b924ecf26203f9f6605d8926742ce8c289f228150b"}, - {file = "pylookyloo-1.22.0.tar.gz", hash = "sha256:60b69a5a3c9b4edab98698a8fbe25e214d7ac19ece7ce3c856e4d8effa54829d"}, + {file = "pylookyloo-1.22.1-py3-none-any.whl", hash = "sha256:abbe50476391873fc70c1c7621118a3d47379617dd7a9f5aecdd599f5b220806"}, + {file = "pylookyloo-1.22.1.tar.gz", hash = "sha256:e4159bd0d08ca4a6e97f03d9c0836c4cadd6ef40cfc9d8566821dc69eff959df"}, ] [package.dependencies] @@ -2630,6 +2733,26 @@ files = [ {file = "rpds_py-0.10.6.tar.gz", hash = "sha256:4ce5a708d65a8dbf3748d2474b580d606b1b9f91b5c6ab2a316e0b0cf7a4ba50"}, ] +[[package]] +name = "s3fs" +version = "2023.10.0" +description = "Convenient Filesystem interface over S3" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "s3fs-2023.10.0-py3-none-any.whl", hash = "sha256:3df68ff4f5f70c3338219a66df92e91fd15c6b78d0f559e57f617dfdd49feb41"}, + {file = "s3fs-2023.10.0.tar.gz", hash = "sha256:c40f238ccc9fefff3f6d09d4b5762abd6c913ba42e1a328976b54d038901b835"}, +] + +[package.dependencies] +aiobotocore = ">=2.7.0,<2.8.0" +aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" +fsspec = "2023.10.0" + +[package.extras] +awscli = ["aiobotocore[awscli] (>=2.7.0,<2.8.0)"] +boto3 = ["aiobotocore[boto3] (>=2.7.0,<2.8.0)"] + [[package]] name = "six" version = "1.16.0" @@ -2840,6 +2963,20 @@ files = [ cryptography = ">=35.0.0" types-pyOpenSSL = "*" +[[package]] +name = "types-requests" +version = "2.30.0.0" +description = "Typing stubs for requests" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.30.0.0.tar.gz", hash = "sha256:dec781054324a70ba64430ae9e62e7e9c8e4618c185a5cb3f87a6738251b5a31"}, + {file = "types_requests-2.30.0.0-py3-none-any.whl", hash = "sha256:c6cf08e120ca9f0dc4fa4e32c3f953c3fba222bcc1db6b97695bce8da1ba9864"}, +] + +[package.dependencies] +types-urllib3 = "*" + [[package]] name = "types-requests" version = "2.31.0.10" @@ -2854,6 +2991,17 @@ files = [ [package.dependencies] urllib3 = ">=2" +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] + [[package]] name = "typing-extensions" version = "4.8.0" @@ -2905,6 +3053,22 @@ files = [ {file = "ua_parser-0.18.0-py2.py3-none-any.whl", hash = "sha256:9d94ac3a80bcb0166823956a779186c746b50ea4c9fd9bf30fdb758553c38950"}, ] +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "urllib3" version = "2.0.7" @@ -3183,5 +3347,5 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" -python-versions = ">=3.8,<3.12" -content-hash = "9400db3325e19346e80949f35913552685dd24288dc88d23a249d1b24c1dab75" +python-versions = ">=3.8,<3.13" +content-hash = "ad66d5e2270a0f9634e6f4091a9bdb976241325e9ac42f9b0f36f3753dced049" diff --git a/pyproject.toml b/pyproject.toml index 89d401ce..ab7d255d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ start_website = "bin.start_website:main" [tool.poetry.dependencies] -python = ">=3.8,<3.12" +python = ">=3.8,<3.13" requests = "^2.31.0" flask = "^2.3.3" gunicorn = "^21.2.0" @@ -47,7 +47,7 @@ defang = "^0.5.3" vt-py = "^0.17.5" pyeupi = "^1.3.0" pysanejs = "^2.0.2" -pylookyloo = "^1.22.0" +pylookyloo = "^1.22.1" dnspython = "^2.4.2" pytaxonomies = "^1.5.0" pymisp = {version = "^2.4.176", extras = ["url", "fileobjects"]} @@ -74,6 +74,11 @@ chardet = "^5.2.0" pysecuritytxt = "^1.2.0" pylookyloomonitoring = "^1.1.2" pytz = {"version" = "^2023.3.post1", python = "<3.9"} +s3fs = "^2023.10.0" +urllib3 = [ + {version = "<2", python = "<3.10"}, + {version = "^2.0.7", python = ">=3.10"} +] [tool.poetry.group.dev.dependencies] mypy = "^1.6.1" @@ -81,8 +86,11 @@ ipython = [ {version = "<8.13.0", python = "<3.9"}, {version = "^8.13.0", python = ">=3.9"} ] +types-requests = [ + {version = "<2.31", python = "<3.10"}, + {version = "^2.31.0.10", python = ">=3.10"} +] types-redis = {version = "^4.6.0.7"} -types-requests = "^2.31.0.10" types-pkg-resources = "^0.1.3" types-Deprecated = "^1.2.9.3" types-python-dateutil = "^2.8.19.14" @@ -95,7 +103,7 @@ requires = ["poetry_core"] build-backend = "poetry.core.masonry.api" [tool.mypy] -python_version = 3.8 +python_version = "3.10" check_untyped_defs = true ignore_errors = false ignore_missing_imports = false