mirror of https://github.com/MISP/PyMISP
Merge remote-tracking branch 'MISP/main' into main
Christophe Vandeplas
commit
e4919b6bbf
|
|
@ -88,7 +88,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>
|
|||
|
||||
[[package]]
|
||||
name = "babel"
|
||||
version = "2.10.1"
|
||||
version = "2.10.3"
|
||||
description = "Internationalization utilities"
|
||||
category = "main"
|
||||
optional = false
|
||||
|
|
@ -168,7 +168,7 @@ cffi = ">=1.0.0"
|
|||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2022.5.18.1"
|
||||
version = "2022.6.15"
|
||||
description = "Python package for providing Mozilla's CA Bundle."
|
||||
category = "main"
|
||||
optional = false
|
||||
|
|
@ -344,7 +344,7 @@ python-versions = ">=3.6"
|
|||
|
||||
[[package]]
|
||||
name = "extract-msg"
|
||||
version = "0.33.0"
|
||||
version = "0.34.3"
|
||||
description = "Extracts emails and attachments saved in Microsoft Outlook's .msg files"
|
||||
category = "main"
|
||||
optional = true
|
||||
|
|
@ -352,10 +352,10 @@ python-versions = "*"
|
|||
|
||||
[package.dependencies]
|
||||
beautifulsoup4 = ">=4.10.0"
|
||||
chardet = ">=4.0.0"
|
||||
compressed-rtf = ">=1.0.6"
|
||||
ebcdic = ">=1.1.1"
|
||||
imapclient = ">=2.1.0"
|
||||
mailbits = ">=0.2.1"
|
||||
olefile = ">=0.46"
|
||||
RTFDE = ">=0.0.2"
|
||||
tzlocal = ">=2.1"
|
||||
|
|
@ -421,7 +421,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
|
|||
|
||||
[[package]]
|
||||
name = "importlib-resources"
|
||||
version = "5.7.1"
|
||||
version = "5.8.0"
|
||||
description = "Read resources from Python packages"
|
||||
category = "main"
|
||||
optional = false
|
||||
|
|
@ -444,7 +444,7 @@ python-versions = "*"
|
|||
|
||||
[[package]]
|
||||
name = "ipykernel"
|
||||
version = "6.13.1"
|
||||
version = "6.15.0"
|
||||
description = "IPython Kernel for Jupyter"
|
||||
category = "dev"
|
||||
optional = false
|
||||
|
|
@ -459,6 +459,7 @@ matplotlib-inline = ">=0.1"
|
|||
nest-asyncio = "*"
|
||||
packaging = "*"
|
||||
psutil = "*"
|
||||
pyzmq = ">=17"
|
||||
tornado = ">=6.1"
|
||||
traitlets = ">=5.1.0"
|
||||
|
||||
|
|
@ -702,18 +703,6 @@ category = "main"
|
|||
optional = true
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[[package]]
|
||||
name = "mailbits"
|
||||
version = "0.2.1"
|
||||
description = "Assorted e-mail utility functions"
|
||||
category = "main"
|
||||
optional = true
|
||||
python-versions = "~=3.6"
|
||||
|
||||
[package.dependencies]
|
||||
attrs = ">=18.1"
|
||||
typing-extensions = {version = "*", markers = "python_version < \"3.8\""}
|
||||
|
||||
[[package]]
|
||||
name = "markupsafe"
|
||||
version = "2.1.1"
|
||||
|
|
@ -1073,6 +1062,18 @@ category = "dev"
|
|||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[[package]]
|
||||
name = "publicsuffixlist"
|
||||
version = "0.7.13"
|
||||
description = "publicsuffixlist implement"
|
||||
category = "main"
|
||||
optional = true
|
||||
python-versions = ">=2.6"
|
||||
|
||||
[package.extras]
|
||||
readme = ["pandoc"]
|
||||
update = ["requests"]
|
||||
|
||||
[[package]]
|
||||
name = "py"
|
||||
version = "1.11.0"
|
||||
|
|
@ -1262,20 +1263,20 @@ rlpycairo = ["rlPyCairo (>=0.0.5)"]
|
|||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
version = "2.27.1"
|
||||
version = "2.28.0"
|
||||
description = "Python HTTP for Humans."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
|
||||
python-versions = ">=3.7, <4"
|
||||
|
||||
[package.dependencies]
|
||||
certifi = ">=2017.4.17"
|
||||
charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""}
|
||||
idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""}
|
||||
charset-normalizer = ">=2.0.0,<2.1.0"
|
||||
idna = ">=2.5,<4"
|
||||
urllib3 = ">=1.21.1,<1.27"
|
||||
|
||||
[package.extras]
|
||||
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
|
||||
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
|
||||
use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1389,7 +1390,7 @@ test = ["pytest (>=4.6)", "html5lib", "cython", "typed-ast"]
|
|||
|
||||
[[package]]
|
||||
name = "sphinx-autodoc-typehints"
|
||||
version = "1.18.2"
|
||||
version = "1.18.3"
|
||||
description = "Type hints (PEP 484) support for the Sphinx autodoc extension"
|
||||
category = "main"
|
||||
optional = true
|
||||
|
|
@ -1399,7 +1400,7 @@ python-versions = ">=3.7"
|
|||
Sphinx = ">=4.5"
|
||||
|
||||
[package.extras]
|
||||
testing = ["covdefaults (>=2.2)", "coverage (>=6.3)", "diff-cover (>=6.4)", "nptyping (>=2.1.1)", "pytest (>=7.1)", "pytest-cov (>=3)", "sphobjinv (>=2)", "typing-extensions (>=4.1)"]
|
||||
testing = ["covdefaults (>=2.2)", "coverage (>=6.3)", "diff-cover (>=6.4)", "nptyping (>=2.1.2)", "pytest (>=7.1)", "pytest-cov (>=3)", "sphobjinv (>=2)", "typing-extensions (>=4.1)"]
|
||||
type_comments = ["typed-ast (>=1.5.2)"]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1589,7 +1590,7 @@ python-versions = "*"
|
|||
|
||||
[[package]]
|
||||
name = "types-redis"
|
||||
version = "4.2.6"
|
||||
version = "4.2.7"
|
||||
description = "Typing stubs for redis"
|
||||
category = "dev"
|
||||
optional = false
|
||||
|
|
@ -1756,7 +1757,7 @@ virustotal = ["validators"]
|
|||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.7"
|
||||
content-hash = "dd80e16b2b6d1ec57659e4ad1f9e9cd253c06c9ecb83ab5cf262f0f334c3b9ab"
|
||||
content-hash = "f1626e907825ccc8f00cffa0c0603359cb7a5d3967912fe7f7ed35e520cf89d7"
|
||||
|
||||
[metadata.files]
|
||||
alabaster = [
|
||||
|
|
@ -1807,8 +1808,8 @@ attrs = [
|
|||
{file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"},
|
||||
]
|
||||
babel = [
|
||||
{file = "Babel-2.10.1-py3-none-any.whl", hash = "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2"},
|
||||
{file = "Babel-2.10.1.tar.gz", hash = "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13"},
|
||||
{file = "Babel-2.10.3-py3-none-any.whl", hash = "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"},
|
||||
{file = "Babel-2.10.3.tar.gz", hash = "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51"},
|
||||
]
|
||||
backcall = [
|
||||
{file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
|
||||
|
|
@ -1937,8 +1938,8 @@ brotlicffi = [
|
|||
{file = "brotlicffi-1.0.9.2.tar.gz", hash = "sha256:0c248a68129d8fc6a217767406c731e498c3e19a7be05ea0a90c3c86637b7d96"},
|
||||
]
|
||||
certifi = [
|
||||
{file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"},
|
||||
{file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"},
|
||||
{file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"},
|
||||
{file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"},
|
||||
]
|
||||
cffi = [
|
||||
{file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"},
|
||||
|
|
@ -2130,8 +2131,8 @@ entrypoints = [
|
|||
{file = "entrypoints-0.4.tar.gz", hash = "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4"},
|
||||
]
|
||||
extract-msg = [
|
||||
{file = "extract_msg-0.33.0-py2.py3-none-any.whl", hash = "sha256:356164eed7619faeddd7e94148aad56d1faa78e6ea03fa09a5624f9e47becc21"},
|
||||
{file = "extract_msg-0.33.0.tar.gz", hash = "sha256:fb6fa8c55ff73a4d372af55878ee405b955f237a2486de65d2cee1172932e4fa"},
|
||||
{file = "extract_msg-0.34.3-py2.py3-none-any.whl", hash = "sha256:2969dbffdf7077ecc48d099f4698f5d35ef7e0cd1d52bf98b8535ce88e6a1e34"},
|
||||
{file = "extract_msg-0.34.3.tar.gz", hash = "sha256:8817935147c47eacc2612eb884c6922fa8da3a25fb6d86f696224de356714bdb"},
|
||||
]
|
||||
fastjsonschema = [
|
||||
{file = "fastjsonschema-2.15.3-py3-none-any.whl", hash = "sha256:ddb0b1d8243e6e3abb822bd14e447a89f4ab7439342912d590444831fa00b6a0"},
|
||||
|
|
@ -2154,16 +2155,16 @@ importlib-metadata = [
|
|||
{file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"},
|
||||
]
|
||||
importlib-resources = [
|
||||
{file = "importlib_resources-5.7.1-py3-none-any.whl", hash = "sha256:e447dc01619b1e951286f3929be820029d48c75eb25d265c28b92a16548212b8"},
|
||||
{file = "importlib_resources-5.7.1.tar.gz", hash = "sha256:b6062987dfc51f0fcb809187cffbd60f35df7acb4589091f154214af6d0d49d3"},
|
||||
{file = "importlib_resources-5.8.0-py3-none-any.whl", hash = "sha256:7952325ffd516c05a8ad0858c74dff2c3343f136fe66a6002b2623dd1d43f223"},
|
||||
{file = "importlib_resources-5.8.0.tar.gz", hash = "sha256:568c9f16cb204f9decc8d6d24a572eeea27dacbb4cee9e6b03a8025736769751"},
|
||||
]
|
||||
iniconfig = [
|
||||
{file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"},
|
||||
{file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"},
|
||||
]
|
||||
ipykernel = [
|
||||
{file = "ipykernel-6.13.1-py3-none-any.whl", hash = "sha256:fedc79bebd8a438162d056e0c7662d5ac8a47d1f6ef33a702e8460248dc4517f"},
|
||||
{file = "ipykernel-6.13.1.tar.gz", hash = "sha256:6f42070a5d87ecbf4a2fc27a7faae8d690fd3794825a090ddf6b00b9678a5b69"},
|
||||
{file = "ipykernel-6.15.0-py3-none-any.whl", hash = "sha256:b9ed519a29eb819eb82e87e0d3754088237b233e5c647b8bb0ff23c8c70ed16f"},
|
||||
{file = "ipykernel-6.15.0.tar.gz", hash = "sha256:b59f9d9672c3a483494bb75915a2b315e78b833a38b039b1ee36dc28683f0d89"},
|
||||
]
|
||||
ipython = [
|
||||
{file = "ipython-7.34.0-py3-none-any.whl", hash = "sha256:c175d2440a1caff76116eb719d40538fbb316e214eda85c5515c303aacbfb23e"},
|
||||
|
|
@ -2242,10 +2243,6 @@ lief = [
|
|||
{file = "lief-0.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:960a2da9f28c8d5dba753bb9ab77e26b3c6ff9b9658918be95650ceb8ee91e68"},
|
||||
{file = "lief-0.12.1.zip", hash = "sha256:4ff4ccfae2e1ee4ccba2b5556027dbb56282b8a973c5835c5b597e8b7b664416"},
|
||||
]
|
||||
mailbits = [
|
||||
{file = "mailbits-0.2.1-py3-none-any.whl", hash = "sha256:04c06b036bba93067d96b08288780ae7002ba604ac7b205bc55dca52474fc3e2"},
|
||||
{file = "mailbits-0.2.1.tar.gz", hash = "sha256:eb53610e01611a95d2ae46559e00d862d907c776e88034dd53020a53baac21d1"},
|
||||
]
|
||||
markupsafe = [
|
||||
{file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"},
|
||||
{file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"},
|
||||
|
|
@ -2478,6 +2475,10 @@ ptyprocess = [
|
|||
{file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"},
|
||||
{file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"},
|
||||
]
|
||||
publicsuffixlist = [
|
||||
{file = "publicsuffixlist-0.7.13-py2.py3-none-any.whl", hash = "sha256:60abb0720c00b635149a7654445fb700822fcbf5187be8f51f7be174a291560e"},
|
||||
{file = "publicsuffixlist-0.7.13.tar.gz", hash = "sha256:07409a5821a1f662b694c7390bdd50539528eb9d1e626811ca5e1447366b185f"},
|
||||
]
|
||||
py = [
|
||||
{file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
|
||||
{file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
|
||||
|
|
@ -2683,8 +2684,8 @@ reportlab = [
|
|||
{file = "reportlab-3.6.10.tar.gz", hash = "sha256:bf8cba95a2d5cf731e8b74c92b4f07d79ef286a2a78b617300e37e51cf955cb2"},
|
||||
]
|
||||
requests = [
|
||||
{file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
|
||||
{file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
|
||||
{file = "requests-2.28.0-py3-none-any.whl", hash = "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f"},
|
||||
{file = "requests-2.28.0.tar.gz", hash = "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b"},
|
||||
]
|
||||
requests-mock = [
|
||||
{file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"},
|
||||
|
|
@ -2719,8 +2720,8 @@ sphinx = [
|
|||
{file = "Sphinx-5.0.1.tar.gz", hash = "sha256:f4da1187785a5bc7312cc271b0e867a93946c319d106363e102936a3d9857306"},
|
||||
]
|
||||
sphinx-autodoc-typehints = [
|
||||
{file = "sphinx_autodoc_typehints-1.18.2-py3-none-any.whl", hash = "sha256:89b7a16c2642dd5580c6f97503252e0c5d82b8aced0cd2c896f6209ad748bb18"},
|
||||
{file = "sphinx_autodoc_typehints-1.18.2.tar.gz", hash = "sha256:6ba02ecced60ba640f891301c863097468560d23df80afbd69b2ddcde261be2d"},
|
||||
{file = "sphinx_autodoc_typehints-1.18.3-py3-none-any.whl", hash = "sha256:20294de2a818bda04953c5cb302ec5af46138c81980ad9efa6d8fc1fc4242518"},
|
||||
{file = "sphinx_autodoc_typehints-1.18.3.tar.gz", hash = "sha256:c04d8f8d70e988960e25b206af39a90df84e7e2c085bb24e123bc3684021b313"},
|
||||
]
|
||||
sphinxcontrib-applehelp = [
|
||||
{file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"},
|
||||
|
|
@ -2852,8 +2853,8 @@ types-python-dateutil = [
|
|||
{file = "types_python_dateutil-2.8.17-py3-none-any.whl", hash = "sha256:0be7435b4d382d1cd00b8c55a8a90f4e515aaad8a96f8f0bc20c22df046792e5"},
|
||||
]
|
||||
types-redis = [
|
||||
{file = "types-redis-4.2.6.tar.gz", hash = "sha256:d6adc77185cf40b300816767a64c0ee9ee0b21dc174e8e5c23b7e83d43189cb8"},
|
||||
{file = "types_redis-4.2.6-py3-none-any.whl", hash = "sha256:1136af954ade0be33b487f440c8cbcbee29f089a83e685484ec91f363c6c69fe"},
|
||||
{file = "types-redis-4.2.7.tar.gz", hash = "sha256:eece573e8dfd51238fae1df84d3602335fbcbd3ba8b064081cc0ff8ec1f058a1"},
|
||||
{file = "types_redis-4.2.7-py3-none-any.whl", hash = "sha256:1362d1dd69e8d6688a192c36c29b7ee61d67196df28ff40610013353023571bd"},
|
||||
]
|
||||
types-requests = [
|
||||
{file = "types-requests-2.27.30.tar.gz", hash = "sha256:ca8d7cc549c3d10dbcb3c69c1b53e3ffd1270089c1001a65c1e9e1017eb5e704"},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,191 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
import idna
|
||||
from publicsuffixlist import PublicSuffixList # type: ignore
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
|
||||
class UrlNotDecoded(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class PSLFaup(object):
|
||||
"""
|
||||
Fake Faup Python Library using PSL for Windows support
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.decoded = False
|
||||
self.psl = PublicSuffixList()
|
||||
self._url = None
|
||||
self._retval = {}
|
||||
self.ip_as_host = False
|
||||
|
||||
def _clear(self):
|
||||
self.decoded = False
|
||||
self._url = None
|
||||
self._retval = {}
|
||||
self.ip_as_host = False
|
||||
|
||||
def decode(self, url) -> None:
|
||||
"""
|
||||
This function creates a dict of all the url fields.
|
||||
:param url: The URL to normalize
|
||||
"""
|
||||
self._clear()
|
||||
if isinstance(url, bytes) and b'//' not in url[:10]:
|
||||
url = b'//' + url
|
||||
elif '//' not in url[:10]:
|
||||
url = '//' + url
|
||||
self._url = urlparse(url)
|
||||
|
||||
self.ip_as_host = False
|
||||
hostname = _ensure_str(self._url.hostname)
|
||||
try:
|
||||
ipv4_bytes = socket.inet_aton(_ensure_str(hostname))
|
||||
ipv4 = ipaddress.IPv4Address(ipv4_bytes)
|
||||
self.ip_as_host = ipv4.compressed
|
||||
except (OSError, ValueError):
|
||||
try:
|
||||
addr, _, _ = hostname.partition('%')
|
||||
ipv6 = ipaddress.IPv6Address(addr)
|
||||
self.ip_as_host = ipv6.compressed
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
self.decoded = True
|
||||
self._retval = {}
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
netloc = self.get_host() + ('' if self.get_port() is None else ':{}'.format(self.get_port()))
|
||||
return _ensure_bytes(
|
||||
urlunparse(
|
||||
(self.get_scheme(), netloc, self.get_resource_path(),
|
||||
'', self.get_query_string(), self.get_fragment(),)
|
||||
)
|
||||
)
|
||||
|
||||
def get_scheme(self):
|
||||
"""
|
||||
Get the scheme of the url given in the decode function
|
||||
:returns: The URL scheme
|
||||
"""
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
return _ensure_str(self._url.scheme)
|
||||
|
||||
def get_credential(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
if self._url.password:
|
||||
return _ensure_str(self._url.username) + ':' + _ensure_str(self._url.password)
|
||||
if self._url.username:
|
||||
return _ensure_str(self._url.username)
|
||||
|
||||
def get_subdomain(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
if self.get_host() is not None and not self.ip_as_host:
|
||||
if self.get_domain() in self.get_host():
|
||||
return self.get_host().rsplit(self.get_domain(), 1)[0].rstrip('.') or None
|
||||
|
||||
def get_domain(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
if self.get_host() is not None and not self.ip_as_host:
|
||||
return self.psl.privatesuffix(self.get_host())
|
||||
|
||||
def get_domain_without_tld(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
if self.get_tld() is not None and not self.ip_as_host:
|
||||
return self.get_domain().rsplit(self.get_tld(), 1)[0].rstrip('.')
|
||||
|
||||
def get_host(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
if self._url.hostname is None:
|
||||
return None
|
||||
elif self._url.hostname.isascii():
|
||||
return _ensure_str(self._url.hostname)
|
||||
else:
|
||||
return _ensure_str(idna.encode(self._url.hostname, uts46=True))
|
||||
|
||||
def get_unicode_host(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
if not self.ip_as_host:
|
||||
return idna.decode(self.get_host(), uts46=True)
|
||||
|
||||
def get_tld(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
if self.get_host() is not None and not self.ip_as_host:
|
||||
return self.psl.publicsuffix(self.get_host())
|
||||
|
||||
def get_port(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
return self._url.port
|
||||
|
||||
def get_resource_path(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
return _ensure_str(self._url.path)
|
||||
|
||||
def get_query_string(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
return _ensure_str(self._url.query)
|
||||
|
||||
def get_fragment(self):
|
||||
if not self.decoded:
|
||||
raise UrlNotDecoded("You must call faup.decode() first")
|
||||
|
||||
return _ensure_str(self._url.fragment)
|
||||
|
||||
def get(self):
|
||||
self._retval["scheme"] = self.get_scheme()
|
||||
self._retval["tld"] = self.get_tld()
|
||||
self._retval["domain"] = self.get_domain()
|
||||
self._retval["domain_without_tld"] = self.get_domain_without_tld()
|
||||
self._retval["subdomain"] = self.get_subdomain()
|
||||
self._retval["host"] = self.get_host()
|
||||
self._retval["port"] = self.get_port()
|
||||
self._retval["resource_path"] = self.get_resource_path()
|
||||
self._retval["query_string"] = self.get_query_string()
|
||||
self._retval["fragment"] = self.get_fragment()
|
||||
self._retval["url"] = self.url
|
||||
return self._retval
|
||||
|
||||
|
||||
def _ensure_bytes(binary) -> bytes:
|
||||
if isinstance(binary, bytes):
|
||||
return binary
|
||||
else:
|
||||
return binary.encode('utf-8')
|
||||
|
||||
|
||||
def _ensure_str(string) -> str:
|
||||
if isinstance(string, str):
|
||||
return string
|
||||
else:
|
||||
return string.decode('utf-8')
|
||||
|
|
@ -3,9 +3,13 @@
|
|||
|
||||
from .abstractgenerator import AbstractMISPObjectGenerator
|
||||
import logging
|
||||
from pyfaup.faup import Faup # type: ignore
|
||||
from urllib.parse import unquote_plus
|
||||
|
||||
try:
|
||||
from pyfaup.faup import Faup # type: ignore
|
||||
except (OSError, ImportError):
|
||||
from ._psl_faup import PSLFaup as Faup
|
||||
|
||||
logger = logging.getLogger('pymisp')
|
||||
|
||||
faup = Faup()
|
||||
|
|
@ -13,8 +17,9 @@ faup = Faup()
|
|||
|
||||
class URLObject(AbstractMISPObjectGenerator):
|
||||
|
||||
def __init__(self, url: str, **kwargs):
|
||||
def __init__(self, url: str, generate_all=False, **kwargs):
|
||||
super().__init__('url', **kwargs)
|
||||
self._generate_all = True if generate_all is True else False
|
||||
faup.decode(unquote_plus(url))
|
||||
self.generate_attributes()
|
||||
|
||||
|
|
@ -24,3 +29,28 @@ class URLObject(AbstractMISPObjectGenerator):
|
|||
self.add_attribute('host', value=faup.get_host())
|
||||
if faup.get_domain():
|
||||
self.add_attribute('domain', value=faup.get_domain())
|
||||
if self._generate_all:
|
||||
if hasattr(faup, 'ip_as_host') and faup.ip_as_host:
|
||||
self.attributes = [attr for attr in self.attributes
|
||||
if attr.object_relation not in ('host', 'domain')]
|
||||
self.add_attribute('ip', value=faup.ip_as_host)
|
||||
if faup.get_credential():
|
||||
self.add_attribute('credential', value=faup.get_credential())
|
||||
if faup.get_fragment():
|
||||
self.add_attribute('fragment', value=faup.get_fragment())
|
||||
if faup.get_port():
|
||||
self.add_attribute('port', value=faup.get_port())
|
||||
if faup.get_query_string():
|
||||
self.add_attribute('query_string', value=faup.get_query_string())
|
||||
if faup.get_resource_path():
|
||||
self.add_attribute('resource_path', value=faup.get_resource_path())
|
||||
if faup.get_scheme():
|
||||
self.add_attribute('scheme', value=faup.get_scheme())
|
||||
if faup.get_tld():
|
||||
self.add_attribute('tld', value=faup.get_tld())
|
||||
if faup.get_domain_without_tld():
|
||||
self.add_attribute('domain_without_tld', value=faup.get_domain_without_tld())
|
||||
if faup.get_subdomain():
|
||||
self.add_attribute('subdomain', value=faup.get_subdomain())
|
||||
if hasattr(faup, 'get_unicode_host') and faup.get_unicode_host() != faup.get_host():
|
||||
self.add_attribute('text', value=faup.get_unicode_host())
|
||||
|
|
|
|||
|
|
@ -42,11 +42,11 @@ include = [
|
|||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.7"
|
||||
requests = "^2.27.1"
|
||||
requests = "^2.28.0"
|
||||
python-dateutil = "^2.8.2"
|
||||
jsonschema = "^4.6.0"
|
||||
deprecated = "^1.2.13"
|
||||
extract_msg = {version = "^0.33.0", optional = true}
|
||||
extract_msg = {version = "^0.34.3", optional = true}
|
||||
RTFDE = {version = "^0.0.2", optional = true}
|
||||
oletools = {version = "^0.60.1", optional = true}
|
||||
python-magic = {version = "^0.4.27", optional = true}
|
||||
|
|
@ -54,10 +54,11 @@ pydeep2 = {version = "^0.5.1", optional = true}
|
|||
lief = {version = "^0.12.1", optional = true}
|
||||
beautifulsoup4 = {version = "^4.11.1", optional = true}
|
||||
validators = {version = "^0.20.0", optional = true}
|
||||
sphinx-autodoc-typehints = {version = "^1.18.2", optional = true}
|
||||
sphinx-autodoc-typehints = {version = "^1.18.3", optional = true}
|
||||
recommonmark = {version = "^0.7.1", optional = true}
|
||||
reportlab = {version = "^3.6.10", optional = true}
|
||||
pyfaup = {version = "^1.2", optional = true}
|
||||
publicsuffixlist = {version = "^0.7.13", optional = true}
|
||||
chardet = {version = "^4.0.0", optional = true}
|
||||
urllib3 = {extras = ["brotli"], version = "^1.26.9", optional = true}
|
||||
|
||||
|
|
@ -78,7 +79,7 @@ ipython = "^7.34.0"
|
|||
jupyterlab = "^3.4.3"
|
||||
types-requests = "^2.27.30"
|
||||
types-python-dateutil = "^2.8.17"
|
||||
types-redis = "^4.2.6"
|
||||
types-redis = "^4.2.7"
|
||||
types-Flask = "^1.1.6"
|
||||
pytest-cov = "^3.0.0"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue