OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity

Use importlib.metadata to read requirements (#12088) 

* Pull runtime dep checks into their own module
* Reimplement `check_requirements` using `importlib`

I've tried to make this clearer. We start by working out which of
Synapse's requirements we need to be installed here and now. I was
surprised that there wasn't an easier way to see which packages were
installed by a given extra.

I've pulled out the error messages into functions that deal with "is
this for an extra or not". And I've rearranged the loop over two
different sets of requirements into one loop with a "must be instaled"
flag.

I hope you agree that this is clearer.

* Test cases
babolivier/sign_json_module
David Robertson 2022-03-01 17:44:41 +00:00 committed by GitHub
parent 4d6b6c17c8
commit 313581e4e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 237 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/12088.misc Normal file
View File

@ -0,0 +1 @@
Inspect application dependencies using `importlib.metadata` or its backport.

6
synapse/app/__init__.py
View File

@ -15,13 +15,13 @@ import logging
import sys
from typing import Container
from synapse import python_dependencies # noqa: E402
from synapse.util import check_dependencies
logger = logging.getLogger(__name__)
try:
python_dependencies.check_requirements()
except python_dependencies.DependencyException as e:
check_dependencies.check_requirements()
except check_dependencies.DependencyException as e:
sys.stderr.writelines(
e.message # noqa: B306, DependencyException.message is a property
)

2
synapse/app/homeserver.py
View File

@ -59,7 +59,6 @@ from synapse.http.server import (
from synapse.http.site import SynapseSite
from synapse.logging.context import LoggingContext
from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
from synapse.python_dependencies import check_requirements
from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
from synapse.rest import ClientRestResource
@ -70,6 +69,7 @@ from synapse.rest.synapse.client import build_synapse_client_resource_tree
from synapse.rest.well_known import well_known_resource
from synapse.server import HomeServer
from synapse.storage import DataStore
from synapse.util.check_dependencies import check_requirements
from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.module_loader import load_module

2
synapse/config/cache.py
View File

@ -20,7 +20,7 @@ from typing import Callable, Dict, Optional
import attr
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError

2
synapse/config/metrics.py
View File

@ -15,7 +15,7 @@
import attr
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError

2
synapse/config/oidc.py
View File

@ -20,11 +20,11 @@ import attr
from synapse.config._util import validate_config
from synapse.config.sso import SsoAttributeRequirement
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
from synapse.util.module_loader import load_module
from synapse.util.stringutils import parse_and_validate_mxc_uri
from ..util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError, read_file
DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.oidc.JinjaOidcMappingProvider"

2
synapse/config/redis.py
View File

@ -13,7 +13,7 @@
# limitations under the License.
from synapse.config._base import Config
from synapse.python_dependencies import check_requirements
from synapse.util.check_dependencies import check_requirements
class RedisConfig(Config):

2
synapse/config/repository.py
View File

@ -20,8 +20,8 @@ from urllib.request import getproxies_environment # type: ignore
import attr
from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
from synapse.util.check_dependencies import DependencyException, check_requirements
from synapse.util.module_loader import load_module
from ._base import Config, ConfigError

2
synapse/config/saml2.py
View File

@ -17,8 +17,8 @@ import logging
from typing import Any, List, Set
from synapse.config.sso import SsoAttributeRequirement
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
from synapse.util.check_dependencies import DependencyException, check_requirements
from synapse.util.module_loader import load_module, load_python_module
from ._base import Config, ConfigError

2
synapse/config/tracer.py
View File

@ -14,7 +14,7 @@
from typing import Set
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError

107
synapse/python_dependencies.py
View File

@ -17,14 +17,7 @@
import itertools
import logging
from typing import List, Set
from pkg_resources import (
DistributionNotFound,
Requirement,
VersionConflict,
get_provider,
)
from typing import Set
logger = logging.getLogger(__name__)
@ -90,6 +83,8 @@ REQUIREMENTS = [
# ijson 3.1.4 fixes a bug with "." in property names
"ijson>=3.1.4",
"matrix-common~=1.1.0",
# For runtime introspection of our dependencies
"packaging~=21.3",
]
CONDITIONAL_REQUIREMENTS = {
@ -144,102 +139,6 @@ def list_requirements():
return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)
class DependencyException(Exception):
@property
def message(self):
return "\n".join(
[
"Missing Requirements: %s" % (", ".join(self.dependencies),),
"To install run:",
" pip install --upgrade --force %s" % (" ".join(self.dependencies),),
"",
]
)
@property
def dependencies(self):
for i in self.args[0]:
yield '"' + i + '"'
def check_requirements(for_feature=None):
deps_needed = []
errors = []
if for_feature:
reqs = CONDITIONAL_REQUIREMENTS[for_feature]
else:
reqs = REQUIREMENTS
for dependency in reqs:
try:
_check_requirement(dependency)
except VersionConflict as e:
deps_needed.append(dependency)
errors.append(
"Needed %s, got %s==%s"
% (
dependency,
e.dist.project_name, # type: ignore[attr-defined] # noqa
e.dist.version, # type: ignore[attr-defined] # noqa
)
)
except DistributionNotFound:
deps_needed.append(dependency)
if for_feature:
errors.append(
"Needed %s for the '%s' feature but it was not installed"
% (dependency, for_feature)
)
else:
errors.append("Needed %s but it was not installed" % (dependency,))
if not for_feature:
# Check the optional dependencies are up to date. We allow them to not be
# installed.
OPTS: List[str] = sum(CONDITIONAL_REQUIREMENTS.values(), [])
for dependency in OPTS:
try:
_check_requirement(dependency)
except VersionConflict as e:
deps_needed.append(dependency)
errors.append(
"Needed optional %s, got %s==%s"
% (
dependency,
e.dist.project_name, # type: ignore[attr-defined] # noqa
e.dist.version, # type: ignore[attr-defined] # noqa
)
)
except DistributionNotFound:
# If it's not found, we don't care
pass
if deps_needed:
for err in errors:
logging.error(err)
raise DependencyException(deps_needed)
def _check_requirement(dependency_string):
"""Parses a dependency string, and checks if the specified requirement is installed
Raises:
VersionConflict if the requirement is installed, but with the the wrong version
DistributionNotFound if nothing is found to provide the requirement
"""
req = Requirement.parse(dependency_string)
# first check if the markers specify that this requirement needs installing
if req.marker is not None and not req.marker.evaluate():
# not required for this environment
return
get_provider(req)
if __name__ == "__main__":
import sys

127
synapse/util/check_dependencies.py Normal file
View File

@ -0,0 +1,127 @@
import logging
from typing import Iterable, NamedTuple, Optional
from packaging.requirements import Requirement
DISTRIBUTION_NAME = "matrix-synapse"
try:
from importlib import metadata
except ImportError:
import importlib_metadata as metadata # type: ignore[no-redef]
class DependencyException(Exception):
@property
def message(self) -> str:
return "\n".join(
[
"Missing Requirements: %s" % (", ".join(self.dependencies),),
"To install run:",
" pip install --upgrade --force %s" % (" ".join(self.dependencies),),
"",
]
)
@property
def dependencies(self) -> Iterable[str]:
for i in self.args[0]:
yield '"' + i + '"'
EXTRAS = set(metadata.metadata(DISTRIBUTION_NAME).get_all("Provides-Extra"))
class Dependency(NamedTuple):
requirement: Requirement
must_be_installed: bool
def _generic_dependencies() -> Iterable[Dependency]:
"""Yield pairs (requirement, must_be_installed)."""
requirements = metadata.requires(DISTRIBUTION_NAME)
assert requirements is not None
for raw_requirement in requirements:
req = Requirement(raw_requirement)
# https://packaging.pypa.io/en/latest/markers.html#usage notes that
# > Evaluating an extra marker with no environment is an error
# so we pass in a dummy empty extra value here.
must_be_installed = req.marker is None or req.marker.evaluate({"extra": ""})
yield Dependency(req, must_be_installed)
def _dependencies_for_extra(extra: str) -> Iterable[Dependency]:
"""Yield additional dependencies needed for a given `extra`."""
requirements = metadata.requires(DISTRIBUTION_NAME)
assert requirements is not None
for raw_requirement in requirements:
req = Requirement(raw_requirement)
# Exclude mandatory deps by only selecting deps needed with this extra.
if (
req.marker is not None
and req.marker.evaluate({"extra": extra})
and not req.marker.evaluate({"extra": ""})
):
yield Dependency(req, True)
def _not_installed(requirement: Requirement, extra: Optional[str] = None) -> str:
if extra:
return f"Need {requirement.name} for {extra}, but it is not installed"
else:
return f"Need {requirement.name}, but it is not installed"
def _incorrect_version(
requirement: Requirement, got: str, extra: Optional[str] = None
) -> str:
if extra:
return f"Need {requirement} for {extra}, but got {requirement.name}=={got}"
else:
return f"Need {requirement}, but got {requirement.name}=={got}"
def check_requirements(extra: Optional[str] = None) -> None:
"""Check Synapse's dependencies are present and correctly versioned.
If provided, `extra` must be the name of an pacakging extra (e.g. "saml2" in
`pip install matrix-synapse[saml2]`).
If `extra` is None, this function checks that
- all mandatory dependencies are installed and correctly versioned, and
- each optional dependency that's installed is correctly versioned.
If `extra` is not None, this function checks that
- the dependencies needed for that extra are installed and correctly versioned.
:raises DependencyException: if a dependency is missing or incorrectly versioned.
:raises ValueError: if this extra does not exist.
"""
# First work out which dependencies are required, and which are optional.
if extra is None:
dependencies = _generic_dependencies()
elif extra in EXTRAS:
dependencies = _dependencies_for_extra(extra)
else:
raise ValueError(f"Synapse does not provide the feature '{extra}'")
deps_unfulfilled = []
errors = []
for (requirement, must_be_installed) in dependencies:
try:
dist: metadata.Distribution = metadata.distribution(requirement.name)
except metadata.PackageNotFoundError:
if must_be_installed:
deps_unfulfilled.append(requirement.name)
errors.append(_not_installed(requirement, extra))
else:
if not requirement.specifier.contains(dist.version):
deps_unfulfilled.append(requirement.name)
errors.append(_incorrect_version(requirement, dist.version, extra))
if deps_unfulfilled:
for err in errors:
logging.error(err)
raise DependencyException(deps_unfulfilled)

95
tests/util/test_check_dependencies.py Normal file
View File

@ -0,0 +1,95 @@
from contextlib import contextmanager
from typing import Generator, Optional
from unittest.mock import patch
from synapse.util.check_dependencies import (
DependencyException,
check_requirements,
metadata,
)
from tests.unittest import TestCase
class DummyDistribution(metadata.Distribution):
def __init__(self, version: str):
self._version = version
@property
def version(self):
return self._version
def locate_file(self, path):
raise NotImplementedError()
def read_text(self, filename):
raise NotImplementedError()
old = DummyDistribution("0.1.2")
new = DummyDistribution("1.2.3")
# could probably use stdlib TestCase --- no need for twisted here
class TestDependencyChecker(TestCase):
@contextmanager
def mock_installed_package(
self, distribution: Optional[DummyDistribution]
) -> Generator[None, None, None]:
"""Pretend that looking up any distribution yields the given `distribution`."""
def mock_distribution(name: str):
if distribution is None:
raise metadata.PackageNotFoundError
else:
return distribution
with patch(
"synapse.util.check_dependencies.metadata.distribution",
mock_distribution,
):
yield
def test_mandatory_dependency(self) -> None:
"""Complain if a required package is missing or old."""
with patch(
"synapse.util.check_dependencies.metadata.requires",
return_value=["dummypkg >= 1"],
):
with self.mock_installed_package(None):
self.assertRaises(DependencyException, check_requirements)
with self.mock_installed_package(old):
self.assertRaises(DependencyException, check_requirements)
with self.mock_installed_package(new):
# should not raise
check_requirements()
def test_generic_check_of_optional_dependency(self) -> None:
"""Complain if an optional package is old."""
with patch(
"synapse.util.check_dependencies.metadata.requires",
return_value=["dummypkg >= 1; extra == 'cool-extra'"],
):
with self.mock_installed_package(None):
# should not raise
check_requirements()
with self.mock_installed_package(old):
self.assertRaises(DependencyException, check_requirements)
with self.mock_installed_package(new):
# should not raise
check_requirements()
def test_check_for_extra_dependencies(self) -> None:
"""Complain if a package required for an extra is missing or old."""
with patch(
"synapse.util.check_dependencies.metadata.requires",
return_value=["dummypkg >= 1; extra == 'cool-extra'"],
), patch("synapse.util.check_dependencies.EXTRAS", {"cool-extra"}):
with self.mock_installed_package(None):
self.assertRaises(DependencyException, check_requirements, "cool-extra")
with self.mock_installed_package(old):
self.assertRaises(DependencyException, check_requirements, "cool-extra")
with self.mock_installed_package(new):
# should not raise
check_requirements()