Merge branch 'develop' of github.com:matrix-org/synapse into matrix-org-hotfixes

changelog.d/4472.feature

@@ -0,0 +1 @@
+Support exposing server capabilities in CS API (MSC1753, MSC1804)

changelog.d/4523.feature

@@ -0,0 +1 @@
+Add support for room version 3

changelog.d/4525.feature

@@ -0,0 +1 @@
+ Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt).

synapse/api/constants.py

@@ -108,6 +108,11 @@ class RoomVersions(object):
     STATE_V2_TEST = "state-v2-test"
 
 
+class RoomDisposition(object):
+    STABLE = "stable",
+    UNSTABLE = "unstable"
+
+
 # the version we will give rooms which are created on this server
 DEFAULT_ROOM_VERSION = RoomVersions.V1
 
@@ -118,6 +123,7 @@ KNOWN_ROOM_VERSIONS = {
     RoomVersions.V2,
     RoomVersions.V3,
     RoomVersions.STATE_V2_TEST,
+    RoomVersions.V3,
 }
 
 

synapse/app/__init__.py

@@ -12,15 +12,38 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import logging
 import sys
 
 from synapse import python_dependencies  # noqa: E402
 
 sys.dont_write_bytecode = True
 
+logger = logging.getLogger(__name__)
+
 try:
     python_dependencies.check_requirements()
 except python_dependencies.DependencyException as e:
     sys.stderr.writelines(e.message)
     sys.exit(1)
+
+
+def check_bind_error(e, address, bind_addresses):
+    """
+    This method checks an exception occurred while binding on 0.0.0.0.
+    If :: is specified in the bind addresses a warning is shown.
+    The exception is still raised otherwise.
+
+    Binding on both 0.0.0.0 and :: causes an exception on Linux and macOS
+    because :: binds on both IPv4 and IPv6 (as per RFC 3493).
+    When binding on 0.0.0.0 after :: this can safely be ignored.
+
+    Args:
+        e (Exception): Exception that was caught.
+        address (str): Address on which binding was attempted.
+        bind_addresses (list): Addresses on which the service listens.
+    """
+    if address == '0.0.0.0' and '::' in bind_addresses:
+        logger.warn('Failed to listen on 0.0.0.0, continuing because listening on [::]')
+    else:
+        raise e

synapse/app/_base.py

@@ -22,6 +22,7 @@ from daemonize import Daemonize
 
 from twisted.internet import error, reactor
 
+from synapse.app import check_bind_error
 from synapse.util import PreserveLoggingContext
 from synapse.util.rlimit import change_resource_limit
 
@@ -188,24 +189,3 @@ def listen_ssl(
 
     logger.info("Synapse now listening on port %d (TLS)", port)
     return r
-
-
-def check_bind_error(e, address, bind_addresses):
-    """
-    This method checks an exception occurred while binding on 0.0.0.0.
-    If :: is specified in the bind addresses a warning is shown.
-    The exception is still raised otherwise.
-
-    Binding on both 0.0.0.0 and :: causes an exception on Linux and macOS
-    because :: binds on both IPv4 and IPv6 (as per RFC 3493).
-    When binding on 0.0.0.0 after :: this can safely be ignored.
-
-    Args:
-        e (Exception): Exception that was caught.
-        address (str): Address on which binding was attempted.
-        bind_addresses (list): Addresses on which the service listens.
-    """
-    if address == '0.0.0.0' and '::' in bind_addresses:
-        logger.warn('Failed to listen on 0.0.0.0, continuing because listening on [::]')
-    else:
-        raise e

synapse/config/tls.py

@@ -31,13 +31,16 @@ logger = logging.getLogger()
 class TlsConfig(Config):
     def read_config(self, config):
 
-        acme_config = config.get("acme", {})
+        acme_config = config.get("acme", None)
+        if acme_config is None:
+            acme_config = {}
+
         self.acme_enabled = acme_config.get("enabled", False)
         self.acme_url = acme_config.get(
             "url", "https://acme-v01.api.letsencrypt.org/directory"
         )
-        self.acme_port = acme_config.get("port", 8449)
+        self.acme_port = acme_config.get("port", 80)
-        self.acme_bind_addresses = acme_config.get("bind_addresses", ["127.0.0.1"])
+        self.acme_bind_addresses = acme_config.get("bind_addresses", ['::', '0.0.0.0'])
         self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30)
 
         self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
@@ -126,21 +129,80 @@ class TlsConfig(Config):
         tls_certificate_path = base_key_name + ".tls.crt"
         tls_private_key_path = base_key_name + ".tls.key"
 
+        # this is to avoid the max line length. Sorrynotsorry
+        proxypassline = (
+            'ProxyPass /.well-known/acme-challenge '
+            'http://localhost:8009/.well-known/acme-challenge'
+        )
+
         return (
             """\
-        # PEM encoded X509 certificate for TLS.
+        # PEM-encoded X509 certificate for TLS.
-        # This certificate, as of Synapse 1.0, will need to be a valid
+        # This certificate, as of Synapse 1.0, will need to be a valid and verifiable
-        # and verifiable certificate, with a root that is available in
+        # certificate, signed by a recognised Certificate Authority.
-        # the root store of other servers you wish to federate to. Any
+        #
-        # required intermediary certificates can be appended after the
+        # See 'ACME support' below to enable auto-provisioning this certificate via
-        # primary certificate in hierarchical order.
+        # Let's Encrypt.
+        #
         tls_certificate_path: "%(tls_certificate_path)s"
 
-        # PEM encoded private key for TLS
+        # PEM-encoded private key for TLS
         tls_private_key_path: "%(tls_private_key_path)s"
 
-        # Don't bind to the https port
+        # ACME support: This will configure Synapse to request a valid TLS certificate
-        no_tls: False
+        # for your configured `server_name` via Let's Encrypt.
+        #
+        # Note that provisioning a certificate in this way requires port 80 to be
+        # routed to Synapse so that it can complete the http-01 ACME challenge.
+        # By default, if you enable ACME support, Synapse will attempt to listen on
+        # port 80 for incoming http-01 challenges - however, this will likely fail
+        # with 'Permission denied' or a similar error.
+        #
+        # There are a couple of potential solutions to this:
+        #
+        #  * If you already have an Apache, Nginx, or similar listening on port 80,
+        #    you can configure Synapse to use an alternate port, and have your web
+        #    server forward the requests. For example, assuming you set 'port: 8009'
+        #    below, on Apache, you would write:
+        #
+        #    %(proxypassline)s
+        #
+        #  * Alternatively, you can use something like `authbind` to give Synapse
+        #    permission to listen on port 80.
+        #
+        acme:
+            # ACME support is disabled by default. Uncomment the following line
+            # to enable it.
+            #
+            # enabled: true
+
+            # Endpoint to use to request certificates. If you only want to test,
+            # use Let's Encrypt's staging url:
+            #     https://acme-staging.api.letsencrypt.org/directory
+            #
+            # url: https://acme-v01.api.letsencrypt.org/directory
+
+            # Port number to listen on for the HTTP-01 challenge. Change this if
+            # you are forwarding connections through Apache/Nginx/etc.
+            #
+            # port: 80
+
+            # Local addresses to listen on for incoming connections.
+            # Again, you may want to change this if you are forwarding connections
+            # through Apache/Nginx/etc.
+            #
+            # bind_addresses: ['::', '0.0.0.0']
+
+            # How many days remaining on a certificate before it is renewed.
+            #
+            # reprovision_threshold: 30
+
+        # If your server runs behind a reverse-proxy which terminates TLS connections
+        # (for both client and federation connections), it may be useful to disable
+        # All TLS support for incoming connections. Setting no_tls to False will
+        # do so (and avoid the need to give synapse a TLS private key).
+        #
+        # no_tls: False
 
         # List of allowed TLS fingerprints for this server to publish along
         # with the signing keys for this server. Other matrix servers that
@@ -170,20 +232,6 @@ class TlsConfig(Config):
         tls_fingerprints: []
         # tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
 
-        ## Support for ACME certificate auto-provisioning.
-        # acme:
-        #    enabled: false
-        ##   ACME path.
-        ##   If you only want to test, use the staging url:
-        ##   https://acme-staging.api.letsencrypt.org/directory
-        #    url: 'https://acme-v01.api.letsencrypt.org/directory'
-        ##   Port number (to listen for the HTTP-01 challenge).
-        ##   Using port 80 requires utilising something like authbind, or proxying to it.
-        #    port: 8449
-        ##   Hosts to bind to.
-        #    bind_addresses: ['127.0.0.1']
-        ##   How many days remaining on a certificate before it is renewed.
-        #    reprovision_threshold: 30
         """
             % locals()
         )

synapse/handlers/acme.py

@@ -18,13 +18,16 @@ import logging
 import attr
 from zope.interface import implementer
 
+import twisted
+import twisted.internet.error
 from twisted.internet import defer
-from twisted.internet.endpoints import serverFromString
 from twisted.python.filepath import FilePath
 from twisted.python.url import URL
 from twisted.web import server, static
 from twisted.web.resource import Resource
 
+from synapse.app import check_bind_error
+
 logger = logging.getLogger(__name__)
 
 try:
@@ -96,16 +99,19 @@ class AcmeHandler(object):
 
         srv = server.Site(responder_resource)
 
-        listeners = []
+        bind_addresses = self.hs.config.acme_bind_addresses
-
+        for host in bind_addresses:
-        for host in self.hs.config.acme_bind_addresses:
             logger.info(
-                "Listening for ACME requests on %s:%s", host, self.hs.config.acme_port
+                "Listening for ACME requests on %s:%i", host, self.hs.config.acme_port,
             )
-            endpoint = serverFromString(
+            try:
-                self.reactor, "tcp:%s:interface=%s" % (self.hs.config.acme_port, host)
+                self.reactor.listenTCP(
-            )
+                    self.hs.config.acme_port,
-            listeners.append(endpoint.listen(srv))
+                    srv,
+                    interface=host,
+                )
+            except twisted.internet.error.CannotListenError as e:
+                check_bind_error(e, host, bind_addresses)
 
         # Make sure we are registered to the ACME server. There's no public API
         # for this, it is usually triggered by startService, but since we don't
@@ -114,9 +120,6 @@ class AcmeHandler(object):
         self._issuer._registered = False
         yield self._issuer._ensure_registered()
 
-        # Return a Deferred that will fire when all the servers have started up.
-        yield defer.DeferredList(listeners, fireOnOneErrback=True, consumeErrors=True)
-
     @defer.inlineCallbacks
     def provision_certificate(self):
 

synapse/replication/http/_base.py

@@ -127,7 +127,10 @@ class ReplicationEndpoint(object):
         def send_request(**kwargs):
             data = yield cls._serialize_payload(**kwargs)
 
-            url_args = [urllib.parse.quote(kwargs[name]) for name in cls.PATH_ARGS]
+            url_args = [
+                urllib.parse.quote(kwargs[name], safe='')
+                for name in cls.PATH_ARGS
+            ]
 
             if cls.CACHE:
                 txn_id = random_string(10)

synapse/rest/__init__.py

@@ -34,6 +34,7 @@ from synapse.rest.client.v2_alpha import (
     account,
     account_data,
     auth,
+    capabilities,
     devices,
     filter,
     groups,
@@ -107,3 +108,4 @@ class ClientRestResource(JsonResource):
         user_directory.register_servlets(hs, client_resource)
         groups.register_servlets(hs, client_resource)
         room_upgrade_rest_servlet.register_servlets(hs, client_resource)
+        capabilities.register_servlets(hs, client_resource)

synapse/rest/client/v2_alpha/capabilities.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from twisted.internet import defer
+
+from synapse.api.constants import DEFAULT_ROOM_VERSION, RoomDisposition, RoomVersions
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_patterns
+
+logger = logging.getLogger(__name__)
+
+
+class CapabilitiesRestServlet(RestServlet):
+    """End point to expose the capabilities of the server."""
+
+    PATTERNS = client_v2_patterns("/capabilities$")
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(CapabilitiesRestServlet, self).__init__()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        requester = yield self.auth.get_user_by_req(request, allow_guest=True)
+        user = yield self.store.get_user_by_id(requester.user.to_string())
+        change_password = bool(user["password_hash"])
+
+        response = {
+            "capabilities": {
+                "m.room_versions": {
+                    "default": DEFAULT_ROOM_VERSION,
+                    "available": {
+                        RoomVersions.V1: RoomDisposition.STABLE,
+                        RoomVersions.V2: RoomDisposition.STABLE,
+                        RoomVersions.STATE_V2_TEST: RoomDisposition.UNSTABLE,
+                        RoomVersions.V3: RoomDisposition.STABLE,
+                    },
+                },
+                "m.change_password": {"enabled": change_password},
+            }
+        }
+        defer.returnValue((200, response))
+
+
+def register_servlets(hs, http_server):
+    CapabilitiesRestServlet(hs).register(http_server)

tests/rest/client/v2_alpha/test_capabilities.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.api.constants import DEFAULT_ROOM_VERSION, KNOWN_ROOM_VERSIONS
+from synapse.rest.client.v1 import admin, login
+from synapse.rest.client.v2_alpha import capabilities
+
+from tests import unittest
+
+
+class CapabilitiesTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        admin.register_servlets,
+        capabilities.register_servlets,
+        login.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+        self.url = b"/_matrix/client/r0/capabilities"
+        hs = self.setup_test_homeserver()
+        self.store = hs.get_datastore()
+        return hs
+
+    def test_check_auth_required(self):
+        request, channel = self.make_request("GET", self.url)
+        self.render(request)
+
+        self.assertEqual(channel.code, 401)
+
+    def test_get_room_version_capabilities(self):
+        self.register_user("user", "pass")
+        access_token = self.login("user", "pass")
+
+        request, channel = self.make_request("GET", self.url, access_token=access_token)
+        self.render(request)
+        capabilities = channel.json_body['capabilities']
+
+        self.assertEqual(channel.code, 200)
+        for room_version in capabilities['m.room_versions']['available'].keys():
+            self.assertTrue(room_version in KNOWN_ROOM_VERSIONS, "" + room_version)
+        self.assertEqual(
+            DEFAULT_ROOM_VERSION, capabilities['m.room_versions']['default']
+        )
+
+    def test_get_change_password_capabilities(self):
+        localpart = "user"
+        password = "pass"
+        user = self.register_user(localpart, password)
+        access_token = self.login(user, password)
+
+        request, channel = self.make_request("GET", self.url, access_token=access_token)
+        self.render(request)
+        capabilities = channel.json_body['capabilities']
+
+        self.assertEqual(channel.code, 200)
+
+        # Test case where password is handled outside of Synapse
+        self.assertTrue(capabilities['m.change_password']['enabled'])
+        self.get_success(self.store.user_set_password_hash(user, None))
+        request, channel = self.make_request("GET", self.url, access_token=access_token)
+        self.render(request)
+        capabilities = channel.json_body['capabilities']
+
+        self.assertEqual(channel.code, 200)
+        self.assertFalse(capabilities['m.change_password']['enabled'])