From ab97b6e33c6e141d185ec203d24ea8266f924900 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:52:57 +0200 Subject: [PATCH 01/96] Fix a test docstring in frontend proxy tests Signed-off-by: Jason Robinson --- tests/app/test_frontend_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py index a83f567ebd..8bdbc608a9 100644 --- a/tests/app/test_frontend_proxy.py +++ b/tests/app/test_frontend_proxy.py @@ -59,7 +59,7 @@ class FrontendProxyTests(HomeserverTestCase): def test_listen_http_with_presence_disabled(self): """ - When presence is on, the stub servlet will register. + When presence is off, the stub servlet will register. """ # Presence is off self.hs.config.use_presence = False From 899e60be80736e3b747eb567171296829e96f716 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:55:37 +0200 Subject: [PATCH 02/96] Add parameterized Python module to test dependencies Allows running parameterized tests. BSD license. Signed-off-by: Jason Robinson --- synapse/python_dependencies.py | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 882e844eb1..b5555fbba9 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -81,7 +81,7 @@ CONDITIONAL_REQUIREMENTS = { "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], - "test": ["mock>=2.0"], + "test": ["mock>=2.0", "parameterized"], } diff --git a/tox.ini b/tox.ini index a0f5486829..9b6cb2036c 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ deps = python-subunit junitxml coverage + parameterized # cyptography 2.2 requires setuptools >= 18.5 # From 4f8f41c824dc326903863625ce79e5386ad0f8e1 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:58:27 +0200 Subject: [PATCH 03/96] Make FederationReaderServer _http_listen use self.get_reactor() For all the homeserver classes, only the FrontendProxyServer passes its reactor when doing the http listen. Looking at previous PR's looks like this was introduced to make it possible to write a test, otherwise when you try to run a test with the test homeserver it tries to do a real bind to a port. Passing the reactor that the homeserver is instantiated with should probably be the right thing to do anyway? Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 228a297fb8..ea594f0f1a 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -99,7 +99,8 @@ class FederationReaderServer(HomeServer): listener_config, root_resource, self.version_string, - ) + ), + reactor=self.get_reactor() ) logger.info("Synapse federation reader now listening on port %d", port) From 6d255990983405fe7eb1322c9d02414c859d1e96 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:59:28 +0200 Subject: [PATCH 04/96] Add tests for the openid lister for FederationReaderServer Check all possible variants of openid and federation listener on/off possibilities. Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/app/test_openid_listener.py diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py new file mode 100644 index 0000000000..cf3baad96f --- /dev/null +++ b/tests/app/test_openid_listener.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from mock import patch, Mock +from parameterized import parameterized + +from synapse.app.federation_reader import FederationReaderServer + +from tests.unittest import HomeserverTestCase + + +@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) +class FederationReaderOpenIDListenerTests(HomeserverTestCase): + def make_homeserver(self, reactor, clock): + hs = self.setup_test_homeserver( + http_client=None, homeserverToUse=FederationReaderServer, + ) + return hs + + @parameterized.expand([ + (["federation"], "auth_fail"), + ([], "no_resource"), + (["openid", "federation"], "auth_fail"), + (["openid"], "auth_fail"), + ]) + def test_openid_listener(self, names, expectation): + """ + Test different openid listener configurations. + + 401 is success here since it means we hit the handler and auth failed. + """ + config = { + "port": 8080, + "bind_addresses": ["0.0.0.0"], + "resources": [{"names": names}], + } + + # Listen with the config + self.hs._listen_http(config) + + # Grab the resource from the site that was told to listen + site = self.reactor.tcpServers[0][1] + try: + self.resource = ( + site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + ) + except KeyError: + if expectation == "no_resource": + return + raise + + request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + self.render(request) + + self.assertEqual(channel.code, 401) From a17bac171f301e54d6bd3d4c769f4f005c38f112 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 18 Jan 2019 10:02:08 +0200 Subject: [PATCH 05/96] Make SynapseHomeServer _http_listener use self.get_reactor() For all the homeserver classes, only the FrontendProxyServer passes its reactor when doing the http listen. Looking at previous PR's looks like this was introduced to make it possible to write a test, otherwise when you try to run a test with the test homeserver it tries to do a real bind to a port. Passing the reactor that the homeserver is instantiated with should probably be the right thing to do anyway? Signed-off-by: Jason Robinson --- synapse/app/homeserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f3ac3d19f0..5652c6201c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -130,6 +130,7 @@ class SynapseHomeServer(HomeServer): self.version_string, ), self.tls_server_context_factory, + reactor=self.get_reactor(), ) else: @@ -142,7 +143,8 @@ class SynapseHomeServer(HomeServer): listener_config, root_resource, self.version_string, - ) + ), + reactor=self.get_reactor(), ) logger.info("Synapse now listening on port %d", port) From 5336e49b39b6ad73491531c68694c92dade028be Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 18 Jan 2019 10:03:32 +0200 Subject: [PATCH 06/96] Add tests for the openid lister for SynapseHomeServer Check all possible variants of openid and federation listener on/off possibilities. Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 49 ++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index cf3baad96f..329aadc3d3 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2018 New Vector Ltd +# Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ from mock import patch, Mock from parameterized import parameterized from synapse.app.federation_reader import FederationReaderServer +from synapse.app.homeserver import SynapseHomeServer from tests.unittest import HomeserverTestCase @@ -64,3 +65,49 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): self.render(request) self.assertEqual(channel.code, 401) + + +@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) +class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): + def make_homeserver(self, reactor, clock): + hs = self.setup_test_homeserver( + http_client=None, homeserverToUse=SynapseHomeServer, + ) + return hs + + @parameterized.expand([ + (["federation"], "auth_fail"), + ([], "no_resource"), + (["openid", "federation"], "auth_fail"), + (["openid"], "auth_fail"), + ]) + def test_openid_listener(self, names, expectation): + """ + Test different openid listener configurations. + + 401 is success here since it means we hit the handler and auth failed. + """ + config = { + "port": 8080, + "bind_addresses": ["0.0.0.0"], + "resources": [{"names": names}], + } + + # Listen with the config + self.hs._listener_http(config, config) + + # Grab the resource from the site that was told to listen + site = self.reactor.tcpServers[0][1] + try: + self.resource = ( + site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + ) + except KeyError: + if expectation == "no_resource": + return + raise + + request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + self.render(request) + + self.assertEqual(channel.code, 401) From 82e13662c03a41c085e784a594b423711e0caffa Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 21 Jan 2019 01:54:43 +0200 Subject: [PATCH 07/96] Split federation OpenID userinfo endpoint out of the federation resource This allows the OpenID userinfo endpoint to be active even if the federation resource is not active. The OpenID userinfo endpoint is called by integration managers to verify user actions using the client API OpenID access token. Without this verification, the integration manager cannot know that the access token is valid. The OpenID userinfo endpoint will be loaded in the case that either "federation" or "openid" resource is defined. The new "openid" resource is defaulted to active in default configuration. Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 7 ++ synapse/app/homeserver.py | 9 +++ synapse/config/server.py | 9 ++- synapse/federation/transport/server.py | 106 ++++++++++++++++--------- 4 files changed, 89 insertions(+), 42 deletions(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index ea594f0f1a..99e8a4cf6a 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -87,6 +87,13 @@ class FederationReaderServer(HomeServer): resources.update({ FEDERATION_PREFIX: TransportLayerServer(self), }) + if name == "openid" and "federation" not in res["names"]: + # Only load the openid resource separately if federation resource + # is not specified since federation resource includes openid + # resource. + resources.update({ + FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + }) root_resource = create_resource_tree(resources, NoResource()) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5652c6201c..0a924d7a80 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -95,6 +95,10 @@ class SynapseHomeServer(HomeServer): resources = {} for res in listener_config["resources"]: for name in res["names"]: + if name == "openid" and "federation" in res["names"]: + # Skip loading openid resource if federation is defined + # since federation resource will include openid + continue resources.update(self._configure_named_resource( name, res.get("compress", False), )) @@ -192,6 +196,11 @@ class SynapseHomeServer(HomeServer): FEDERATION_PREFIX: TransportLayerServer(self), }) + if name == "openid": + resources.update({ + FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + }) + if name in ["static", "client"]: resources.update({ STATIC_PREFIX: File( diff --git a/synapse/config/server.py b/synapse/config/server.py index fb57791098..556f1efee5 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -151,7 +151,7 @@ class ServerConfig(Config): "compress": gzip_responses, }, { - "names": ["federation"], + "names": ["federation", "openid"], "compress": False, } ] @@ -170,7 +170,7 @@ class ServerConfig(Config): "compress": gzip_responses, }, { - "names": ["federation"], + "names": ["federation", "openid"], "compress": False, } ] @@ -328,7 +328,7 @@ class ServerConfig(Config): # that can do automatic compression. compress: true - - names: [federation] # Federation APIs + - names: [federation, openid] # Federation APIs compress: false # optional list of additional endpoints which can be loaded via @@ -350,7 +350,7 @@ class ServerConfig(Config): resources: - names: [client] compress: true - - names: [federation] + - names: [federation, openid] compress: false # Turn on the twisted ssh manhole service on localhost on the given @@ -477,6 +477,7 @@ KNOWN_RESOURCES = ( 'keys', 'media', 'metrics', + 'openid', 'replication', 'static', 'webclient', diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 4557a9e66e..49b80beecb 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -43,9 +43,10 @@ logger = logging.getLogger(__name__) class TransportLayerServer(JsonResource): """Handles incoming federation HTTP requests""" - def __init__(self, hs): + def __init__(self, hs, servlet_groups=None): self.hs = hs self.clock = hs.get_clock() + self.servlet_groups = servlet_groups super(TransportLayerServer, self).__init__(hs, canonical_json=False) @@ -67,6 +68,7 @@ class TransportLayerServer(JsonResource): resource=self, ratelimiter=self.ratelimiter, authenticator=self.authenticator, + servlet_groups=self.servlet_groups, ) @@ -1308,10 +1310,12 @@ FEDERATION_SERVLET_CLASSES = ( FederationClientKeysClaimServlet, FederationThirdPartyInviteExchangeServlet, On3pidBindServlet, - OpenIdUserInfo, FederationVersionServlet, ) +OPENID_SERVLET_CLASSES = ( + OpenIdUserInfo, +) ROOM_LIST_CLASSES = ( PublicRoomList, @@ -1350,44 +1354,70 @@ GROUP_ATTESTATION_SERVLET_CLASSES = ( FederationGroupsRenewAttestaionServlet, ) +DEFAULT_SERVLET_GROUPS = ( + "federation", + "room_list", + "group_server", + "group_local", + "group_attestation", + "openid", +) -def register_servlets(hs, resource, authenticator, ratelimiter): - for servletclass in FEDERATION_SERVLET_CLASSES: - servletclass( - handler=hs.get_federation_server(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) - for servletclass in ROOM_LIST_CLASSES: - servletclass( - handler=hs.get_room_list_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) +def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None): + if not servlet_groups: + servlet_groups = DEFAULT_SERVLET_GROUPS - for servletclass in GROUP_SERVER_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_server_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) + if "federation" in servlet_groups: + for servletclass in FEDERATION_SERVLET_CLASSES: + servletclass( + handler=hs.get_federation_server(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) - for servletclass in GROUP_LOCAL_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_local_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) + if "openid" in servlet_groups: + for servletclass in OPENID_SERVLET_CLASSES: + servletclass( + handler=hs.get_federation_server(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) - for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_attestation_renewer(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) + if "room_list" in servlet_groups: + for servletclass in ROOM_LIST_CLASSES: + servletclass( + handler=hs.get_room_list_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_server" in servlet_groups: + for servletclass in GROUP_SERVER_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_server_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_local" in servlet_groups: + for servletclass in GROUP_LOCAL_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_local_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_attestation" in servlet_groups: + for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_attestation_renewer(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) From 1d2c69fee897cf052cfa03f0cc6f9f419c898bb1 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 21 Jan 2019 01:59:18 +0200 Subject: [PATCH 08/96] Add changelog for openid resource addition Signed-off-by: Jason Robinson --- changelog.d/4420.feature | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 changelog.d/4420.feature diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature new file mode 100644 index 0000000000..5e684d01e0 --- /dev/null +++ b/changelog.d/4420.feature @@ -0,0 +1,13 @@ +New listener resource for the federation API "openid/userinfo" endpoint + +Integration managers use the OpenID userinfo endpoint in the federation API to verify that user +OpenID access tokens are valid. If the federation resource is disabled, integration managers will not be able +to verify the access token, causing a broken experience for users. The OpenID userinfo endpoint has now been split +to a separate `openid` resource, which is enabled by default in newly generated configuration. It is also enabled +automatically if the federation resource is enabled. + +If your homeserver runs federation enabled, this change does not require any actions. + +If you run a homeserver with federation disabled, we recommend adding the `openid` resource to your homeserver +configuration in the `type: http` listener `resources` list to allow your users access to +integration manager features. From d39b7b6d38b0d9876ad305491ea05af92ea35b04 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:00:17 +0200 Subject: [PATCH 09/96] Document `servlet_groups` parameters Signed-off-by: Jason Robinson --- synapse/federation/transport/server.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 49b80beecb..9c1c9c39e1 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -44,6 +44,16 @@ class TransportLayerServer(JsonResource): """Handles incoming federation HTTP requests""" def __init__(self, hs, servlet_groups=None): + """Initialize the TransportLayerServer + + Will by default register all servlets. For custom behaviour, pass in + a list of servlet_groups to register. + + Args: + hs (synapse.server.HomeServer): homeserver + servlet_groups (list[str], optional): List of servlet groups to register. + Defaults to ``DEFAULT_SERVLET_GROUPS``. + """ self.hs = hs self.clock = hs.get_clock() self.servlet_groups = servlet_groups @@ -1365,6 +1375,19 @@ DEFAULT_SERVLET_GROUPS = ( def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None): + """Initialize and register servlet classes. + + Will by default register all servlets. For custom behaviour, pass in + a list of servlet_groups to register. + + Args: + hs (synapse.server.HomeServer): homeserver + resource (TransportLayerServer): resource class to register to + authenticator (Authenticator): authenticator to use + ratelimiter (util.ratelimitutils.FederationRateLimiter): ratelimiter to use + servlet_groups (list[str], optional): List of servlet groups to register. + Defaults to ``DEFAULT_SERVLET_GROUPS``. + """ if not servlet_groups: servlet_groups = DEFAULT_SERVLET_GROUPS From 0516dc4d85f1018beb241df6833117b8b49d08d3 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:04:10 +0200 Subject: [PATCH 10/96] Remove openid resource from default config Instead document it commented out. Signed-off-by: Jason Robinson --- synapse/config/server.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 556f1efee5..eebbfccafe 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -151,7 +151,7 @@ class ServerConfig(Config): "compress": gzip_responses, }, { - "names": ["federation", "openid"], + "names": ["federation"], "compress": False, } ] @@ -170,7 +170,7 @@ class ServerConfig(Config): "compress": gzip_responses, }, { - "names": ["federation", "openid"], + "names": ["federation"], "compress": False, } ] @@ -328,8 +328,13 @@ class ServerConfig(Config): # that can do automatic compression. compress: true - - names: [federation, openid] # Federation APIs + - names: [federation] # Federation APIs compress: false + + # # If federation is disabled synapse can still expose the open ID endpoint + # # to allow integrations to authenticate users + # - names: [openid] + # compress: false # optional list of additional endpoints which can be loaded via # dynamic modules @@ -350,8 +355,12 @@ class ServerConfig(Config): resources: - names: [client] compress: true - - names: [federation, openid] + - names: [federation] compress: false + # # If federation is disabled synapse can still expose the open ID endpoint + # # to allow integrations to authenticate users + # - names: [openid] + # compress: false # Turn on the twisted ssh manhole service on localhost on the given # port. From db33634b1dc47167cffce31ff4ae44c5d3fae2af Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:05:22 +0200 Subject: [PATCH 11/96] Collapse changelog to one line Signed-off-by: Jason Robinson --- changelog.d/4420.feature | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature index 5e684d01e0..05e777c624 100644 --- a/changelog.d/4420.feature +++ b/changelog.d/4420.feature @@ -1,13 +1 @@ -New listener resource for the federation API "openid/userinfo" endpoint - -Integration managers use the OpenID userinfo endpoint in the federation API to verify that user -OpenID access tokens are valid. If the federation resource is disabled, integration managers will not be able -to verify the access token, causing a broken experience for users. The OpenID userinfo endpoint has now been split -to a separate `openid` resource, which is enabled by default in newly generated configuration. It is also enabled -automatically if the federation resource is enabled. - -If your homeserver runs federation enabled, this change does not require any actions. - -If you run a homeserver with federation disabled, we recommend adding the `openid` resource to your homeserver -configuration in the `type: http` listener `resources` list to allow your users access to -integration manager features. +Federation OpenID listener resource can now be activated even if federation is disabled From a47fac9af6b17ae60c381675caece6e846faefff Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:07:28 +0200 Subject: [PATCH 12/96] Fix sorting of imports in tests. Remove an unnecessary mock Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 329aadc3d3..c1ebc93f52 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -12,7 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from mock import patch, Mock +from mock import Mock, patch + from parameterized import parameterized from synapse.app.federation_reader import FederationReaderServer @@ -21,7 +22,6 @@ from synapse.app.homeserver import SynapseHomeServer from tests.unittest import HomeserverTestCase -@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) class FederationReaderOpenIDListenerTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( From 1838ef1ac39d624e21786120b402d04a0c4485ba Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 23 Jan 2019 10:38:13 +0200 Subject: [PATCH 13/96] Fix openid tests after rebase Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index c1ebc93f52..46a3b61a3c 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -54,7 +54,7 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): site = self.reactor.tcpServers[0][1] try: self.resource = ( - site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + site.resource.children[b"_matrix"].children[b"federation"] ) except KeyError: if expectation == "no_resource": @@ -100,7 +100,7 @@ class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): site = self.reactor.tcpServers[0][1] try: self.resource = ( - site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + site.resource.children[b"_matrix"].children[b"federation"] ) except KeyError: if expectation == "no_resource": From 6f680241bd7f53c3a3f912c257d2bfa437dfd486 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 23 Jan 2019 10:53:48 +0200 Subject: [PATCH 14/96] Fix flake8 issues Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 5 ++++- synapse/config/server.py | 2 +- tests/app/test_openid_listener.py | 10 ++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 99e8a4cf6a..2c99ce8c64 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -92,7 +92,10 @@ class FederationReaderServer(HomeServer): # is not specified since federation resource includes openid # resource. resources.update({ - FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + FEDERATION_PREFIX: TransportLayerServer( + self, + servlet_groups=["openid"], + ), }) root_resource = create_resource_tree(resources, NoResource()) diff --git a/synapse/config/server.py b/synapse/config/server.py index eebbfccafe..4eefd06f4a 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -330,7 +330,7 @@ class ServerConfig(Config): - names: [federation] # Federation APIs compress: false - + # # If federation is disabled synapse can still expose the open ID endpoint # # to allow integrations to authenticate users # - names: [openid] diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 46a3b61a3c..590abc1e92 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -61,7 +61,10 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): return raise - request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + request, channel = self.make_request( + "GET", + "/_matrix/federation/v1/openid/userinfo", + ) self.render(request) self.assertEqual(channel.code, 401) @@ -107,7 +110,10 @@ class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): return raise - request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + request, channel = self.make_request( + "GET", + "/_matrix/federation/v1/openid/userinfo", + ) self.render(request) self.assertEqual(channel.code, 401) From d621c5562ea6bce4fd8282da642b3123ae016d94 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 30 Jan 2019 16:33:51 +0000 Subject: [PATCH 15/96] Copy over non-federatable trait on room upgrade --- synapse/handlers/room.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 13ba9291b0..c04ba3a0c5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -263,6 +263,20 @@ class RoomCreationHandler(BaseHandler): } } + # Check if old room was non-federatable + + # Get old room's create event + old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( + old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), + ) + old_room_create_event_dict = yield self.store.get_events(old_room_create_event_ids.values()) + old_room_create_event = list(old_room_create_event_dict.values())[0] + + # Check if the create event specified a non-federatable room + if old_room_create_event.content.get("m.federate", True) == False: + # If so, mark the new room as non-federatable as well + creation_content["m.federate"] = False + initial_state = dict() # Replicate relevant room events From cf9a2676d026c7864de44fe3440475a9cf1dc31d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 30 Jan 2019 19:04:48 +0000 Subject: [PATCH 16/96] Add changelog --- changelog.d/4530.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4530.bugfix diff --git a/changelog.d/4530.bugfix b/changelog.d/4530.bugfix new file mode 100644 index 0000000000..d010af927e --- /dev/null +++ b/changelog.d/4530.bugfix @@ -0,0 +1 @@ +Copy over room federation ability on room upgrade. \ No newline at end of file From fb50934b8ff605a70b2a325eaaf51148a9651de5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 11:34:45 +0000 Subject: [PATCH 17/96] lint --- synapse/handlers/room.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c04ba3a0c5..a69441b96f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -269,11 +269,13 @@ class RoomCreationHandler(BaseHandler): old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), ) - old_room_create_event_dict = yield self.store.get_events(old_room_create_event_ids.values()) + old_room_create_event_dict = yield self.store.get_events( + old_room_create_event_ids.values(), + ) old_room_create_event = list(old_room_create_event_dict.values())[0] # Check if the create event specified a non-federatable room - if old_room_create_event.content.get("m.federate", True) == False: + if not old_room_create_event.content.get("m.federate", True): # If so, mark the new room as non-federatable as well creation_content["m.federate"] = False From 3ed3cb43394b41e76f4739f22760c1d8ebfed3c7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 18:21:39 +0000 Subject: [PATCH 18/96] New function for getting room's create event --- synapse/handlers/room.py | 8 +------- synapse/storage/state.py | 31 ++++++++++++++++++++----------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a69441b96f..5e40e9ea46 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -266,13 +266,7 @@ class RoomCreationHandler(BaseHandler): # Check if old room was non-federatable # Get old room's create event - old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( - old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), - ) - old_room_create_event_dict = yield self.store.get_events( - old_room_create_event_ids.values(), - ) - old_room_create_event = list(old_room_create_event_dict.values())[0] + old_room_create_event = yield self.store.get_create_event_for_room(old_room_id) # Check if the create event specified a non-federatable room if not old_room_create_event.content.get("m.federate", True): diff --git a/synapse/storage/state.py b/synapse/storage/state.py index c3ab7db7ae..522aaee918 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -24,7 +24,6 @@ import attr from twisted.internet import defer from synapse.api.constants import EventTypes -from synapse.api.errors import NotFoundError from synapse.storage._base import SQLBaseStore from synapse.storage.background_updates import BackgroundUpdateStore from synapse.storage.engines import PostgresEngine @@ -428,13 +427,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): """ # for now we do this by looking at the create event. We may want to cache this # more intelligently in future. - state_ids = yield self.get_current_state_ids(room_id) - create_id = state_ids.get((EventTypes.Create, "")) - if not create_id: - raise NotFoundError("Unknown room %s" % (room_id)) - - create_event = yield self.get_event(create_id) + # Retrieve the room's create event + create_event = yield self.get_create_event_for_room(room_id) defer.returnValue(create_event.content.get("room_version", "1")) @defer.inlineCallbacks @@ -448,6 +443,22 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: Deferred[unicode|None]: predecessor room id """ + # Retrieve the room's create event + create_event = yield self.get_create_event_for_room(room_id) + + # Return predecessor if present + defer.returnValue(create_event.content.get("predecessor", None)) + + @defer.inlineCallbacks + def get_create_event_for_room(self, room_id): + """Get the create state event for a room. + + Args: + room_id (str) + + Returns: + Deferred[EventBase|None]: The room creation event. None if can not be found + """ state_ids = yield self.get_current_state_ids(room_id) create_id = state_ids.get((EventTypes.Create, "")) @@ -455,11 +466,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): if not create_id: defer.returnValue(None) - # Retrieve the room's create event + # Retrieve the room's create event and return create_event = yield self.get_event(create_id) - - # Return predecessor if present - defer.returnValue(create_event.content.get("predecessor", None)) + defer.returnValue(create_event) @cached(max_entries=100000, iterable=True) def get_current_state_ids(self, room_id): From d239f67c25a31257a11852be08e6615e99423fe7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 18:34:15 +0000 Subject: [PATCH 19/96] Raise an exception instead of returning None --- synapse/storage/state.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 522aaee918..d14a7b2538 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -24,6 +24,7 @@ import attr from twisted.internet import defer from synapse.api.constants import EventTypes +from synapse.api.errors import NotFoundError from synapse.storage._base import SQLBaseStore from synapse.storage.background_updates import BackgroundUpdateStore from synapse.storage.engines import PostgresEngine @@ -442,6 +443,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: Deferred[unicode|None]: predecessor room id + + Raises: + NotFoundError if the room is unknown """ # Retrieve the room's create event create_event = yield self.get_create_event_for_room(room_id) @@ -457,14 +461,17 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): room_id (str) Returns: - Deferred[EventBase|None]: The room creation event. None if can not be found + Deferred[EventBase]: The room creation event. + + Raises: + NotFoundError if the room is unknown """ state_ids = yield self.get_current_state_ids(room_id) create_id = state_ids.get((EventTypes.Create, "")) # If we can't find the create event, assume we've hit a dead end if not create_id: - defer.returnValue(None) + raise NotFoundError("Unknown room %s" % (room_id)) # Retrieve the room's create event and return create_event = yield self.get_event(create_id) From f0ba34f581906e3809de6313bc48b696ac4ef07c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 1 Feb 2019 12:22:57 +0000 Subject: [PATCH 20/96] Fix noisy "twisted.internet.task.TaskStopped" errors in logs Fixes #4003 --- changelog.d/4546.bugfix | 1 + synapse/http/matrixfederationclient.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4546.bugfix diff --git a/changelog.d/4546.bugfix b/changelog.d/4546.bugfix new file mode 100644 index 0000000000..056f2848ed --- /dev/null +++ b/changelog.d/4546.bugfix @@ -0,0 +1 @@ +Fix noisy "twisted.internet.task.TaskStopped" errors in logs diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index bb2e64ed80..5ee4d528d2 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -28,7 +28,7 @@ from canonicaljson import encode_canonical_json from prometheus_client import Counter from signedjson.sign import sign_json -from twisted.internet import defer, protocol +from twisted.internet import defer, protocol, task from twisted.internet.error import DNSLookupError from twisted.internet.task import _EPSILON, Cooperator from twisted.web._newclient import ResponseDone @@ -286,7 +286,7 @@ class MatrixFederationHttpClient(object): json, ) data = encode_canonical_json(json) - producer = FileBodyProducer( + producer = QuieterFileBodyProducer( BytesIO(data), cooperator=self._cooperator, ) @@ -839,3 +839,16 @@ def encode_query_args(args): query_bytes = urllib.parse.urlencode(encoded_args, True) return query_bytes.encode('utf8') + + +class QuieterFileBodyProducer(FileBodyProducer): + """Wrapper for FileBodyProducer that avoids CRITICAL errors when the connection drops. + + Workaround for https://github.com/matrix-org/synapse/issues/4003 / + https://twistedmatrix.com/trac/ticket/6528 + """ + def stopProducing(self): + try: + FileBodyProducer.stopProducing(self) + except task.TaskStopped: + pass From a451d960cc389649a0f9cb7759801e82a6bf20e1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 1 Feb 2019 14:07:10 +0000 Subject: [PATCH 21/96] Add docs for ACME setup --- README.rst | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/README.rst b/README.rst index e6354ccba0..a3e384ae96 100644 --- a/README.rst +++ b/README.rst @@ -227,6 +227,85 @@ to read `Using a reverse proxy with Synapse`_ when doing so. Apart from port 8448 using TLS, both ports are the same in the default configuration. +ACME setup +---------- + +Synapse requires valid TLS certificates for communication between servers +(port ``8448`` by default) in addition to those that are client-facing (port +``443``). Synapse **will provision server-to-server certificates +automatically for you for free** through `Let's Encrypt +`_ if you tell it to. + + Note: Synapse does not currently hot-renew Let's Encrypt certificates for + you, it only checks for certificates that need renewing on restart. This + functionality will be implemented promptly, but if in the meantime your + federation certificates expire, simply restarting Synapse should renew + them automatically. + +In order for Synapse to complete the ACME challenge to provision a +certificate, it needs access to port 80. Typically listening on port 80 is +only granted to applications running as root. There are thus two solutions to +this problem. + +**Using a reverse proxy** + +A reverse proxy such as Apache or Nginx allows a single process (the web +server) to listen on port 80 and redirect traffic to the appropriate program +running on your server. It is the recommended method for setting up ACME as +it allows you to use your existing webserver while also allowing Synapse to +provision certificates as needed. + +For Nginx users, add the following line to your existing ``server`` block:: + + location /.well-known/acme-challenge { + proxy_pass http://localhost:8009/; + } + +For Apache, add the following to your existing webserver config:: + + ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge + +Make sure to restart/reload your webserver after making changes. + + +**Authbind** + +``authbind`` allows a program which does not or should not run as root to +bind to low-numbered ports in a controlled way. The setup is simpler, but +requires a webserver not to already be running on port 80. **This includes +every time Synapse renews a certificate**, which may be cumbersome if you +usually run a web server on port 80. Nevertheless, if you're sure port 80 is +not being used for any other purpose then all that is necessary is the +following: + +Install ``authbind``:: + + sudo apt-get install authbind + +Allow ``authbind`` to bind port 80:: + + sudo touch /etc/authbind/byport/80 + sudo chmod 777 /etc/authbind/byport/80 + +When Synapse is started (do not start it yet), use the following syntax:: + + # authbind syntax. don't start Synapse yet + authbind --deep + +If using the `Systemd`_ service file above, you can change the following line +from:: + + ExecStart=/home/matrix/matrix-synapse/bin/python -m synapse.app.homeserver + +to:: + + ExecStart=authbind --deep /home/matrix/matrix-synapse/bin/python -m synapse.app.homeserver + + +If you would like to use your own certificates, specifying them in Synapse's +config file is sufficient. + + Registering a user ------------------ From c5fc09322cacfc3fdd49a40ca99b105111dd2914 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 1 Feb 2019 14:10:44 +0000 Subject: [PATCH 22/96] Add changelog --- changelog.d/4547.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4547.misc diff --git a/changelog.d/4547.misc b/changelog.d/4547.misc new file mode 100644 index 0000000000..b6e421d095 --- /dev/null +++ b/changelog.d/4547.misc @@ -0,0 +1 @@ +Add docs for ACME setup to README. \ No newline at end of file From 57fe91f87b1db242d03e1dea278d4156ba65a60d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 1 Feb 2019 14:39:50 +0000 Subject: [PATCH 23/96] Clean up portions of docs that talk about reversing fed port --- README.rst | 121 +++++++++++------------------------------------------ 1 file changed, 24 insertions(+), 97 deletions(-) diff --git a/README.rst b/README.rst index a3e384ae96..248b5fbfe1 100644 --- a/README.rst +++ b/README.rst @@ -212,27 +212,26 @@ key in the ``.signing.key`` file (the second word) to something different. See `the spec`__ for more information on key management.) .. __: `key_management`_ - The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is configured without TLS; it should be behind a reverse proxy for TLS/SSL termination on port 443 which in turn should be used for clients. Port 8448 -is configured to use TLS with a self-signed certificate. If you would like -to do initial test with a client without having to setup a reverse proxy, -you can temporarly use another certificate. (Note that a self-signed -certificate is fine for `Federation`_). You can do so by changing -``tls_certificate_path`` and ``tls_private_key_path`` -in ``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure -to read `Using a reverse proxy with Synapse`_ when doing so. +is configured to use TLS for `Federation`_ with a self-signed or verified +certificate, but please be aware that a valid certificate will be required in +Synapse v1.0. -Apart from port 8448 using TLS, both ports are the same in the default -configuration. +If you would like to do initial testing with a client without having to setup +a reverse proxy, you can temporarly use another certificate. You can do so by +changing ``tls_certificate_path`` and ``tls_private_key_path`` in +``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure +to read `Using a reverse proxy with Synapse`_ when doing so. Apart from port +8448 using TLS, both ports are the same in the default configuration. ACME setup ---------- -Synapse requires valid TLS certificates for communication between servers +Synapse v1.0 requires valid TLS certificates for communication between servers (port ``8448`` by default) in addition to those that are client-facing (port -``443``). Synapse **will provision server-to-server certificates +``443``). Synapse v0.99.0+ **will provision server-to-server certificates automatically for you for free** through `Let's Encrypt `_ if you tell it to. @@ -287,23 +286,12 @@ Allow ``authbind`` to bind port 80:: sudo touch /etc/authbind/byport/80 sudo chmod 777 /etc/authbind/byport/80 -When Synapse is started (do not start it yet), use the following syntax:: +When Synapse is started, use the following syntax:: - # authbind syntax. don't start Synapse yet authbind --deep -If using the `Systemd`_ service file above, you can change the following line -from:: - - ExecStart=/home/matrix/matrix-synapse/bin/python -m synapse.app.homeserver - -to:: - - ExecStart=authbind --deep /home/matrix/matrix-synapse/bin/python -m synapse.app.homeserver - - -If you would like to use your own certificates, specifying them in Synapse's -config file is sufficient. +If you would like to use your own certificates, simply specify them in +``homeserver.yaml``. Registering a user @@ -360,10 +348,11 @@ following the recommended setup, or ``https://localhost:8448`` - remember to spe port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity server as the default - see `Identity servers`_.) -If using port 8448 you will run into errors until you accept the self-signed -certificate. You can easily do this by going to ``https://localhost:8448`` +If using port 8448 you will run into errors if you are using a self-signed +certificate. To overcome this, simply go to ``https://localhost:8448`` directly with your browser and accept the presented certificate. You can then -go back in your web client and proceed further. +go back in your web client and proceed further. Valid federation certificates +should not have this problem. If all goes well you should at least be able to log in, create a room, and start sending messages. @@ -632,9 +621,7 @@ you to run your server on a machine that might not have the same name as your domain name. For example, you might want to run your server at ``synapse.example.com``, but have your Matrix user-ids look like ``@user:example.com``. (A SRV record also allows you to change the port from -the default 8448. However, if you are thinking of using a reverse-proxy on the -federation port, which is not recommended, be sure to read -`Reverse-proxying the federation port`_ first.) +the default 8448. To use a SRV record, first create your SRV record and publish it in DNS. This should have the format ``_matrix._tcp. IN SRV 10 0 @@ -736,14 +723,10 @@ port. Indeed, clients will use port 443 by default, whereas servers default to port 8448. Where these are different, we refer to the 'client port' and the 'federation port'. -The next most important thing to know is that using a reverse-proxy on the -federation port has a number of pitfalls. It is possible, but be sure to read -`Reverse-proxying the federation port`_. - -The recommended setup is therefore to configure your reverse-proxy on port 443 -to port 8008 of synapse for client connections, but to also directly expose port -8448 for server-server connections. All the Matrix endpoints begin ``/_matrix``, -so an example nginx configuration might look like:: +The recommended setup is therefore to configure your reverse-proxy on port +443 to port 8008 of synapse for client connections, and port 8448 for +server-server connections. All Matrix endpoints begin with ``/_matrix``, so an +example nginx configuration might look like:: server { listen 443 ssl; @@ -784,63 +767,7 @@ Having done so, you can then use ``https://matrix.example.com`` (instead of ``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to Synapse from a client`_. -Reverse-proxying the federation port ------------------------------------- - -There are two issues to consider before using a reverse-proxy on the federation -port: - -* Due to the way SSL certificates are managed in the Matrix federation protocol - (see `spec`__), Synapse needs to be configured with the path to the SSL - certificate, *even if you do not terminate SSL at Synapse*. - - .. __: `key_management`_ - -* Until v0.33.3, Synapse did not support SNI on the federation port - (`bug #1491 `_). This bug - is now fixed, but means that federating with older servers can be unreliable - when using name-based virtual hosting. - -Furthermore, a number of the normal reasons for using a reverse-proxy do not -apply: - -* Other servers will connect on port 8448 by default, so there is no need to - listen on port 443 (for federation, at least), which avoids the need for root - privileges and virtual hosting. - -* A self-signed SSL certificate is fine for federation, so there is no need to - automate renewals. (The certificate generated by ``--generate-config`` is - valid for 10 years.) - -If you want to set up a reverse-proxy on the federation port despite these -caveats, you will need to do the following: - -* In ``homeserver.yaml``, set ``tls_certificate_path`` to the path to the SSL - certificate file used by your reverse-proxy, and set ``no_tls`` to ``True``. - (``tls_private_key_path`` will be ignored if ``no_tls`` is ``True``.) - -* In your reverse-proxy configuration: - - * If there are other virtual hosts on the same port, make sure that the - *default* one uses the certificate configured above. - - * Forward ``/_matrix`` to Synapse. - -* If your reverse-proxy is not listening on port 8448, publish a SRV record to - tell other servers how to find you. See `Setting up Federation`_. - -When updating the SSL certificate, just update the file pointed to by -``tls_certificate_path`` and then restart Synapse. (You may like to use a symbolic link -to help make this process atomic.) - -The most common mistake when setting up federation is not to tell Synapse about -your SSL certificate. To check it, you can visit -``https://matrix.org/federationtester/api/report?server_name=``. -Unfortunately, there is no UI for this yet, but, you should see -``"MatchingTLSFingerprint": true``. If not, check that -``Certificates[0].SHA256Fingerprint`` (the fingerprint of the certificate -presented by your reverse-proxy) matches ``Keys.tls_fingerprints[0].sha256`` -(the fingerprint of the certificate Synapse is using). +Please see `ACME setup`_ for details on reverse-proxying the federation port. Identity Servers From 7f914a2dbff902b25fbf01a33925329e6a00742e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 1 Feb 2019 14:52:39 +0000 Subject: [PATCH 24/96] Remove error and add link to foks fed tester project --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 248b5fbfe1..8181ff20c1 100644 --- a/README.rst +++ b/README.rst @@ -212,6 +212,7 @@ key in the ``.signing.key`` file (the second word) to something different. See `the spec`__ for more information on key management.) .. __: `key_management`_ + The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is configured without TLS; it should be behind a reverse proxy for TLS/SSL termination on port 443 which in turn should be used for clients. Port 8448 @@ -659,6 +660,8 @@ Troubleshooting You can use the federation tester to check if your homeserver is all set: ``https://matrix.org/federationtester/api/report?server_name=`` If any of the attributes under "checks" is false, federation won't work. +There is also a nicer interface available from a community member at +``_. The typical failure mode with federation is that when you try to join a room, it is rejected with "401: Unauthorized". Generally this means that other @@ -667,8 +670,6 @@ complicated dance which requires connections in both directions). So, things to check are: -* If you are trying to use a reverse-proxy, read `Reverse-proxying the - federation port`_. * If you are not using a SRV record, check that your ``server_name`` (the part of your user-id after the ``:``) matches your hostname, and that port 8448 on that hostname is reachable from outside your network. From 0d70288c595bf53a59963be89fcc35f60613c930 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 1 Feb 2019 15:38:09 +0000 Subject: [PATCH 25/96] Address changes --- README.rst | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 8181ff20c1..02828966c9 100644 --- a/README.rst +++ b/README.rst @@ -220,28 +220,19 @@ is configured to use TLS for `Federation`_ with a self-signed or verified certificate, but please be aware that a valid certificate will be required in Synapse v1.0. -If you would like to do initial testing with a client without having to setup -a reverse proxy, you can temporarly use another certificate. You can do so by -changing ``tls_certificate_path`` and ``tls_private_key_path`` in -``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure -to read `Using a reverse proxy with Synapse`_ when doing so. Apart from port -8448 using TLS, both ports are the same in the default configuration. ACME setup ---------- Synapse v1.0 requires valid TLS certificates for communication between servers (port ``8448`` by default) in addition to those that are client-facing (port -``443``). Synapse v0.99.0+ **will provision server-to-server certificates -automatically for you for free** through `Let's Encrypt +``443``). In the case that your `server_name` config variable is the same as +the hostname that the client connects to, then the same certificate can be +used between client and federation ports without issue. Synapse v0.99.0+ +**will provision server-to-server certificates automatically for you for +free** through `Let's Encrypt `_ if you tell it to. - Note: Synapse does not currently hot-renew Let's Encrypt certificates for - you, it only checks for certificates that need renewing on restart. This - functionality will be implemented promptly, but if in the meantime your - federation certificates expire, simply restarting Synapse should renew - them automatically. - In order for Synapse to complete the ACME challenge to provision a certificate, it needs access to port 80. Typically listening on port 80 is only granted to applications running as root. There are thus two solutions to @@ -250,7 +241,7 @@ this problem. **Using a reverse proxy** A reverse proxy such as Apache or Nginx allows a single process (the web -server) to listen on port 80 and redirect traffic to the appropriate program +server) to listen on port 80 and proxy traffic to the appropriate program running on your server. It is the recommended method for setting up ACME as it allows you to use your existing webserver while also allowing Synapse to provision certificates as needed. @@ -278,7 +269,7 @@ usually run a web server on port 80. Nevertheless, if you're sure port 80 is not being used for any other purpose then all that is necessary is the following: -Install ``authbind``:: +Install ``authbind``. For example, on Debian/Ubuntu:: sudo apt-get install authbind @@ -291,9 +282,11 @@ When Synapse is started, use the following syntax:: authbind --deep -If you would like to use your own certificates, simply specify them in -``homeserver.yaml``. - +If you would like to use your own certificates, you can do so by +changing ``tls_certificate_path`` and ``tls_private_key_path`` in +``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure +to read `Using a reverse proxy with Synapse`_ when doing so. Apart from port +8448 using TLS, both ports are the same in the default configuration. Registering a user ------------------ @@ -622,7 +615,7 @@ you to run your server on a machine that might not have the same name as your domain name. For example, you might want to run your server at ``synapse.example.com``, but have your Matrix user-ids look like ``@user:example.com``. (A SRV record also allows you to change the port from -the default 8448. +the default 8448). To use a SRV record, first create your SRV record and publish it in DNS. This should have the format ``_matrix._tcp. IN SRV 10 0 @@ -768,8 +761,6 @@ Having done so, you can then use ``https://matrix.example.com`` (instead of ``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to Synapse from a client`_. -Please see `ACME setup`_ for details on reverse-proxying the federation port. - Identity Servers ================ From 30fd2f89db67f1362904997cd328310afd7e6509 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 1 Feb 2019 15:52:28 +0000 Subject: [PATCH 26/96] 0.99.0rc4 --- CHANGES.md | 11 +++++++++++ changelog.d/4539.misc | 1 - changelog.d/4542.misc | 1 - changelog.d/4544.misc | 1 - synapse/__init__.py | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/4539.misc delete mode 100644 changelog.d/4542.misc delete mode 100644 changelog.d/4544.misc diff --git a/CHANGES.md b/CHANGES.md index 458bbaf118..66991340a4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +Synapse 0.99.0rc4 (2019-02-01) +============================== + +Internal Changes +---------------- + +- Update federation routing logic to check .well-known before SRV ([\#4539](https://github.com/matrix-org/synapse/issues/4539)) +- Improve performance of handling servers with invalid .well-known ([\#4542](https://github.com/matrix-org/synapse/issues/4542)) +- Treat an invalid .well-known file the same as an absent one ([\#4544](https://github.com/matrix-org/synapse/issues/4544)) + + Synapse 0.99.0rc3 (2019-01-31) ============================== diff --git a/changelog.d/4539.misc b/changelog.d/4539.misc deleted file mode 100644 index b222c8d23f..0000000000 --- a/changelog.d/4539.misc +++ /dev/null @@ -1 +0,0 @@ -Update federation routing logic to check .well-known before SRV diff --git a/changelog.d/4542.misc b/changelog.d/4542.misc deleted file mode 100644 index 74c84e0209..0000000000 --- a/changelog.d/4542.misc +++ /dev/null @@ -1 +0,0 @@ -Improve performance of handling servers with invalid .well-known diff --git a/changelog.d/4544.misc b/changelog.d/4544.misc deleted file mode 100644 index b29fc8575c..0000000000 --- a/changelog.d/4544.misc +++ /dev/null @@ -1 +0,0 @@ -Treat an invalid .well-known file the same as an absent one \ No newline at end of file diff --git a/synapse/__init__.py b/synapse/__init__.py index e5f680bb31..54745af283 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.0rc3" +__version__ = "0.99.0rc4" From 14de15eaa49a91178169fd8b49817360cecb4e97 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 1 Feb 2019 16:48:07 +0000 Subject: [PATCH 27/96] Actually need to enable it --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 02828966c9..34b65b7758 100644 --- a/README.rst +++ b/README.rst @@ -288,6 +288,14 @@ changing ``tls_certificate_path`` and ``tls_private_key_path`` in to read `Using a reverse proxy with Synapse`_ when doing so. Apart from port 8448 using TLS, both ports are the same in the default configuration. +Finally, once Synapse's is able to listen on port 80 for ACME challenge +requests, it must be told to perform ACME provisioning by setting ``enabled`` +to true under the ``acme`` section in ``homeserver.yaml``:: + + acme: + enabled: true + + Registering a user ------------------ From 897230f634ec14f3b3becdf3577231554ae5553e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 1 Feb 2019 16:54:32 +0000 Subject: [PATCH 28/96] Update README.rst Co-Authored-By: anoadragon453 <1342360+anoadragon453@users.noreply.github.com> --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 34b65b7758..4e34eaeee9 100644 --- a/README.rst +++ b/README.rst @@ -224,7 +224,7 @@ Synapse v1.0. ACME setup ---------- -Synapse v1.0 requires valid TLS certificates for communication between servers +Synapse v1.0 will require valid TLS certificates for communication between servers (port ``8448`` by default) in addition to those that are client-facing (port ``443``). In the case that your `server_name` config variable is the same as the hostname that the client connects to, then the same certificate can be From 9e89a420e841be1559dd1fc98c7843403cb82314 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 1 Feb 2019 16:54:47 +0000 Subject: [PATCH 29/96] Update README.rst Co-Authored-By: anoadragon453 <1342360+anoadragon453@users.noreply.github.com> --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4e34eaeee9..fcf828528d 100644 --- a/README.rst +++ b/README.rst @@ -261,7 +261,7 @@ Make sure to restart/reload your webserver after making changes. **Authbind** -``authbind`` allows a program which does not or should not run as root to +``authbind`` allows a program which does not not run as root to bind to low-numbered ports in a controlled way. The setup is simpler, but requires a webserver not to already be running on port 80. **This includes every time Synapse renews a certificate**, which may be cumbersome if you From 57164e17dace3c12706a2393385f6eefb6bc8c74 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 1 Feb 2019 16:59:06 +0000 Subject: [PATCH 30/96] Address comments --- README.rst | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/README.rst b/README.rst index fcf828528d..6e3ee01f2e 100644 --- a/README.rst +++ b/README.rst @@ -220,6 +220,11 @@ is configured to use TLS for `Federation`_ with a self-signed or verified certificate, but please be aware that a valid certificate will be required in Synapse v1.0. +If you would like to use your own certificates, you can do so by changing +``tls_certificate_path`` and ``tls_private_key_path`` in ``homeserver.yaml``; +alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, +both ports are the same in the default configuration. + ACME setup ---------- @@ -261,13 +266,12 @@ Make sure to restart/reload your webserver after making changes. **Authbind** -``authbind`` allows a program which does not not run as root to -bind to low-numbered ports in a controlled way. The setup is simpler, but -requires a webserver not to already be running on port 80. **This includes -every time Synapse renews a certificate**, which may be cumbersome if you -usually run a web server on port 80. Nevertheless, if you're sure port 80 is -not being used for any other purpose then all that is necessary is the -following: +``authbind`` allows a program which does not run as root to bind to +low-numbered ports in a controlled way. The setup is simpler, but requires a +webserver not to already be running on port 80. **This includes every time +Synapse renews a certificate**, which may be cumbersome if you usually run a +web server on port 80. Nevertheless, if you're sure port 80 is not being used +for any other purpose then all that is necessary is the following: Install ``authbind``. For example, on Debian/Ubuntu:: @@ -282,12 +286,6 @@ When Synapse is started, use the following syntax:: authbind --deep -If you would like to use your own certificates, you can do so by -changing ``tls_certificate_path`` and ``tls_private_key_path`` in -``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure -to read `Using a reverse proxy with Synapse`_ when doing so. Apart from port -8448 using TLS, both ports are the same in the default configuration. - Finally, once Synapse's is able to listen on port 80 for ACME challenge requests, it must be told to perform ACME provisioning by setting ``enabled`` to true under the ``acme`` section in ``homeserver.yaml``:: @@ -725,10 +723,8 @@ port. Indeed, clients will use port 443 by default, whereas servers default to port 8448. Where these are different, we refer to the 'client port' and the 'federation port'. -The recommended setup is therefore to configure your reverse-proxy on port -443 to port 8008 of synapse for client connections, and port 8448 for -server-server connections. All Matrix endpoints begin with ``/_matrix``, so an -example nginx configuration might look like:: +All Matrix endpoints begin with ``/_matrix``, so an example nginx +configuration for forwarding client connections to Synapse might look like:: server { listen 443 ssl; From da6df65e195bd1a29b2d186d4ed4c988ca4f1f34 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 1 Feb 2019 16:59:23 +0000 Subject: [PATCH 31/96] Fix nginx capatilization --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6e3ee01f2e..9e3d85de4c 100644 --- a/README.rst +++ b/README.rst @@ -245,13 +245,13 @@ this problem. **Using a reverse proxy** -A reverse proxy such as Apache or Nginx allows a single process (the web +A reverse proxy such as Apache or nginx allows a single process (the web server) to listen on port 80 and proxy traffic to the appropriate program running on your server. It is the recommended method for setting up ACME as it allows you to use your existing webserver while also allowing Synapse to provision certificates as needed. -For Nginx users, add the following line to your existing ``server`` block:: +For nginx users, add the following line to your existing ``server`` block:: location /.well-known/acme-challenge { proxy_pass http://localhost:8009/; From d7e27a1f08aefb020d1158d4b8210e995dcdac78 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 11:32:45 +0000 Subject: [PATCH 32/96] fix typo in config comments (#4557) --- changelog.d/4557.misc | 1 + synapse/config/tls.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4557.misc diff --git a/changelog.d/4557.misc b/changelog.d/4557.misc new file mode 100644 index 0000000000..7ebfb23f43 --- /dev/null +++ b/changelog.d/4557.misc @@ -0,0 +1 @@ +Fix comment typo in TLS section of config diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 5f63676d9c..9498fc5f5b 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -199,10 +199,10 @@ class TlsConfig(Config): # If your server runs behind a reverse-proxy which terminates TLS connections # (for both client and federation connections), it may be useful to disable - # All TLS support for incoming connections. Setting no_tls to False will + # All TLS support for incoming connections. Setting no_tls to True will # do so (and avoid the need to give synapse a TLS private key). # - # no_tls: False + # no_tls: True # List of allowed TLS fingerprints for this server to publish along # with the signing keys for this server. Other matrix servers that From 9a75c0b52e6609809522ae47753c9a93d86f5128 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 11:33:26 +0000 Subject: [PATCH 33/96] switch docker image to py3 by default (#4558) Switch the matrixdotorg/synapse:latest Docker image to use python 3 --- .circleci/config.yml | 8 ++++---- changelog.d/4558.feature | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/4558.feature diff --git a/.circleci/config.yml b/.circleci/config.yml index 697e6c577f..ee72ad7a14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,8 +4,8 @@ jobs: machine: true steps: - checkout - - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} . - - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 . + - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py2 . + - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 . - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - run: docker push matrixdotorg/synapse:${CIRCLE_TAG} - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3 @@ -13,8 +13,8 @@ jobs: machine: true steps: - checkout - - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest . - - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 . + - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py2 . + - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 . - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - run: docker push matrixdotorg/synapse:latest - run: docker push matrixdotorg/synapse:latest-py3 diff --git a/changelog.d/4558.feature b/changelog.d/4558.feature new file mode 100644 index 0000000000..40724d1c3f --- /dev/null +++ b/changelog.d/4558.feature @@ -0,0 +1 @@ +The matrixdotorg/synapse Docker images now use Python 3 by default. \ No newline at end of file From bf1e4d96ad22f246402241ef6dcabdebede50e10 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 11:37:33 +0000 Subject: [PATCH 34/96] Fix default ACME config for py2 (#4564) Fixes #4559 --- changelog.d/4564.bugfix | 1 + synapse/config/tls.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4564.bugfix diff --git a/changelog.d/4564.bugfix b/changelog.d/4564.bugfix new file mode 100644 index 0000000000..78e78504b4 --- /dev/null +++ b/changelog.d/4564.bugfix @@ -0,0 +1 @@ +Fix default ACME config for py2 diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 9498fc5f5b..b5f2cfd9b7 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -37,7 +37,7 @@ class TlsConfig(Config): self.acme_enabled = acme_config.get("enabled", False) self.acme_url = acme_config.get( - "url", "https://acme-v01.api.letsencrypt.org/directory" + "url", u"https://acme-v01.api.letsencrypt.org/directory" ) self.acme_port = acme_config.get("port", 80) self.acme_bind_addresses = acme_config.get("bind_addresses", ['::', '0.0.0.0']) From 3ef71a6ea057c7ad8b87933f06861e9973660b51 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 11:44:40 +0000 Subject: [PATCH 35/96] Docker: only copy what we need to the build image (#4562) There are two reasons this is a good thing: * first, it means that you don't end up with stuff kicking around your working copy ending up in the build image by mistake (which can upset the pip install process) * second: it means that the docker image cache is more effective, and we can reuse docker images when iterating on the docker stuff. --- changelog.d/4562.misc | 1 + docker/Dockerfile | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4562.misc diff --git a/changelog.d/4562.misc b/changelog.d/4562.misc new file mode 100644 index 0000000000..f7185fa768 --- /dev/null +++ b/changelog.d/4562.misc @@ -0,0 +1 @@ +Docker: only copy what we need to the build image diff --git a/docker/Dockerfile b/docker/Dockerfile index 4b739e7d02..d212334844 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -31,7 +31,10 @@ RUN pip install --prefix="/install" --no-warn-script-location \ # now install synapse and all of the python deps to /install. -COPY . /synapse +COPY synapse /synapse/synapse/ +COPY scripts /synapse/scripts/ +COPY MANIFEST.in README.rst setup.py synctl /synapse/ + RUN pip install --prefix="/install" --no-warn-script-location \ /synapse[all] From 627ecd358eda66f81afd676523f520762571728c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 5 Feb 2019 12:16:28 +0000 Subject: [PATCH 36/96] Filter user directory state query to a subset of state events (#4462) * Filter user directory state query to a subset of state events * Add changelog --- changelog.d/4462.misc | 1 + synapse/storage/user_directory.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 changelog.d/4462.misc diff --git a/changelog.d/4462.misc b/changelog.d/4462.misc new file mode 100644 index 0000000000..03a4d7ae1c --- /dev/null +++ b/changelog.d/4462.misc @@ -0,0 +1 @@ +Change the user directory state query to use a filtered call to the db instead of a generic one. \ No newline at end of file diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index ce48212265..e8b574ee5e 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -22,6 +22,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, JoinRules from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.state import StateFilter from synapse.types import get_domain_from_id, get_localpart_from_id from synapse.util.caches.descriptors import cached, cachedInlineCallbacks @@ -31,12 +32,19 @@ logger = logging.getLogger(__name__) class UserDirectoryStore(SQLBaseStore): - @cachedInlineCallbacks(cache_context=True) - def is_room_world_readable_or_publicly_joinable(self, room_id, cache_context): + @defer.inlineCallbacks + def is_room_world_readable_or_publicly_joinable(self, room_id): """Check if the room is either world_readable or publically joinable """ - current_state_ids = yield self.get_current_state_ids( - room_id, on_invalidate=cache_context.invalidate + + # Create a state filter that only queries join and history state event + types_to_filter = ( + (EventTypes.JoinRules, ""), + (EventTypes.RoomHistoryVisibility, ""), + ) + + current_state_ids = yield self.get_filtered_current_state_ids( + room_id, StateFilter.from_types(types_to_filter), ) join_rules_id = current_state_ids.get((EventTypes.JoinRules, "")) From 40b35fb87516f461ae562b247ab13a80f57beede Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 13:42:21 +0000 Subject: [PATCH 37/96] Enable ACME support in the docker image (#4566) Also: * Fix wrapping in docker readme * Clean up some docs on the docker image * a workaround for #4554 --- changelog.d/4566.feature | 1 + docker/Dockerfile | 15 ++++++- docker/README.md | 88 +++++++++++++++++++++++-------------- docker/conf/dummy.tls.crt | 17 +++++++ docker/conf/homeserver.yaml | 18 +++++++- docker/start.py | 18 ++++++-- 6 files changed, 116 insertions(+), 41 deletions(-) create mode 100644 changelog.d/4566.feature create mode 100644 docker/conf/dummy.tls.crt diff --git a/changelog.d/4566.feature b/changelog.d/4566.feature new file mode 100644 index 0000000000..11fc07476e --- /dev/null +++ b/changelog.d/4566.feature @@ -0,0 +1 @@ +enable ACME support in the docker image diff --git a/docker/Dockerfile b/docker/Dockerfile index d212334844..c35da67a2a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,16 @@ +# Dockerfile to build the matrixdotorg/synapse docker images. +# +# To build the image, run `docker build` command from the root of the +# synapse repository: +# +# docker build -f docker/Dockerfile . +# +# There is an optional PYTHON_VERSION build argument which sets the +# version of python to build against: for example: +# +# docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 . +# + ARG PYTHON_VERSION=2 ### @@ -59,6 +72,6 @@ COPY ./docker/conf /conf VOLUME ["/data"] -EXPOSE 8008/tcp 8448/tcp +EXPOSE 8008/tcp 8009/tcp 8448/tcp ENTRYPOINT ["/start.py"] diff --git a/docker/README.md b/docker/README.md index 3c00d1e948..3faedf629f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,22 +1,21 @@ # Synapse Docker -This Docker image will run Synapse as a single process. It does not provide a database -server or a TURN server, you should run these separately. +This Docker image will run Synapse as a single process. By default it uses a +sqlite database; for production use you should connect it to a separate +postgres database. + +The image also does *not* provide a TURN server. ## Run -We do not currently offer a `latest` image, as this has somewhat undefined semantics. -We instead release only tagged versions so upgrading between releases is entirely -within your control. - ### Using docker-compose (easier) -This image is designed to run either with an automatically generated configuration -file or with a custom configuration that requires manual editing. +This image is designed to run either with an automatically generated +configuration file or with a custom configuration that requires manual editing. An easy way to make use of this image is via docker-compose. See the -[contrib/docker](../contrib/docker) -section of the synapse project for examples. +[contrib/docker](../contrib/docker) section of the synapse project for +examples. ### Without Compose (harder) @@ -32,7 +31,7 @@ docker run \ -v ${DATA_PATH}:/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ -e SYNAPSE_REPORT_STATS=yes \ - docker.io/matrixdotorg/synapse:latest + matrixdotorg/synapse:latest ``` ## Volumes @@ -53,6 +52,28 @@ In order to setup an application service, simply create an ``appservices`` directory in the data volume and write the application service Yaml configuration file there. Multiple application services are supported. +## TLS certificates + +Synapse requires a valid TLS certificate. You can do one of the following: + + * Provide your own certificate and key (as + `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.crt` and + `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.key`, or elsewhere by providing an + entire config as `${SYNAPSE_CONFIG_PATH}`). + + * Use a reverse proxy to terminate incoming TLS, and forward the plain http + traffic to port 8008 in the container. In this case you should set `-e + SYNAPSE_NO_TLS=1`. + + * Use the ACME (Let's Encrypt) support built into Synapse. This requires + `${SYNAPSE_SERVER_NAME}` port 80 to be forwarded to port 8009 in the + container, for example with `-p 80:8009`. To enable it in the docker + container, set `-e SYNAPSE_ACME=1`. + +If you don't do any of these, Synapse will fail to start with an error similar to: + + synapse.config._base.ConfigError: Error accessing file '/data/.tls.crt' (config for tls_certificate): No such file or directory + ## Environment Unless you specify a custom path for the configuration file, a very generic @@ -71,7 +92,7 @@ then customize it manually. No other environment variable is required. Otherwise, a dynamic configuration file will be used. The following environment variables are available for configuration: -* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. +* ``SYNAPSE_SERVER_NAME`` (mandatory), the server public hostname. * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if @@ -80,7 +101,6 @@ variables are available for configuration: the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. -* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public key in order to enable recaptcha upon registration. * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private @@ -88,7 +108,9 @@ variables are available for configuration: * ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN uris to enable TURN for this homeserver. * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. -* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`]. +* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size + [default `10M`]. +* ``SYNAPSE_ACME``: set this to enable the ACME certificate renewal support. Shared secrets, that will be initialized to random values if not set: @@ -99,27 +121,25 @@ Shared secrets, that will be initialized to random values if not set: Database specific values (will use SQLite if not set): -* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`] -* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] -* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. -* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] +* `POSTGRES_DB` - The database name for the synapse postgres + database. [default: `synapse`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use + postgresql instead of sqlite3. [default: `db` which is useful when using a + container on the same docker network in a compose file where the postgres + service is called `db`] +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If + this is set then postgres will be used instead of sqlite3.** [default: none] + **NOTE**: You are highly encouraged to use postgresql! Please use the compose + file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: + `matrix`] Mail server specific values (will not send emails if not set): * ``SYNAPSE_SMTP_HOST``, hostname to the mail server. -* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``]. -* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any. -* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any. - -## Build - -Build the docker image with the `docker build` command from the root of the synapse repository. - -``` -docker build -t docker.io/matrixdotorg/synapse . -f docker/Dockerfile -``` - -The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. - -You may have a local Python wheel cache available, in which case copy the relevant -packages in the ``cache/`` directory at the root of the project. +* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default + ``25``]. +* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if + any. +* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail + server if any. diff --git a/docker/conf/dummy.tls.crt b/docker/conf/dummy.tls.crt new file mode 100644 index 0000000000..8e3b1a9aaa --- /dev/null +++ b/docker/conf/dummy.tls.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICnTCCAYUCAgPoMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMMCWxvY2FsaG9z +dDAeFw0xOTAxMTUwMDQxNTBaFw0yOTAxMTIwMDQxNTBaMBQxEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMKqm81/8j5d +R1s7VZ8ueg12gJrPVCCAOkp0UnuC/ZlXhN0HTvnhQ+B0IlSgB4CcQZyf4jnA6o4M +rwSc7VX0MPE9x/idoA0g/0WoC6tsxugOrvbzCw8Tv+fnXglm6uVc7aFPfx69wU3q +lUHGD/8jtEoHxmCG177Pt2lHAfiVLBAyMQGtETzxt/yAfkloaybe316qoljgK5WK +cokdAt9G84EEqxNeEnx5FG3Vc100bAqJS4GvQlFgtF9KFEqZKEyB1yKBpPMDfPIS +V9hIV0gswSmYI8dpyBlGf5lPElY68ZGABmOQgr0RI5qHK/h28OpFPE0q3v4AMHgZ +I36wii4NrAUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfD8kcpZ+dn08xh1qtKtp +X+/YNZaOBIeVdlCzfoZKNblSFAFD/jCfObNJYvZMUQ8NX2UtEJp1lTA6m7ltSsdY +gpC2k1VD8iN+ooXklJmL0kxc7UUqho8I0l9vn35h+lhLF0ihT6XfZVi/lDHWl+4G +rG+v9oxvCSCWrNWLearSlFPtQQ8xPtOE0nLwfXtOI/H/2kOuC38ihaIWM4jjbWXK +E/ksgUfuDv0mFiwf1YdBF5/M3/qOowqzU8HgMJ3WoT/9Po5Ya1pWc+3BcxxytUDf +XdMu0tWHKX84tZxLcR1nZHzluyvFFM8xNtLi9xV0Z7WbfT76V0C/ulEOybGInYsv +nQ== +-----END CERTIFICATE----- diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index 529118d184..f07d5c1001 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -2,10 +2,24 @@ ## TLS ## +{% if SYNAPSE_NO_TLS %} +no_tls: True + +# workaround for https://github.com/matrix-org/synapse/issues/4554 +tls_certificate_path: "/conf/dummy.tls.crt" + +{% else %} + tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" -no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }} -tls_fingerprints: [] + +{% if SYNAPSE_ACME %} +acme: + enabled: true + port: 8009 +{% endif %} + +{% endif %} ## Server ## diff --git a/docker/start.py b/docker/start.py index 346df8c87f..941d9996a8 100755 --- a/docker/start.py +++ b/docker/start.py @@ -47,9 +47,8 @@ if mode == "generate": # In normal mode, generate missing keys if any, then run synapse else: - # Parse the configuration file if "SYNAPSE_CONFIG_PATH" in environ: - args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] + config_path = environ["SYNAPSE_CONFIG_PATH"] else: check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) generate_secrets(environ, { @@ -58,10 +57,21 @@ else: }) environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") - convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) + + config_path = "/compiled/homeserver.yaml" + + convert("/conf/homeserver.yaml", config_path, environ) convert("/conf/log.config", "/compiled/log.config", environ) subprocess.check_output(["chown", "-R", ownership, "/data"]) - args += ["--config-path", "/compiled/homeserver.yaml"] + + + args += [ + "--config-path", config_path, + + # tell synapse to put any generated keys in /data rather than /compiled + "--keys-directory", "/data", + ] + # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) os.execv("/sbin/su-exec", ["su-exec", ownership] + args) From 4561f3baa02dfc6a2858aa5ea3e7d21b7825ccb2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 13:55:21 +0000 Subject: [PATCH 38/96] Move things from README.rst to UPDATE.md (#4569) The readme was getting pretty unmanageable and hard to grok. This is an attempt to simplify things by moving installation instructions from the README to a separate file. I've tried to resist the temptation to fix too much stuff while I'm here - it mostly just copies-and-pastes from one doc to the other, and changes from rst to md syntax. --- INSTALL.md | 487 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 445 +----------------------------------------------- 2 files changed, 490 insertions(+), 442 deletions(-) create mode 100644 INSTALL.md diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000000..a04524cdc7 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,487 @@ +* [Installing Synapse](#installing-synapse) + * [Installing from source](#installing-from-source) + * [Platform-Specific Instructions](#platform-specific-instructions) + * [Troubleshooting Installation](#troubleshooting-installation) + * [Prebuilt packages](#prebuilt-packages) +* [Setting up Synapse](#setting-up-synapse) + * [TLS certificates](#tls-certificates) + * [Registering a user](#registering-a-user) + * [Setting up a TURN server](#setting-up-a-turn-server) + * [URL previews](#url-previews) + +# Installing Synapse + +## Installing from source + +(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).) + +System requirements: + +- POSIX-compliant system (tested on Linux & OS X) +- Python 3.5, 3.6, 3.7, or 2.7 +- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org + +Synapse is written in Python but some of the libraries it uses are written in +C. So before we can install Synapse itself we need a working C compiler and the +header files for Python C extensions. See [Platform-Specific +Instructions](#platform-specific-instructions) for information on installing +these on various platforms. + +To install the Synapse homeserver run: + +``` +mkdir -p ~/synapse +virtualenv -p python3 ~/synapse/env +source ~/synapse/env/bin/activate +pip install --upgrade pip +pip install --upgrade setuptools +pip install matrix-synapse[all] +``` + +This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse) +and install it, along with the python libraries it uses, into a virtual environment +under ``~/synapse/env``. Feel free to pick a different directory if you +prefer. + +This Synapse installation can then be later upgraded by using pip again with the +update flag: + +``` +source ~/synapse/env/bin/activate +pip install -U matrix-synapse[all] +``` + +Before you can start Synapse, you will need to generate a configuration +file. To do this, run (in your virtualenv, as before):: + +``` +cd ~/synapse +python -m synapse.app.homeserver \ + --server-name my.domain.name \ + --config-path homeserver.yaml \ + --generate-config \ + --report-stats=[yes|no] +``` + +... substituting an appropriate value for `--server-name`. The server name +determines the "domain" part of user-ids for users on your server: these will +all be of the format `@user:my.domain.name`. It also determines how other +matrix servers will reach yours for Federation. For a test configuration, +set this to the hostname of your server. For a more production-ready setup, you +will probably want to specify your domain (`example.com`) rather than a +matrix-specific hostname here (in the same way that your email address is +probably `user@example.com` rather than `user@email.example.com`) - but +doing so may require more advanced setup. - see [Setting up Federation](README.rst#setting-up-federation). Beware that the server name cannot be changed later. + +This command will generate you a config file that you can then customise, but it will +also generate a set of keys for you. These keys will allow your Home Server to +identify itself to other Home Servers, so don't lose or delete them. It would be +wise to back them up somewhere safe. (If, for whatever reason, you do need to +change your Home Server's keys, you may find that other Home Servers have the +old key cached. If you update the signing key, you should change the name of the +key in the `.signing.key` file (the second word) to something +different. See the +[spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys) +for more information on key management.) + +You will need to give Synapse a TLS certficate before it will start - see [TLS +certificates](#tls-certificates). + +To actually run your new homeserver, pick a working directory for Synapse to +run (e.g. ``~/synapse``), and:: + + cd ~/synapse + source env/bin/activate + synctl start + +### Platform-Specific Instructions + +#### Debian/Ubuntu/Raspbian + +Installing prerequisites on Ubuntu or Debian: + +``` +sudo apt-get install build-essential python3-dev libffi-dev \ + python-pip python-setuptools sqlite3 \ + libssl-dev python-virtualenv libjpeg-dev libxslt1-dev +``` + +#### ArchLinux + +Installing prerequisites on ArchLinux: + +``` +sudo pacman -S base-devel python python-pip \ + python-setuptools python-virtualenv sqlite3 +``` + +#### CentOS/Fedora + +Installing prerequisites on CentOS 7 or Fedora 25: + +``` +sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \ + lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \ + python-virtualenv libffi-devel openssl-devel +sudo yum groupinstall "Development Tools" +``` + +#### Mac OS X + +Installing prerequisites on Mac OS X: + +``` +xcode-select --install +sudo easy_install pip +sudo pip install virtualenv +brew install pkg-config libffi +``` + +#### OpenSUSE + +Installing prerequisites on openSUSE: + +``` +sudo zypper in -t pattern devel_basis +sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \ + python-devel libffi-devel libopenssl-devel libjpeg62-devel +``` + +#### OpenBSD + +Installing prerequisites on OpenBSD: + +``` +doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \ + libxslt jpeg +``` + +There is currently no port for OpenBSD. Additionally, OpenBSD's security +settings require a slightly more difficult installation process. + +XXX: I suspect this is out of date. + +1. Create a new directory in `/usr/local` called `_synapse`. Also, create a + new user called `_synapse` and set that directory as the new user's home. + This is required because, by default, OpenBSD only allows binaries which need + write and execute permissions on the same memory space to be run from + `/usr/local`. +2. `su` to the new `_synapse` user and change to their home directory. +3. Create a new virtualenv: `virtualenv -p python2.7 ~/.synapse` +4. Source the virtualenv configuration located at + `/usr/local/_synapse/.synapse/bin/activate`. This is done in `ksh` by + using the `.` command, rather than `bash`'s `source`. +5. Optionally, use `pip` to install `lxml`, which Synapse needs to parse + webpages for their titles. +6. Use `pip` to install this repository: `pip install matrix-synapse` +7. Optionally, change `_synapse`'s shell to `/bin/false` to reduce the + chance of a compromised Synapse server being used to take over your box. + +After this, you may proceed with the rest of the install directions. + +#### Windows + +If you wish to run or develop Synapse on Windows, the Windows Subsystem For +Linux provides a Linux environment on Windows 10 which is capable of using the +Debian, Fedora, or source installation methods. More information about WSL can +be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for +Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server +for Windows Server. + +### Troubleshooting Installation + +XXX a bunch of this is no longer relevant. + +Synapse requires pip 8 or later, so if your OS provides too old a version you +may need to manually upgrade it:: + + sudo pip install --upgrade pip + +Installing may fail with ``Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)``. +You can fix this by manually upgrading pip and virtualenv:: + + sudo pip install --upgrade virtualenv + +You can next rerun ``virtualenv -p python3 synapse`` to update the virtual env. + +Installing may fail during installing virtualenv with ``InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.`` +You can fix this by manually installing ndg-httpsclient:: + + pip install --upgrade ndg-httpsclient + +Installing may fail with ``mock requires setuptools>=17.1. Aborting installation``. +You can fix this by upgrading setuptools:: + + pip install --upgrade setuptools + +If pip crashes mid-installation for reason (e.g. lost terminal), pip may +refuse to run until you remove the temporary installation directory it +created. To reset the installation:: + + rm -rf /tmp/pip_install_matrix + +pip seems to leak *lots* of memory during installation. For instance, a Linux +host with 512MB of RAM may run out of memory whilst installing Twisted. If this +happens, you will have to individually install the dependencies which are +failing, e.g.:: + + pip install twisted + +## Prebuilt packages + +As an alternative to installing from source, prebuilt packages are available +for a number of platforms. + +### Docker images and Ansible playbooks + +There is an offical synapse image available at +https://hub.docker.com/r/matrixdotorg/synapse which can be used with +the docker-compose file available at [contrib/docker](contrib/docker). Further information on +this including configuration options is available in the README on +hub.docker.com. + +Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a +Dockerfile to automate a synapse server in a single Docker image, at +https://hub.docker.com/r/avhost/docker-matrix/tags/ + +Slavi Pantaleev has created an Ansible playbook, +which installs the offical Docker image of Matrix Synapse +along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.). +For more details, see +https://github.com/spantaleev/matrix-docker-ansible-deploy + + +### Debian/Ubuntu + +#### Matrix.org packages + +Matrix.org provides Debian/Ubuntu packages of the latest stable version of +Synapse via https://matrix.org/packages/debian/. To use them: + +``` +sudo apt install -y lsb-release curl apt-transport-https +echo "deb https://matrix.org/packages/debian `lsb_release -cs` main" | + sudo tee /etc/apt/sources.list.d/matrix-org.list +curl "https://matrix.org/packages/debian/repo-key.asc" | + sudo apt-key add - +sudo apt update +sudo apt install matrix-synapse-py3 +``` + +#### Downstream Debian/Ubuntu packages + +For `buster` and `sid`, Synapse is available in the Debian repositories and +it should be possible to install it with simply: + +``` + sudo apt install matrix-synapse +``` + +There is also a version of `matrix-synapse` in `stretch-backports`. Please see +the [Debian documentation on +backports](https://backports.debian.org/Instructions/) for information on how +to use them. + +We do not recommend using the packages in downstream Ubuntu at this time, as +they are old and suffer from known security vulnerabilities. + +### Fedora + +Synapse is in the Fedora repositories as `matrix-synapse`: + +``` +sudo dnf install matrix-synapse +``` + +Oleg Girko provides Fedora RPMs at +https://obs.infoserver.lv/project/monitor/matrix-synapse + +### OpenSUSE + +Synapse is in the OpenSUSE repositories as `matrix-synapse`: + +``` +sudo zypper install matrix-synapse +``` + +### SUSE Linux Enterprise Server + +Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at +https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/ + +### ArchLinux + +The quickest way to get up and running with ArchLinux is probably with the community package +https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of +the necessary dependencies. + +pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ): + +``` +sudo pip install --upgrade pip +``` + +If you encounter an error with lib bcrypt causing an Wrong ELF Class: +ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly +compile it under the right architecture. (This should not be needed if +installing under virtualenv): + +``` +sudo pip uninstall py-bcrypt +sudo pip install py-bcrypt +``` + +### FreeBSD + +Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from: + + - Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean` + - Packages: `pkg install py27-matrix-synapse` + + +### NixOS + +Robin Lambertz has packaged Synapse for NixOS at: +https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix + +# Setting up Synapse + +Once you have installed synapse as above, you will need to configure it. + +## TLS certificates + +The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is +configured without TLS; it should be behind a reverse proxy for TLS/SSL +termination on port 443 which in turn should be used for clients. Port 8448 +is configured to use TLS for Federation with a self-signed or verified +certificate, but please be aware that a valid certificate will be required in +Synapse v1.0. + +If you would like to use your own certificates, you can do so by changing +`tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`; +alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, +both ports are the same in the default configuration. + +### ACME setup + +Synapse v1.0 will require valid TLS certificates for communication between servers +(port `8448` by default) in addition to those that are client-facing (port +`443`). In the case that your `server_name` config variable is the same as +the hostname that the client connects to, then the same certificate can be +used between client and federation ports without issue. Synapse v0.99.0+ +**will provision server-to-server certificates automatically for you for +free** through [Let's Encrypt](https://letsencrypt.org/) if you tell it to. + +In order for Synapse to complete the ACME challenge to provision a +certificate, it needs access to port 80. Typically listening on port 80 is +only granted to applications running as root. There are thus two solutions to +this problem. + +#### Using a reverse proxy + +A reverse proxy such as Apache or nginx allows a single process (the web +server) to listen on port 80 and proxy traffic to the appropriate program +running on your server. It is the recommended method for setting up ACME as +it allows you to use your existing webserver while also allowing Synapse to +provision certificates as needed. + +For nginx users, add the following line to your existing `server` block: + +``` +location /.well-known/acme-challenge { + proxy_pass http://localhost:8009/; +} +``` + +For Apache, add the following to your existing webserver config:: + +``` +ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge +``` + +Make sure to restart/reload your webserver after making changes. + + +#### Authbind + +`authbind` allows a program which does not run as root to bind to +low-numbered ports in a controlled way. The setup is simpler, but requires a +webserver not to already be running on port 80. **This includes every time +Synapse renews a certificate**, which may be cumbersome if you usually run a +web server on port 80. Nevertheless, if you're sure port 80 is not being used +for any other purpose then all that is necessary is the following: + +Install `authbind`. For example, on Debian/Ubuntu: + +``` +sudo apt-get install authbind +``` + +Allow `authbind` to bind port 80: + +``` +sudo touch /etc/authbind/byport/80 +sudo chmod 777 /etc/authbind/byport/80 +``` + +When Synapse is started, use the following syntax:: + +``` +authbind --deep +``` + +Finally, once Synapse is able to listen on port 80 for ACME challenge +requests, it must be told to perform ACME provisioning by setting `enabled` +to true under the `acme` section in `homeserver.yaml`: + +``` +acme: + enabled: true +``` + +## Registering a user + +You will need at least one user on your server in order to use a Matrix +client. Users can be registered either via a Matrix client, or via a +commandline script. + +To get started, it is easiest to use the command line to register new +users. This can be done as follows: + +``` +$ source ~/synapse/env/bin/activate +$ synctl start # if not already running +$ register_new_matrix_user -c homeserver.yaml https://localhost:8448 +New user localpart: erikj +Password: +Confirm password: +Make admin [no]: +Success! +``` + +This process uses a setting ``registration_shared_secret`` in +``homeserver.yaml``, which is shared between Synapse itself and the +``register_new_matrix_user`` script. It doesn't matter what it is (a random +value is generated by ``--generate-config``), but it should be kept secret, as +anyone with knowledge of it can register users on your server even if +``enable_registration`` is ``false``. + +## Setting up a TURN server + +For reliable VoIP calls to be routed via this homeserver, you MUST configure +a TURN server. See [docs/turn-howto.rst](docs/turn-howto.rst) for details. + +## URL previews + +Synapse includes support for previewing URLs, which is disabled by default. To +turn it on you must enable the ``url_preview_enabled: True`` config parameter +and explicitly specify the IP ranges that Synapse is not allowed to spider for +previewing in the ``url_preview_ip_range_blacklist`` configuration parameter. +This is critical from a security perspective to stop arbitrary Matrix users +spidering 'internal' URLs on your network. At the very least we recommend that +your loopback and RFC1918 IP addresses are blacklisted. + +This also requires the optional lxml and netaddr python dependencies to be +installed. This in turn requires the libxml2 library to be available - on +Debian/Ubuntu this means ``apt-get install libxml2-dev``, or equivalent for +your OS. diff --git a/README.rst b/README.rst index 9e3d85de4c..8c5220be88 100644 --- a/README.rst +++ b/README.rst @@ -81,261 +81,8 @@ Thanks for using Matrix! Synapse Installation ==================== -Synapse is the reference Python/Twisted Matrix homeserver implementation. +For details on how to install synapse, see ``_. -System requirements: - -- POSIX-compliant system (tested on Linux & OS X) -- Python 3.5, 3.6, 3.7, or 2.7 -- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org - -Installing from source ----------------------- - -(Prebuilt packages are available for some platforms - see `Platform-Specific -Instructions`_.) - -Synapse is written in Python but some of the libraries it uses are written in -C. So before we can install Synapse itself we need a working C compiler and the -header files for Python C extensions. - -Installing prerequisites on Ubuntu or Debian:: - - sudo apt-get install build-essential python3-dev libffi-dev \ - python-pip python-setuptools sqlite3 \ - libssl-dev python-virtualenv libjpeg-dev libxslt1-dev - -Installing prerequisites on ArchLinux:: - - sudo pacman -S base-devel python python-pip \ - python-setuptools python-virtualenv sqlite3 - -Installing prerequisites on CentOS 7 or Fedora 25:: - - sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \ - lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \ - python-virtualenv libffi-devel openssl-devel - sudo yum groupinstall "Development Tools" - -Installing prerequisites on Mac OS X:: - - xcode-select --install - sudo easy_install pip - sudo pip install virtualenv - brew install pkg-config libffi - -Installing prerequisites on Raspbian:: - - sudo apt-get install build-essential python3-dev libffi-dev \ - python-pip python-setuptools sqlite3 \ - libssl-dev python-virtualenv libjpeg-dev - -Installing prerequisites on openSUSE:: - - sudo zypper in -t pattern devel_basis - sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \ - python-devel libffi-devel libopenssl-devel libjpeg62-devel - -Installing prerequisites on OpenBSD:: - - doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \ - libxslt jpeg - -To install the Synapse homeserver run:: - - mkdir -p ~/synapse - virtualenv -p python3 ~/synapse/env - source ~/synapse/env/bin/activate - pip install --upgrade pip - pip install --upgrade setuptools - pip install matrix-synapse[all] - -This installs Synapse, along with the libraries it uses, into a virtual -environment under ``~/synapse/env``. Feel free to pick a different directory -if you prefer. - -This Synapse installation can then be later upgraded by using pip again with the -update flag:: - - source ~/synapse/env/bin/activate - pip install -U matrix-synapse[all] - -In case of problems, please see the _`Troubleshooting` section below. - -There is an offical synapse image available at -https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with -the docker-compose file available at `contrib/docker `_. Further information on -this including configuration options is available in the README on -hub.docker.com. - -Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a -Dockerfile to automate a synapse server in a single Docker image, at -https://hub.docker.com/r/avhost/docker-matrix/tags/ - -Slavi Pantaleev has created an Ansible playbook, -which installs the offical Docker image of Matrix Synapse -along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.). -For more details, see -https://github.com/spantaleev/matrix-docker-ansible-deploy - -Configuring Synapse -------------------- - -Before you can start Synapse, you will need to generate a configuration -file. To do this, run (in your virtualenv, as before):: - - cd ~/synapse - python -m synapse.app.homeserver \ - --server-name my.domain.name \ - --config-path homeserver.yaml \ - --generate-config \ - --report-stats=[yes|no] - -... substituting an appropriate value for ``--server-name``. The server name -determines the "domain" part of user-ids for users on your server: these will -all be of the format ``@user:my.domain.name``. It also determines how other -matrix servers will reach yours for `Federation`_. For a test configuration, -set this to the hostname of your server. For a more production-ready setup, you -will probably want to specify your domain (``example.com``) rather than a -matrix-specific hostname here (in the same way that your email address is -probably ``user@example.com`` rather than ``user@email.example.com``) - but -doing so may require more advanced setup - see `Setting up -Federation`_. Beware that the server name cannot be changed later. - -This command will generate you a config file that you can then customise, but it will -also generate a set of keys for you. These keys will allow your Home Server to -identify itself to other Home Servers, so don't lose or delete them. It would be -wise to back them up somewhere safe. (If, for whatever reason, you do need to -change your Home Server's keys, you may find that other Home Servers have the -old key cached. If you update the signing key, you should change the name of the -key in the ``.signing.key`` file (the second word) to something -different. See `the spec`__ for more information on key management.) - -.. __: `key_management`_ - -The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is -configured without TLS; it should be behind a reverse proxy for TLS/SSL -termination on port 443 which in turn should be used for clients. Port 8448 -is configured to use TLS for `Federation`_ with a self-signed or verified -certificate, but please be aware that a valid certificate will be required in -Synapse v1.0. - -If you would like to use your own certificates, you can do so by changing -``tls_certificate_path`` and ``tls_private_key_path`` in ``homeserver.yaml``; -alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, -both ports are the same in the default configuration. - - -ACME setup ----------- - -Synapse v1.0 will require valid TLS certificates for communication between servers -(port ``8448`` by default) in addition to those that are client-facing (port -``443``). In the case that your `server_name` config variable is the same as -the hostname that the client connects to, then the same certificate can be -used between client and federation ports without issue. Synapse v0.99.0+ -**will provision server-to-server certificates automatically for you for -free** through `Let's Encrypt -`_ if you tell it to. - -In order for Synapse to complete the ACME challenge to provision a -certificate, it needs access to port 80. Typically listening on port 80 is -only granted to applications running as root. There are thus two solutions to -this problem. - -**Using a reverse proxy** - -A reverse proxy such as Apache or nginx allows a single process (the web -server) to listen on port 80 and proxy traffic to the appropriate program -running on your server. It is the recommended method for setting up ACME as -it allows you to use your existing webserver while also allowing Synapse to -provision certificates as needed. - -For nginx users, add the following line to your existing ``server`` block:: - - location /.well-known/acme-challenge { - proxy_pass http://localhost:8009/; - } - -For Apache, add the following to your existing webserver config:: - - ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge - -Make sure to restart/reload your webserver after making changes. - - -**Authbind** - -``authbind`` allows a program which does not run as root to bind to -low-numbered ports in a controlled way. The setup is simpler, but requires a -webserver not to already be running on port 80. **This includes every time -Synapse renews a certificate**, which may be cumbersome if you usually run a -web server on port 80. Nevertheless, if you're sure port 80 is not being used -for any other purpose then all that is necessary is the following: - -Install ``authbind``. For example, on Debian/Ubuntu:: - - sudo apt-get install authbind - -Allow ``authbind`` to bind port 80:: - - sudo touch /etc/authbind/byport/80 - sudo chmod 777 /etc/authbind/byport/80 - -When Synapse is started, use the following syntax:: - - authbind --deep - -Finally, once Synapse's is able to listen on port 80 for ACME challenge -requests, it must be told to perform ACME provisioning by setting ``enabled`` -to true under the ``acme`` section in ``homeserver.yaml``:: - - acme: - enabled: true - - -Registering a user ------------------- - -You will need at least one user on your server in order to use a Matrix -client. Users can be registered either `via a Matrix client`__, or via a -commandline script. - -.. __: `client-user-reg`_ - -To get started, it is easiest to use the command line to register new users:: - - $ source ~/synapse/env/bin/activate - $ synctl start # if not already running - $ register_new_matrix_user -c homeserver.yaml https://localhost:8448 - New user localpart: erikj - Password: - Confirm password: - Make admin [no]: - Success! - -This process uses a setting ``registration_shared_secret`` in -``homeserver.yaml``, which is shared between Synapse itself and the -``register_new_matrix_user`` script. It doesn't matter what it is (a random -value is generated by ``--generate-config``), but it should be kept secret, as -anyone with knowledge of it can register users on your server even if -``enable_registration`` is ``false``. - -Setting up a TURN server ------------------------- - -For reliable VoIP calls to be routed via this homeserver, you MUST configure -a TURN server. See ``_ for details. - -Running Synapse -=============== - -To actually run your new homeserver, pick a working directory for Synapse to -run (e.g. ``~/synapse``), and:: - - cd ~/synapse - source env/bin/activate - synctl start Connecting to Synapse from a client =================================== @@ -397,177 +144,11 @@ server on the same domain. See https://github.com/vector-im/riot-web/issues/1977 and https://developer.github.com/changes/2014-04-25-user-content-security for more details. - -Platform-Specific Instructions -============================== - -Debian/Ubuntu -------------- - -Matrix.org packages -~~~~~~~~~~~~~~~~~~~ - -Matrix.org provides Debian/Ubuntu packages of the latest stable version of -Synapse via https://matrix.org/packages/debian/. To use them:: - - sudo apt install -y lsb-release curl apt-transport-https - echo "deb https://matrix.org/packages/debian `lsb_release -cs` main" | - sudo tee /etc/apt/sources.list.d/matrix-org.list - curl "https://matrix.org/packages/debian/repo-key.asc" | - sudo apt-key add - - sudo apt update - sudo apt install matrix-synapse-py3 - -Downstream Debian/Ubuntu packages -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For ``buster`` and ``sid``, Synapse is available in the Debian repositories and -it should be possible to install it with simply:: - - sudo apt install matrix-synapse - -There is also a version of ``matrix-synapse`` in ``stretch-backports``. Please -see the `Debian documentation on backports -`_ for information on how to use -them. - -We do not recommend using the packages in downstream Ubuntu at this time, as -they are old and suffer from known security vulnerabilities. - -Fedora ------- - -Synapse is in the Fedora repositories as ``matrix-synapse``:: - - sudo dnf install matrix-synapse - -Oleg Girko provides Fedora RPMs at -https://obs.infoserver.lv/project/monitor/matrix-synapse - -OpenSUSE --------- - -Synapse is in the OpenSUSE repositories as ``matrix-synapse``:: - - sudo zypper install matrix-synapse - -SUSE Linux Enterprise Server ----------------------------- - -Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at -https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/ - -ArchLinux ---------- - -The quickest way to get up and running with ArchLinux is probably with the community package -https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of -the necessary dependencies. - -pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: - - sudo pip install --upgrade pip - -If you encounter an error with lib bcrypt causing an Wrong ELF Class: -ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly -compile it under the right architecture. (This should not be needed if -installing under virtualenv):: - - sudo pip uninstall py-bcrypt - sudo pip install py-bcrypt - -FreeBSD -------- - -Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from: - - - Ports: ``cd /usr/ports/net-im/py-matrix-synapse && make install clean`` - - Packages: ``pkg install py27-matrix-synapse`` - - -OpenBSD -------- - -There is currently no port for OpenBSD. Additionally, OpenBSD's security -settings require a slightly more difficult installation process. - -1) Create a new directory in ``/usr/local`` called ``_synapse``. Also, create a - new user called ``_synapse`` and set that directory as the new user's home. - This is required because, by default, OpenBSD only allows binaries which need - write and execute permissions on the same memory space to be run from - ``/usr/local``. -2) ``su`` to the new ``_synapse`` user and change to their home directory. -3) Create a new virtualenv: ``virtualenv -p python2.7 ~/.synapse`` -4) Source the virtualenv configuration located at - ``/usr/local/_synapse/.synapse/bin/activate``. This is done in ``ksh`` by - using the ``.`` command, rather than ``bash``'s ``source``. -5) Optionally, use ``pip`` to install ``lxml``, which Synapse needs to parse - webpages for their titles. -6) Use ``pip`` to install this repository: ``pip install matrix-synapse`` -7) Optionally, change ``_synapse``'s shell to ``/bin/false`` to reduce the - chance of a compromised Synapse server being used to take over your box. - -After this, you may proceed with the rest of the install directions. - -NixOS ------ - -Robin Lambertz has packaged Synapse for NixOS at: -https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix - -Windows Install ---------------- - -If you wish to run or develop Synapse on Windows, the Windows Subsystem For -Linux provides a Linux environment on Windows 10 which is capable of using the -Debian, Fedora, or source installation methods. More information about WSL can -be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for -Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server -for Windows Server. - Troubleshooting =============== -Troubleshooting Installation ----------------------------- - -Synapse requires pip 8 or later, so if your OS provides too old a version you -may need to manually upgrade it:: - - sudo pip install --upgrade pip - -Installing may fail with ``Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)``. -You can fix this by manually upgrading pip and virtualenv:: - - sudo pip install --upgrade virtualenv - -You can next rerun ``virtualenv -p python3 synapse`` to update the virtual env. - -Installing may fail during installing virtualenv with ``InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.`` -You can fix this by manually installing ndg-httpsclient:: - - pip install --upgrade ndg-httpsclient - -Installing may fail with ``mock requires setuptools>=17.1. Aborting installation``. -You can fix this by upgrading setuptools:: - - pip install --upgrade setuptools - -If pip crashes mid-installation for reason (e.g. lost terminal), pip may -refuse to run until you remove the temporary installation directory it -created. To reset the installation:: - - rm -rf /tmp/pip_install_matrix - -pip seems to leak *lots* of memory during installation. For instance, a Linux -host with 512MB of RAM may run out of memory whilst installing Twisted. If this -happens, you will have to individually install the dependencies which are -failing, e.g.:: - - pip install twisted - Running out of File Handles -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- If synapse runs out of filehandles, it typically fails badly - live-locking at 100% CPU, and/or failing to accept new TCP connections (blocking the @@ -609,7 +190,7 @@ Federation is the process by which users on different servers can participate in the same room. For this to work, those other servers must be able to contact yours to send messages. -As explained in `Configuring synapse`_, the ``server_name`` in your +The ``server_name`` in your ``homeserver.yaml`` file determines the way that other servers will reach yours. By default, they will treat it as a hostname and try to connect to port 8448. This is easy to set up and will work with the default configuration, @@ -796,24 +377,6 @@ an email address with your account, or send an invite to another user via their email address. -URL Previews -============ - -Synapse 0.15.0 introduces a new API for previewing URLs at -``/_matrix/media/r0/preview_url``. This is disabled by default. To turn it on -you must enable the ``url_preview_enabled: True`` config parameter and -explicitly specify the IP ranges that Synapse is not allowed to spider for -previewing in the ``url_preview_ip_range_blacklist`` configuration parameter. -This is critical from a security perspective to stop arbitrary Matrix users -spidering 'internal' URLs on your network. At the very least we recommend that -your loopback and RFC1918 IP addresses are blacklisted. - -This also requires the optional lxml and netaddr python dependencies to be -installed. This in turn requires the libxml2 library to be available - on -Debian/Ubuntu this means ``apt-get install libxml2-dev``, or equivalent for -your OS. - - Password reset ============== @@ -915,5 +478,3 @@ by installing the ``libjemalloc1`` package and adding this line to ``/etc/default/matrix-synapse``:: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 - -.. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys From cd6fee3169659b13bfdb0f4b4d2a6132fd6b542c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 14:29:09 +0000 Subject: [PATCH 39/96] Don't imply self-signed certs are required --- UPGRADE.rst | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index c46f70f699..f6cdec4734 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -51,34 +51,35 @@ returned by the Client-Server API: Upgrading to v0.99.0 ==================== -In preparation for Synapse v1.0, you must update your TLS certificates from -self-signed ones to verifiable ones signed by a trusted root CA. +In preparation for Synapse v1.0, you must ensure your federation TLS +certificates are verifiable by signed by a trusted root CA. -If you do not already have a certificate for your domain, the easiest way to get -one is with Synapse's new ACME support, which will use the ACME protocol to -provision a certificate automatically. By default, certificates will be obtained -from the publicly trusted CA Let's Encrypt. +If you do not already have a valid certificate for your domain, the easiest +way to get one is with Synapse's new ACME support, which will use the ACME +protocol to provision a certificate automatically. By default, certificates +will be obtained from the publicly trusted CA Let's Encrypt. For a sample configuration, please inspect the new ACME section in the example generated config by running the ``generate-config`` executable. For example:: ~/synapse/env3/bin/generate-config -You will need to provide Let's Encrypt (or other ACME provider) access to your -Synapse ACME challenge responder on port 80, at the domain of your homeserver. -This requires you either change the port of the ACME listener provided by -Synapse to a high port and reverse proxy to it, or use a tool like authbind to -allow Synapse to listen on port 80 without root access. (Do not run Synapse with -root permissions!) +You will need to provide Let's Encrypt (or another ACME provider) access to +your Synapse ACME challenge responder on port 80, at the domain of your +homeserver. This requires you to either change the port of the ACME listener +provided by Synapse to a high port and reverse proxy to it, or use a tool +like ``authbind`` to allow Synapse to listen on port 80 without root access. +(Do not run Synapse with root permissions!) -You will need to back up or delete your self signed TLS certificate -(``example.com.tls.crt`` and ``example.com.tls.key``), Synapse's ACME -implementation will not overwrite them. +If you are already using self-signed ceritifcates, you will need to back up +or delete them (files ``example.com.tls.crt`` and ``example.com.tls.key`` in +Synapse's root directory), Synapse's ACME implementation will not overwrite +them. You may wish to use alternate methods such as Certbot to obtain a certificate from Let's Encrypt, depending on your server configuration. Of course, if you already have a valid certificate for your homeserver's domain, that can be -placed in Synapse's config directory without the need for ACME. +placed in Synapse's config directory without the need for any ACME setup. Upgrading to v0.34.0 ==================== From 08b26afeee7b7db8c9d511cb63244927cf48ba9d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 15:29:42 +0000 Subject: [PATCH 40/96] Move ACME docs to docs/ACME.rst and link from UPGRADE. --- README.rst | 69 ------------------------------------ UPGRADE.rst | 33 +++-------------- docs/ACME.rst | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 98 deletions(-) create mode 100644 docs/ACME.rst diff --git a/README.rst b/README.rst index 9e3d85de4c..829de0864c 100644 --- a/README.rst +++ b/README.rst @@ -225,75 +225,6 @@ If you would like to use your own certificates, you can do so by changing alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, both ports are the same in the default configuration. - -ACME setup ----------- - -Synapse v1.0 will require valid TLS certificates for communication between servers -(port ``8448`` by default) in addition to those that are client-facing (port -``443``). In the case that your `server_name` config variable is the same as -the hostname that the client connects to, then the same certificate can be -used between client and federation ports without issue. Synapse v0.99.0+ -**will provision server-to-server certificates automatically for you for -free** through `Let's Encrypt -`_ if you tell it to. - -In order for Synapse to complete the ACME challenge to provision a -certificate, it needs access to port 80. Typically listening on port 80 is -only granted to applications running as root. There are thus two solutions to -this problem. - -**Using a reverse proxy** - -A reverse proxy such as Apache or nginx allows a single process (the web -server) to listen on port 80 and proxy traffic to the appropriate program -running on your server. It is the recommended method for setting up ACME as -it allows you to use your existing webserver while also allowing Synapse to -provision certificates as needed. - -For nginx users, add the following line to your existing ``server`` block:: - - location /.well-known/acme-challenge { - proxy_pass http://localhost:8009/; - } - -For Apache, add the following to your existing webserver config:: - - ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge - -Make sure to restart/reload your webserver after making changes. - - -**Authbind** - -``authbind`` allows a program which does not run as root to bind to -low-numbered ports in a controlled way. The setup is simpler, but requires a -webserver not to already be running on port 80. **This includes every time -Synapse renews a certificate**, which may be cumbersome if you usually run a -web server on port 80. Nevertheless, if you're sure port 80 is not being used -for any other purpose then all that is necessary is the following: - -Install ``authbind``. For example, on Debian/Ubuntu:: - - sudo apt-get install authbind - -Allow ``authbind`` to bind port 80:: - - sudo touch /etc/authbind/byport/80 - sudo chmod 777 /etc/authbind/byport/80 - -When Synapse is started, use the following syntax:: - - authbind --deep - -Finally, once Synapse's is able to listen on port 80 for ACME challenge -requests, it must be told to perform ACME provisioning by setting ``enabled`` -to true under the ``acme`` section in ``homeserver.yaml``:: - - acme: - enabled: true - - Registering a user ------------------ diff --git a/UPGRADE.rst b/UPGRADE.rst index f6cdec4734..74d2452749 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -51,35 +51,10 @@ returned by the Client-Server API: Upgrading to v0.99.0 ==================== -In preparation for Synapse v1.0, you must ensure your federation TLS -certificates are verifiable by signed by a trusted root CA. - -If you do not already have a valid certificate for your domain, the easiest -way to get one is with Synapse's new ACME support, which will use the ACME -protocol to provision a certificate automatically. By default, certificates -will be obtained from the publicly trusted CA Let's Encrypt. - -For a sample configuration, please inspect the new ACME section in the example -generated config by running the ``generate-config`` executable. For example:: - - ~/synapse/env3/bin/generate-config - -You will need to provide Let's Encrypt (or another ACME provider) access to -your Synapse ACME challenge responder on port 80, at the domain of your -homeserver. This requires you to either change the port of the ACME listener -provided by Synapse to a high port and reverse proxy to it, or use a tool -like ``authbind`` to allow Synapse to listen on port 80 without root access. -(Do not run Synapse with root permissions!) - -If you are already using self-signed ceritifcates, you will need to back up -or delete them (files ``example.com.tls.crt`` and ``example.com.tls.key`` in -Synapse's root directory), Synapse's ACME implementation will not overwrite -them. - -You may wish to use alternate methods such as Certbot to obtain a certificate -from Let's Encrypt, depending on your server configuration. Of course, if you -already have a valid certificate for your homeserver's domain, that can be -placed in Synapse's config directory without the need for any ACME setup. +No special steps are required, but please be aware that you will need to +replace any self-signed certificates with those verified by a root CA before +Synapse v1.0 releases in roughly a month's time after v0.99.0. Information on +how to do so can be found at `the ACME docs `_. Upgrading to v0.34.0 ==================== diff --git a/docs/ACME.rst b/docs/ACME.rst new file mode 100644 index 0000000000..2562e85dbc --- /dev/null +++ b/docs/ACME.rst @@ -0,0 +1,98 @@ +ACME +==== + +Synapse v1.0 requires that federation TLS certificates are verifiable by a +trusted root CA. If you do not already have a valid certificate for your domain, the easiest +way to get one is with Synapse's new ACME support, which will use the ACME +protocol to provision a certificate automatically. By default, certificates +will be obtained from the publicly trusted CA Let's Encrypt. + +For a sample configuration, please inspect the new ACME section in the example +generated config by running the ``generate-config`` executable. For example:: + + ~/synapse/env3/bin/generate-config + +You will need to provide Let's Encrypt (or another ACME provider) access to +your Synapse ACME challenge responder on port 80, at the domain of your +homeserver. This requires you to either change the port of the ACME listener +provided by Synapse to a high port and reverse proxy to it, or use a tool +like ``authbind`` to allow Synapse to listen on port 80 without root access. +(Do not run Synapse with root permissions!) Detailed instructions are +available under "ACME setup" below. + +If you are already using self-signed certificates, you will need to back up +or delete them (files ``example.com.tls.crt`` and ``example.com.tls.key`` in +Synapse's root directory), Synapse's ACME implementation will not overwrite +them. + +You may wish to use alternate methods such as Certbot to obtain a certificate +from Let's Encrypt, depending on your server configuration. Of course, if you +already have a valid certificate for your homeserver's domain, that can be +placed in Synapse's config directory without the need for any ACME setup. + +ACME setup +---------- + +Synapse v1.0 will require valid TLS certificates for communication between servers +(port ``8448`` by default) in addition to those that are client-facing (port +``443``). In the case that your `server_name` config variable is the same as +the hostname that the client connects to, then the same certificate can be +used between client and federation ports without issue. Synapse v0.99.0+ +will provision server-to-server certificates automatically for you for +free through `Let's Encrypt +`_ if you tell it to. + +In order for Synapse to complete the ACME challenge to provision a +certificate, it needs access to port 80. Typically listening on port 80 is +only granted to applications running as root. There are thus two solutions to +this problem. + +**Using a reverse proxy** + +A reverse proxy such as Apache or nginx allows a single process (the web +server) to listen on port 80 and proxy traffic to the appropriate program +running on your server. It is the recommended method for setting up ACME as +it allows you to use your existing webserver while also allowing Synapse to +provision certificates as needed. + +For nginx users, add the following line to your existing ``server`` block:: + + location /.well-known/acme-challenge { + proxy_pass http://localhost:8009/; + } + +For Apache, add the following to your existing webserver config:: + + ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge + +Make sure to restart/reload your webserver after making changes. + + +**Authbind** + +``authbind`` allows a program which does not run as root to bind to +low-numbered ports in a controlled way. The setup is simpler, but requires a +webserver not to already be running on port 80. **This includes every time +Synapse renews a certificate**, which may be cumbersome if you usually run a +web server on port 80. Nevertheless, if you're sure port 80 is not being used +for any other purpose then all that is necessary is the following: + +Install ``authbind``. For example, on Debian/Ubuntu:: + + sudo apt-get install authbind + +Allow ``authbind`` to bind port 80:: + + sudo touch /etc/authbind/byport/80 + sudo chmod 777 /etc/authbind/byport/80 + +When Synapse is started, use the following syntax:: + + authbind --deep + +Finally, once Synapse's is able to listen on port 80 for ACME challenge +requests, it must be told to perform ACME provisioning by setting ``enabled`` +to true under the ``acme`` section in ``homeserver.yaml``:: + + acme: + enabled: true From 9793bf366e85a39507b43bbbf7bdcb387ff5bf83 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 15:34:52 +0000 Subject: [PATCH 41/96] Add link to ACME docs from README --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 829de0864c..47f8ef62db 100644 --- a/README.rst +++ b/README.rst @@ -225,6 +225,12 @@ If you would like to use your own certificates, you can do so by changing alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, both ports are the same in the default configuration. +ACME setup +---------- + +For details on having Synapse manage your federation TLS certificates +automatically, please see ``_. + Registering a user ------------------ From cbdc01cc3b826879b43ad4e8e9c1acacc60cd34b Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 15:38:27 +0000 Subject: [PATCH 42/96] Convert ACME docs to md --- README.rst | 2 +- UPGRADE.rst | 2 +- docs/ACME.rst | 98 --------------------------------------------------- 3 files changed, 2 insertions(+), 100 deletions(-) delete mode 100644 docs/ACME.rst diff --git a/README.rst b/README.rst index 47f8ef62db..b7b3b20159 100644 --- a/README.rst +++ b/README.rst @@ -229,7 +229,7 @@ ACME setup ---------- For details on having Synapse manage your federation TLS certificates -automatically, please see ``_. +automatically, please see ``_. Registering a user ------------------ diff --git a/UPGRADE.rst b/UPGRADE.rst index 74d2452749..948867f189 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -54,7 +54,7 @@ Upgrading to v0.99.0 No special steps are required, but please be aware that you will need to replace any self-signed certificates with those verified by a root CA before Synapse v1.0 releases in roughly a month's time after v0.99.0. Information on -how to do so can be found at `the ACME docs `_. +how to do so can be found at `the ACME docs `_. Upgrading to v0.34.0 ==================== diff --git a/docs/ACME.rst b/docs/ACME.rst deleted file mode 100644 index 2562e85dbc..0000000000 --- a/docs/ACME.rst +++ /dev/null @@ -1,98 +0,0 @@ -ACME -==== - -Synapse v1.0 requires that federation TLS certificates are verifiable by a -trusted root CA. If you do not already have a valid certificate for your domain, the easiest -way to get one is with Synapse's new ACME support, which will use the ACME -protocol to provision a certificate automatically. By default, certificates -will be obtained from the publicly trusted CA Let's Encrypt. - -For a sample configuration, please inspect the new ACME section in the example -generated config by running the ``generate-config`` executable. For example:: - - ~/synapse/env3/bin/generate-config - -You will need to provide Let's Encrypt (or another ACME provider) access to -your Synapse ACME challenge responder on port 80, at the domain of your -homeserver. This requires you to either change the port of the ACME listener -provided by Synapse to a high port and reverse proxy to it, or use a tool -like ``authbind`` to allow Synapse to listen on port 80 without root access. -(Do not run Synapse with root permissions!) Detailed instructions are -available under "ACME setup" below. - -If you are already using self-signed certificates, you will need to back up -or delete them (files ``example.com.tls.crt`` and ``example.com.tls.key`` in -Synapse's root directory), Synapse's ACME implementation will not overwrite -them. - -You may wish to use alternate methods such as Certbot to obtain a certificate -from Let's Encrypt, depending on your server configuration. Of course, if you -already have a valid certificate for your homeserver's domain, that can be -placed in Synapse's config directory without the need for any ACME setup. - -ACME setup ----------- - -Synapse v1.0 will require valid TLS certificates for communication between servers -(port ``8448`` by default) in addition to those that are client-facing (port -``443``). In the case that your `server_name` config variable is the same as -the hostname that the client connects to, then the same certificate can be -used between client and federation ports without issue. Synapse v0.99.0+ -will provision server-to-server certificates automatically for you for -free through `Let's Encrypt -`_ if you tell it to. - -In order for Synapse to complete the ACME challenge to provision a -certificate, it needs access to port 80. Typically listening on port 80 is -only granted to applications running as root. There are thus two solutions to -this problem. - -**Using a reverse proxy** - -A reverse proxy such as Apache or nginx allows a single process (the web -server) to listen on port 80 and proxy traffic to the appropriate program -running on your server. It is the recommended method for setting up ACME as -it allows you to use your existing webserver while also allowing Synapse to -provision certificates as needed. - -For nginx users, add the following line to your existing ``server`` block:: - - location /.well-known/acme-challenge { - proxy_pass http://localhost:8009/; - } - -For Apache, add the following to your existing webserver config:: - - ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge - -Make sure to restart/reload your webserver after making changes. - - -**Authbind** - -``authbind`` allows a program which does not run as root to bind to -low-numbered ports in a controlled way. The setup is simpler, but requires a -webserver not to already be running on port 80. **This includes every time -Synapse renews a certificate**, which may be cumbersome if you usually run a -web server on port 80. Nevertheless, if you're sure port 80 is not being used -for any other purpose then all that is necessary is the following: - -Install ``authbind``. For example, on Debian/Ubuntu:: - - sudo apt-get install authbind - -Allow ``authbind`` to bind port 80:: - - sudo touch /etc/authbind/byport/80 - sudo chmod 777 /etc/authbind/byport/80 - -When Synapse is started, use the following syntax:: - - authbind --deep - -Finally, once Synapse's is able to listen on port 80 for ACME challenge -requests, it must be told to perform ACME provisioning by setting ``enabled`` -to true under the ``acme`` section in ``homeserver.yaml``:: - - acme: - enabled: true From d75e15edce64ee48a80f5f7f253464be1ec6866d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 15:45:00 +0000 Subject: [PATCH 43/96] Re-add link to ACME docs from README --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index b8c1909101..8db7bf94de 100644 --- a/README.rst +++ b/README.rst @@ -134,6 +134,12 @@ create the account. Your name will take the form of:: As when logging in, you will need to specify a "Custom server". Specify your desired ``localpart`` in the 'User name' box. +ACME setup +---------- + +For details on having Synapse manage your federation TLS certificates +automatically, please see ``_. + Security Note ============= From 0af50020fd9bb591fde82876a6b543d50683bae0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 15:49:34 +0000 Subject: [PATCH 44/96] Move ACME docs from INSTALL.md to ACME.md --- INSTALL.md | 79 +----------------------------------------------------- 1 file changed, 1 insertion(+), 78 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index fd37c2d9b9..cbe4bda120 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -355,90 +355,13 @@ configured without TLS; it should be behind a reverse proxy for TLS/SSL termination on port 443 which in turn should be used for clients. Port 8448 is configured to use TLS for Federation with a self-signed or verified certificate, but please be aware that a valid certificate will be required in -Synapse v1.0. +Synapse v1.0. Instructions for having Synapse automatically provision and renew federation certificates through ACME can be found at [ACME.md](docs/ACME.md). If you would like to use your own certificates, you can do so by changing `tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`; alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, both ports are the same in the default configuration. -### ACME setup - -Synapse v1.0 will require valid TLS certificates for communication between servers -(port `8448` by default) in addition to those that are client-facing (port -`443`). In the case that your `server_name` config variable is the same as -the hostname that the client connects to, then the same certificate can be -used between client and federation ports without issue. Synapse v0.99.0+ -**will provision server-to-server certificates automatically for you for -free** through [Let's Encrypt](https://letsencrypt.org/) if you tell it to. - -In order for Synapse to complete the ACME challenge to provision a -certificate, it needs access to port 80. Typically listening on port 80 is -only granted to applications running as root. There are thus two solutions to -this problem. - -#### Using a reverse proxy - -A reverse proxy such as Apache or nginx allows a single process (the web -server) to listen on port 80 and proxy traffic to the appropriate program -running on your server. It is the recommended method for setting up ACME as -it allows you to use your existing webserver while also allowing Synapse to -provision certificates as needed. - -For nginx users, add the following line to your existing `server` block: - -``` -location /.well-known/acme-challenge { - proxy_pass http://localhost:8009/; -} -``` - -For Apache, add the following to your existing webserver config:: - -``` -ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge -``` - -Make sure to restart/reload your webserver after making changes. - - -#### Authbind - -`authbind` allows a program which does not run as root to bind to -low-numbered ports in a controlled way. The setup is simpler, but requires a -webserver not to already be running on port 80. **This includes every time -Synapse renews a certificate**, which may be cumbersome if you usually run a -web server on port 80. Nevertheless, if you're sure port 80 is not being used -for any other purpose then all that is necessary is the following: - -Install `authbind`. For example, on Debian/Ubuntu: - -``` -sudo apt-get install authbind -``` - -Allow `authbind` to bind port 80: - -``` -sudo touch /etc/authbind/byport/80 -sudo chmod 777 /etc/authbind/byport/80 -``` - -When Synapse is started, use the following syntax:: - -``` -authbind --deep -``` - -Finally, once Synapse is able to listen on port 80 for ACME challenge -requests, it must be told to perform ACME provisioning by setting `enabled` -to true under the `acme` section in `homeserver.yaml`: - -``` -acme: - enabled: true -``` - ## Registering a user You will need at least one user on your server in order to use a Matrix From ffcbd80982ad4164eda38c45d8b367b1748904c4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 15:50:18 +0000 Subject: [PATCH 45/96] Actually add ACME docs --- docs/ACME.md | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docs/ACME.md diff --git a/docs/ACME.md b/docs/ACME.md new file mode 100644 index 0000000000..f1a0c25697 --- /dev/null +++ b/docs/ACME.md @@ -0,0 +1,107 @@ +# ACME + +Synapse v1.0 requires that federation TLS certificates are verifiable by a +trusted root CA. If you do not already have a valid certificate for your domain, the easiest +way to get one is with Synapse's new ACME support, which will use the ACME +protocol to provision a certificate automatically. By default, certificates +will be obtained from the publicly trusted CA Let's Encrypt. + +For a sample configuration, please inspect the new ACME section in the example +generated config by running the `generate-config` executable. For example:: + + ~/synapse/env3/bin/generate-config + +You will need to provide Let's Encrypt (or another ACME provider) access to +your Synapse ACME challenge responder on port 80, at the domain of your +homeserver. This requires you to either change the port of the ACME listener +provided by Synapse to a high port and reverse proxy to it, or use a tool +like `authbind` to allow Synapse to listen on port 80 without root access. +(Do not run Synapse with root permissions!) Detailed instructions are +available under "ACME setup" below. + +If you are already using self-signed certificates, you will need to back up +or delete them (files `example.com.tls.crt` and `example.com.tls.key` in +Synapse's root directory), Synapse's ACME implementation will not overwrite +them. + +You may wish to use alternate methods such as Certbot to obtain a certificate +from Let's Encrypt, depending on your server configuration. Of course, if you +already have a valid certificate for your homeserver's domain, that can be +placed in Synapse's config directory without the need for any ACME setup. + +## ACME setup + +Synapse v1.0 will require valid TLS certificates for communication between servers +(port `8448` by default) in addition to those that are client-facing (port +`443`). In the case that your `server_name` config variable is the same as +the hostname that the client connects to, then the same certificate can be +used between client and federation ports without issue. Synapse v0.99.0+ +**will provision server-to-server certificates automatically for you for +free** through [Let's Encrypt](https://letsencrypt.org/) if you tell it to. + +In order for Synapse to complete the ACME challenge to provision a +certificate, it needs access to port 80. Typically listening on port 80 is +only granted to applications running as root. There are thus two solutions to +this problem. + +### Using a reverse proxy + +A reverse proxy such as Apache or nginx allows a single process (the web +server) to listen on port 80 and proxy traffic to the appropriate program +running on your server. It is the recommended method for setting up ACME as +it allows you to use your existing webserver while also allowing Synapse to +provision certificates as needed. + +For nginx users, add the following line to your existing `server` block: + +``` +location /.well-known/acme-challenge { + proxy_pass http://localhost:8009/; +} +``` + +For Apache, add the following to your existing webserver config:: + +``` +ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge +``` + +Make sure to restart/reload your webserver after making changes. + + +### Authbind + +`authbind` allows a program which does not run as root to bind to +low-numbered ports in a controlled way. The setup is simpler, but requires a +webserver not to already be running on port 80. **This includes every time +Synapse renews a certificate**, which may be cumbersome if you usually run a +web server on port 80. Nevertheless, if you're sure port 80 is not being used +for any other purpose then all that is necessary is the following: + +Install `authbind`. For example, on Debian/Ubuntu: + +``` +sudo apt-get install authbind +``` + +Allow `authbind` to bind port 80: + +``` +sudo touch /etc/authbind/byport/80 +sudo chmod 777 /etc/authbind/byport/80 +``` + +When Synapse is started, use the following syntax:: + +``` +authbind --deep +``` + +Finally, once Synapse is able to listen on port 80 for ACME challenge +requests, it must be told to perform ACME provisioning by setting `enabled` +to true under the `acme` section in `homeserver.yaml`: + +``` +acme: + enabled: true +``` \ No newline at end of file From ed8b3289ff1d82f7fa512000ac085e8f395f44d3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 16:44:31 +0000 Subject: [PATCH 46/96] Update README --- README.rst | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 8db7bf94de..e666b3b427 100644 --- a/README.rst +++ b/README.rst @@ -84,13 +84,6 @@ Synapse Installation For details on how to install synapse, see ``_. -To actually run your new homeserver, pick a working directory for Synapse to -run (e.g. ``~/synapse``), and:: - - cd ~/synapse - source env/bin/activate - synctl start - Connecting to Synapse from a client =================================== @@ -135,7 +128,7 @@ As when logging in, you will need to specify a "Custom server". Specify your desired ``localpart`` in the 'User name' box. ACME setup ----------- +========== For details on having Synapse manage your federation TLS certificates automatically, please see ``_. From e119cec229202b8c6f0429fa1586c1a28f1925e3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 16:45:41 +0000 Subject: [PATCH 47/96] Update INSTALL --- UPGRADE.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index 948867f189..eee38d5228 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -51,10 +51,10 @@ returned by the Client-Server API: Upgrading to v0.99.0 ==================== -No special steps are required, but please be aware that you will need to -replace any self-signed certificates with those verified by a root CA before -Synapse v1.0 releases in roughly a month's time after v0.99.0. Information on -how to do so can be found at `the ACME docs `_. +Please be aware that, before Synapse v1.0 is released around March 2019, you +will need to replace any self-signed certificates with those verified by a +root CA. Information on how to do so can be found at `the ACME docs +`_. Upgrading to v0.34.0 ==================== From 13828f7d5811c6c0de1ccc4d734a68d01a004dec Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 16:46:28 +0000 Subject: [PATCH 48/96] Update docs/ACME.md Co-Authored-By: anoadragon453 <1342360+anoadragon453@users.noreply.github.com> --- docs/ACME.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ACME.md b/docs/ACME.md index f1a0c25697..309296cc0b 100644 --- a/docs/ACME.md +++ b/docs/ACME.md @@ -2,7 +2,7 @@ Synapse v1.0 requires that federation TLS certificates are verifiable by a trusted root CA. If you do not already have a valid certificate for your domain, the easiest -way to get one is with Synapse's new ACME support, which will use the ACME +way to get one is with Synapse's ACME support (new as of Synapse 0.99), which will use the ACME protocol to provision a certificate automatically. By default, certificates will be obtained from the publicly trusted CA Let's Encrypt. @@ -104,4 +104,4 @@ to true under the `acme` section in `homeserver.yaml`: ``` acme: enabled: true -``` \ No newline at end of file +``` From 2ca63df83b49599613b3801be2577a1d869a918b Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 16:50:00 +0000 Subject: [PATCH 49/96] Update ACME --- docs/ACME.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/ACME.md b/docs/ACME.md index f1a0c25697..341044dac1 100644 --- a/docs/ACME.md +++ b/docs/ACME.md @@ -1,15 +1,23 @@ # ACME -Synapse v1.0 requires that federation TLS certificates are verifiable by a -trusted root CA. If you do not already have a valid certificate for your domain, the easiest -way to get one is with Synapse's new ACME support, which will use the ACME -protocol to provision a certificate automatically. By default, certificates -will be obtained from the publicly trusted CA Let's Encrypt. +Synapse v1.0 will require valid TLS certificates for communication between +servers (port `8448` by default) in addition to those that are client-facing +(port `443`). If you do not already have a valid certificate for your domain, +the easiest way to get one is with Synapse's new ACME support, which will use +the ACME protocol to provision a certificate automatically. Synapse v0.99.0+ +will provision server-to-server certificates automatically for you for free +through [Let's Encrypt](https://letsencrypt.org/) if you tell it to. + +In the case that your `server_name` config variable is the same as +the hostname that the client connects to, then the same certificate can be +used between client and federation ports without issue. For a sample configuration, please inspect the new ACME section in the example -generated config by running the `generate-config` executable. For example:: +generated config by running the `generate-config` executable. For example: - ~/synapse/env3/bin/generate-config +``` +~/synapse/env3/bin/generate-config +``` You will need to provide Let's Encrypt (or another ACME provider) access to your Synapse ACME challenge responder on port 80, at the domain of your @@ -31,13 +39,6 @@ placed in Synapse's config directory without the need for any ACME setup. ## ACME setup -Synapse v1.0 will require valid TLS certificates for communication between servers -(port `8448` by default) in addition to those that are client-facing (port -`443`). In the case that your `server_name` config variable is the same as -the hostname that the client connects to, then the same certificate can be -used between client and federation ports without issue. Synapse v0.99.0+ -**will provision server-to-server certificates automatically for you for -free** through [Let's Encrypt](https://letsencrypt.org/) if you tell it to. In order for Synapse to complete the ACME challenge to provision a certificate, it needs access to port 80. Typically listening on port 80 is @@ -97,6 +98,8 @@ When Synapse is started, use the following syntax:: authbind --deep ``` +## Config file editing + Finally, once Synapse is able to listen on port 80 for ACME challenge requests, it must be told to perform ACME provisioning by setting `enabled` to true under the `acme` section in `homeserver.yaml`: From a6345009f92d8ffef4bd0d42196d18eac9b9bf38 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 5 Feb 2019 17:04:34 +0000 Subject: [PATCH 50/96] Add TL;DR and final step details to ACME --- docs/ACME.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/ACME.md b/docs/ACME.md index 15752ad9c9..8fb2bd66a9 100644 --- a/docs/ACME.md +++ b/docs/ACME.md @@ -39,13 +39,23 @@ placed in Synapse's config directory without the need for any ACME setup. ## ACME setup +The main steps for enabling ACME support in short summary are: + +1. Allow Synapse to listen on port 80 with authbind, or forward it from a reverse-proxy. +1. Set `acme:enabled` to `true` in homeserver.yaml. +1. Move your old certificates (files `example.com.tls.crt` and `example.com.tls.key` out of the way if they currently exist at the paths specified in `homeserver.yaml`. +1. Restart Synapse + +Detailed instructions for each step are provided below. + +### Listening on port 80 In order for Synapse to complete the ACME challenge to provision a certificate, it needs access to port 80. Typically listening on port 80 is only granted to applications running as root. There are thus two solutions to this problem. -### Using a reverse proxy +#### Using a reverse proxy A reverse proxy such as Apache or nginx allows a single process (the web server) to listen on port 80 and proxy traffic to the appropriate program @@ -70,7 +80,7 @@ ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-cha Make sure to restart/reload your webserver after making changes. -### Authbind +#### Authbind `authbind` allows a program which does not run as root to bind to low-numbered ports in a controlled way. The setup is simpler, but requires a @@ -98,9 +108,9 @@ When Synapse is started, use the following syntax:: authbind --deep ``` -## Config file editing +### Config file editing -Finally, once Synapse is able to listen on port 80 for ACME challenge +Once Synapse is able to listen on port 80 for ACME challenge requests, it must be told to perform ACME provisioning by setting `enabled` to true under the `acme` section in `homeserver.yaml`: @@ -108,3 +118,9 @@ to true under the `acme` section in `homeserver.yaml`: acme: enabled: true ``` + +### Starting synapse + +Ensure that the certificate paths specified in `homeserver.yaml` (`tls_certificate_path` and `tls_private_key_path`) do not currently point to any files. Synapse will not provision certificates if files exist, as it does not want to overwrite existing certificates. + +Finally, start/restart Synapse. \ No newline at end of file From 6585ef47991478ed1617f78f992b35b63475a2d7 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 5 Feb 2019 17:19:28 +0000 Subject: [PATCH 51/96] Neilj/1711faq (#4572) MSC1711 certificates FAQ --- UPGRADE.rst | 2 + changelog.d/4572.misc | 1 + docs/MSC1711_certificates_FAQ.md | 260 +++++++++++++++++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 changelog.d/4572.misc create mode 100644 docs/MSC1711_certificates_FAQ.md diff --git a/UPGRADE.rst b/UPGRADE.rst index c46f70f699..7bd631f14c 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -80,6 +80,8 @@ from Let's Encrypt, depending on your server configuration. Of course, if you already have a valid certificate for your homeserver's domain, that can be placed in Synapse's config directory without the need for ACME. +For more information on configuring TLS certificates see the `FAQ `_ + Upgrading to v0.34.0 ==================== diff --git a/changelog.d/4572.misc b/changelog.d/4572.misc new file mode 100644 index 0000000000..ea5d49b706 --- /dev/null +++ b/changelog.d/4572.misc @@ -0,0 +1 @@ +FAQ to help server admins configure TLS certs in 0.99.0 diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md new file mode 100644 index 0000000000..0dcef57733 --- /dev/null +++ b/docs/MSC1711_certificates_FAQ.md @@ -0,0 +1,260 @@ +# MSC 1711 Certificates FAQ + +The goal of Synapse 0.99.0 is to act as a stepping stone to Synapse 1.0.0. It +supports the r0.1 release of the server to server specification, but is +compatible with both the legacy Matrix federation behaviour (pre-r0.1) as well +as post-r0.1 behaviour, in order to allow for a smooth upgrade across the +federation. + +The most important thing to know is that Synapse 1.0.0 will require a valid TLS +certificate on federation endpoints. Self signed certificates will not be +sufficient. + +Synapse 0.99.0 makes it easy to configure TLS certificates and will +interoperate with both >= 1.0.0 servers as well as existing servers yet to +upgrade. + +**It is critical that all admins upgrade to 0.99.0 and configure a valid TLS +certificate.** Admins will have 1 month to do so, after which 1.0.0 will be +released and those servers without a valid certificate will not longer be able +to federate with >= 1.0.0 servers. + +If you are unable to generate a valid TLS certificate for your server (e.g. +because you run it on behalf of someone who doesn't want to give you a TLS +certificate for their domain, or simply because the matrix domain is hosted on +a different server), then you can now create a /.well-known/matrix/server file +on the matrix domain in order to delegate Matrix hosting to another domain. + Admins who currently use SRV records to delegate a domain **which they do not +control TLS for** will need to switch to using .well-known/matrix/server - though +they should retain their SRV record while the federation upgrades over the +course of the month.  Other SRV records are unaffected. + +Full upgrade notes can be found in +[UPGRADE.rst](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst). +What follows is a timeline and some frequently asked questions. + +For more details and context on the release of the r0.1 Server/Server API and +imminent Matrix 1.0 release, you can also see our +[main talk from FOSDEM 2019](https://matrix.org/blog/2019/02/04/matrix-at-fosdem-2019/). + +## Contents +* Timeline +* Synapse 0.99.0 has just been released, what do I need to do right now? +* How do I upgrade? +* What will happen if I do not set up a valid federation certificate + immediately? +* What will happen if I do nothing at all? +* When do I need a SRV record or .well-known URI? +* Can I still use an SRV record? +* I have created a .well-known URI. Do I still need an SRV record? +* It used to work just fine, why are you breaking everything? +* Can I manage my own certificates rather than having Synapse renew + certificates itself? +* Do you still recommend against using a reverse-proxy on the federation port? +* Do I still need to give my TLS certificates to Synapse if I am using a + reverse-proxy? +* Do I need the same certificate for the client and federation port? +* How do I tell Synapse to reload my keys/certificates after I replace them? + + +### Timeline + +**5th Feb 2019 - Synapse 0.99.0 is released.** + +All server admins are encouraged to upgrade. + +0.99.0: + +- provides support for ACME to make setting up Let's Encrypt certs easy, as + well as .well-known support. + +- does not enforce that a valid CA cert is present on the federation API, but + rather makes it easy to set one up. + +- provides support for .well-known + +Admins should upgrade and configure a valid CA cert. Homeservers that require a +.well-known entry (see below), should retain their SRV record and use it +alongside their .well-known record. + +**>= 5th March 2019 - Synapse 1.0.0 is released** + +1.0.0 will land no sooner than 1 month after 0.99.0, leaving server admins one +month after 5th February to upgrade to 0.99.0 and deploy their certificates. In +accordance with the the [S2S spec](https://matrix.org/docs/spec/server_server/r0.1.0.html) +1.0.0 will enforce federation checks. This means that any homeserver without a +valid certificate after this point will no longer be able to federate with +1.0.0 servers. + +### Synapse 0.99.0 has just been released, what do I need to do right now? + +Upgrade as soon as you can in preparation for Synapse 1.0.0. + +### How do I upgrade? + +Follow the upgrade notes here [UPGRADE.rst](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst) + +### What will happen if I do not set up a valid federation certificate immediately? + +Nothing initially, but once 1.0.0 is in the wild it will not be possible to +federate with 1.0.0 servers. + +### What will happen if I do nothing at all? + +If the admin takes no action at all, and remains on a Synapse < 0.99.0 then the +homeserver will be unable to federate with those who have implemented +.well-known. Then, as above, once the month upgrade window has expired the +homeserver will not be able to federate with any Synapse >= 1.0.0 + +### When do I need a SRV record or .well-known URI? + +If your homeserver listens on the default federation port (8448), and your +server_name points to the host that your homeserver runs on, you do not need an +SRV record or .well-known/matrix/server URI.\ +For instance, if you registered example.com and pointed its DNS A record at a +fresh Upcloud VPS or similar, you could install Synapse 0.99 on that host, +giving it a server_name of example.com, and it would automatically generate a +valid TLS certificate for you via Let's Encrypt and no SRV record or +.well-known URI would be needed. + +This is the common case, although you can add an SRV record or +.well-known/matrix/server URI for completeness if you wish. + +**However**, if your server does not listen on port 8448, or if your server_name +does not point to the host that your homeserver runs on, you will need to let +other servers know how to find it. + +The easiest way to do this is with a .well-known/matrix/server URI on the +webroot of the domain to advertise your server. For instance, if you ran +"matrixhosting.com" and you were hosting a Matrix server for example.com, you +would ask example.com to create a file at: + +`https://example.com/.well-known/matrix/server` + +with contents: + +`{"m.server": "example.matrixhosting.com:8448"}` + +...which would tell servers trying to connect to example.com to instead connect +to example.matrixhosting.com on port 8448. You would then configure Synapse +with a server_name of "example.com", but generate a TLS certificate for +example.matrixhosting.com. + +As an alternative, you can still use an SRV DNS record for the delegation, but +this will require you to have a certificate for the matrix domain (example.com +in this example). See "Can I still use an SRV record?". + +### Can I still use an SRV record? + +Firstly, if you didn't need an SRV record before (because your server is +listening on port 8448 of your server_name), you certainly don't need one now: +the defaults are still the same. + +If you previously had an SRV record, you can keep using it provided you are +able to give Synapse a TLS certificate corresponding to your server name. For +example, suppose you had the following SRV record, which directs matrix traffic +for example.com to matrix.example.com:443: + +``` +_matrix._tcp.example.com. IN SRV 10 5 443 matrix.example.com +``` + +In this case, Synapse must be given a certificate for example.com - or be +configured to acquire one from Let's Encrypt. + +If you are unable to give Synapse a certificate for your server_name, you will +also need to use a .well-known URI instead. However, see also "I have created a +.well-known URI. Do I still need an SRV record?". + +### I have created a .well-known URI. Do I still need an SRV record? + +As of Synapse 0.99, Synapse will first check for the existence of a .well-known +URL and follow any delegation it suggests. It will only then check for the +existence of an SRV record. + +That means that the SRV record will often be redundant. However, you should +remember that there may still be older versions of Synapse in the federation +which do not understand .well-known URIs, so if you removed your SRV record you +would no longer be able to federate with them. + +It is therefore best to leave the SRV record in place for now. Synapse 0.34 and +earlier will follow the SRV record (and not care about the invalid +certificate). Synapse 0.99 and later will follow the .well-known URI, with the +correct certificate chain. + +### It used to work just fine, why are you breaking everything? + +We have always wanted Matrix servers to be as easy to set up as possible, and +so back when we started federation in 2014 we didn't want admins to have to go +through the cumbersome process of buying a valid TLS certificate to run a +server. This was before Let's Encrypt came along and made getting a free and +valid TLS certificate straightforward. So instead, we adopted a system based on +[Perspectives](https://en.wikipedia.org/wiki/Convergence_(SSL)): an approach +where you check a set of "notary servers" (in practice, homeservers) to vouch +for the validity of a certificate rather than having it signed by a CA. As long +as enough different notaries agree on the certificate's validity, then it is +trusted. + +However, in practice this has never worked properly. Most people only use the +default notary server (matrix.org), leading to inadvertent centralisation which +we want to eliminate. Meanwhile, we never implemented the full consensus +algorithm to query the servers participating in a room to determine consensus +on whether a given certificate is valid. This is fiddly to get right +(especially in face of sybil attacks), and we found ourselves questioning +whether it was worth the effort to finish the work and commit to maintaining a +secure certificate validation system as opposed to focusing on core Matrix +development. + +Meanwhile, Let's Encrypt came along in 2016, and put the final nail in the +coffin of the Perspectives project (which was already pretty dead). So, the +Spec Core Team decided that a better approach would be to mandate valid TLS +certificates for federation alongside the rest of the Web. More details can be +found in +[MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md#background-the-failure-of-the-perspectives-approach). + +This results in a breaking change, which is disruptive, but absolutely critical +for the security model. However, the existence of Let's Encrypt as a trivial +way to replace the old self-signed certificates with valid CA-signed ones helps +smooth things over massively, especially as Synapse can now automate Let's +Encrypt certificate generation if needed. + +### Can I manage my own certificates rather than having Synapse renew certificates itself? + +Yes, you are welcome to manage your certificates yourself. Synapse will only +attempt to obtain certificates from Let's Encrypt if you configure it to do +so.The only requirement is that there is a valid TLS cert present for +federation end points. + +### Do you still recommend against using a reverse-proxy on the federation port? + +We no longer actively recommend against using a reverse proxy. Many admins will +find it easier to direct federation traffic to a reverse-proxy and manage their +own TLS certificates, and this is a supported configuration. + +### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy? + +Practically speaking, this is no longer necessary. + +If you are using a reverse-proxy for all of your TLS traffic, then you can set +`no_tls: True`. In that case, the only reason Synapse needs the certificate is +to populate a legacy 'tls_fingerprints' field in the federation API. This is +ignored by Synapse 0.99.0 and later, and the only time pre-0.99 Synapses will +check it is when attempting to fetch the server keys - and generally this is +delegated via `matrix.org`, which is on 0.99.0. + +However, there is a bug in Synapse 0.99.0 +[4554]() which prevents +Synapse from starting if you do not give it a TLS certificate. To work around +this, you can give it any TLS certificate at all. This will be fixed soon. + +### Do I need the same certificate for the client and federation port? + +No. There is nothing stopping you doing so, particularly if you are using a +reverse-proxy. However, Synapse will use the same certificate on any ports +where TLS is configured. + +### How do I tell Synapse to reload my keys/certificates after I replace them? + +Synapse will reload the keys and certificates when it receives a SIGHUP - for +example kill -HUP $(cat homeserver.pid). Alternatively, simply restart Synapse, +though this will result in downtime while it restarts. From 61dc53abe9ac6fce9a69a6655262853a66c09b11 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 5 Feb 2019 17:36:05 +0000 Subject: [PATCH 52/96] fix some thinkos in UPGRADE.rst --- UPGRADE.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index 75aef366bd..3643477250 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -5,20 +5,20 @@ Before upgrading check if any special steps are required to upgrade from the what you currently have installed to current version of synapse. The extra instructions that may be required are listed later in this document. -1. If synapse was installed in a virtualenv then active that virtualenv before - upgrading. If synapse is installed in a virtualenv in ``~/.synapse/`` then +1. If synapse was installed in a virtualenv then activate that virtualenv before + upgrading. If synapse is installed in a virtualenv in ``~/synapse/env`` then run: .. code:: bash - source ~/.synapse/bin/activate + source ~/synapse/env/bin/activate 2. If synapse was installed using pip then upgrade to the latest version by running: .. code:: bash - pip install --upgrade matrix-synapse + pip install --upgrade matrix-synapse[all] # restart synapse synctl restart @@ -31,8 +31,9 @@ instructions that may be required are listed later in this document. # Pull the latest version of the master branch. git pull - # Update the versions of synapse's python dependencies. - python synapse/python_dependencies.py | xargs pip install --upgrade + + # Update synapse and its python dependencies. + pip install --upgrade .[all] # restart synapse ./synctl restart From 39bf0ea2e860948adcd47f0f9cf8ff73b8abf841 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 18:11:26 +0000 Subject: [PATCH 53/96] Add notes on SRV and .well-known (#4573) --- docs/MSC1711_certificates_FAQ.md | 156 +++++++++++++++++++++++-------- 1 file changed, 116 insertions(+), 40 deletions(-) diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 0dcef57733..efe6330647 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -19,19 +19,9 @@ certificate.** Admins will have 1 month to do so, after which 1.0.0 will be released and those servers without a valid certificate will not longer be able to federate with >= 1.0.0 servers. -If you are unable to generate a valid TLS certificate for your server (e.g. -because you run it on behalf of someone who doesn't want to give you a TLS -certificate for their domain, or simply because the matrix domain is hosted on -a different server), then you can now create a /.well-known/matrix/server file -on the matrix domain in order to delegate Matrix hosting to another domain. - Admins who currently use SRV records to delegate a domain **which they do not -control TLS for** will need to switch to using .well-known/matrix/server - though -they should retain their SRV record while the federation upgrades over the -course of the month.  Other SRV records are unaffected. - -Full upgrade notes can be found in -[UPGRADE.rst](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst). -What follows is a timeline and some frequently asked questions. +Full details on how to carry out this configuration change is given +[below](#configuring-certificates-for-compatibility-with-synapse-100). A +timeline and some frequently asked questions are also given below. For more details and context on the release of the r0.1 Server/Server API and imminent Matrix 1.0 release, you can also see our @@ -39,25 +29,26 @@ imminent Matrix 1.0 release, you can also see our ## Contents * Timeline -* Synapse 0.99.0 has just been released, what do I need to do right now? -* How do I upgrade? -* What will happen if I do not set up a valid federation certificate - immediately? -* What will happen if I do nothing at all? -* When do I need a SRV record or .well-known URI? -* Can I still use an SRV record? -* I have created a .well-known URI. Do I still need an SRV record? -* It used to work just fine, why are you breaking everything? -* Can I manage my own certificates rather than having Synapse renew - certificates itself? -* Do you still recommend against using a reverse-proxy on the federation port? -* Do I still need to give my TLS certificates to Synapse if I am using a - reverse-proxy? -* Do I need the same certificate for the client and federation port? -* How do I tell Synapse to reload my keys/certificates after I replace them? +* Configuring certificates for compatibility with Synapse 1.0 +* FAQ + * Synapse 0.99.0 has just been released, what do I need to do right now? + * How do I upgrade? + * What will happen if I do not set up a valid federation certificate + immediately? + * What will happen if I do nothing at all? + * When do I need a SRV record or .well-known URI? + * Can I still use an SRV record? + * I have created a .well-known URI. Do I still need an SRV record? + * It used to work just fine, why are you breaking everything? + * Can I manage my own certificates rather than having Synapse renew + certificates itself? + * Do you still recommend against using a reverse-proxy on the federation port? + * Do I still need to give my TLS certificates to Synapse if I am using a + reverse-proxy? + * Do I need the same certificate for the client and federation port? + * How do I tell Synapse to reload my keys/certificates after I replace them? - -### Timeline +## Timeline **5th Feb 2019 - Synapse 0.99.0 is released.** @@ -82,10 +73,96 @@ alongside their .well-known record. 1.0.0 will land no sooner than 1 month after 0.99.0, leaving server admins one month after 5th February to upgrade to 0.99.0 and deploy their certificates. In accordance with the the [S2S spec](https://matrix.org/docs/spec/server_server/r0.1.0.html) -1.0.0 will enforce federation checks. This means that any homeserver without a +1.0.0 will enforce certificate validity. This means that any homeserver without a valid certificate after this point will no longer be able to federate with 1.0.0 servers. + +## Configuring certificates for compatibility with Synapse 1.0.0 + +### If you do not currently have an SRV record + +In this case, your `server_name` points to the host where your Synapse is +running. There is no need to create a `.well-known` URI or an SRV record, but +you will need to give Synapse a valid, signed, certificate. + +The easiest way to do that is with Synapse's built-in ACME (Let's Encrypt) +support. Full details are in [ACME.md](./ACME.md) but, in a nutshell: + + 1. Allow Synapse to listen on port 80 with `authbind`, or forward it from a + reverse proxy. + 2. Enable acme support in `homeserver.yaml`. + 3. Move your old certificates out of the way. + 4. Restart Synapse. + +### If you do have an SRV record currently + +If you are using an SRV record, your matrix domain (`server_name`) may not +point to the same host that your Synapse is running on (the 'target +domain'). (If it does, you can follow the recommendation above; otherwise, read +on.) + +Let's assume that your `server_name` is `example.com`, and your Synapse is +hosted at a target domain of `customer.example.net`. Currently you should have +an SRV record which looks like: + +``` +_matrix._tcp.example.com. IN SRV 10 5 443 customer.example.net. +``` + +In this situation, you have two choices for how to proceed: + +#### Option 1: give Synapse a certificate for your matrix domain + +Synapse 1.0 will expect your server to present a TLS certificate for your +`server_name` (`example.com` in the above example). You can achieve this by +doing one of the following: + + * Acquire a certificate for the `server_name` yourself (for example, using + `certbot`), and give it and the key to Synapse via `tls_certificate_path` + and `tls_private_key_path`, or: + + * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the + `server_name` domain to your Synapse instance, or: + + * Set up a reverse-proxy on port 8448 on the `server_name` domain, which + forwards to Synapse. Once it is set up, you can remove the SRV record. + +#### Option 2: add a .well-known file to delegate your matrix traffic + +This will allow you to keep Synapse on a separate domain, without having to +give it a certificate for the matrix domain. + +You can do this with a `.well-known` file as follows: + + 1. Keep the SRV record in place - it is needed for backwards compatibility + with Synapse 0.34 and earlier. + + 2. Give synapse a certificate corresponding to the target domain + (`customer.example.net` in the above example). Currently Synapse's ACME + support [does not support + this](https://github.com/matrix-org/synapse/issues/4552), so you will have + to acquire a certificate yourself and give it to Synapse via + `tls_certificate_path` and `tls_private_key_path`. + + 3. Restart Synapse to ensure the new certificate is loaded. + + 4. Arrange for a `.well-known` file at + `https:///.well-known/matrix/server` with contents: + + ```json + {"m.server": ":"} + ``` + + In the above example, `https://example.com/.well-known/matrix/server` + should have the contents: + + ```json + {"m.server": "customer.example.net:443"} + ``` + +## FAQ + ### Synapse 0.99.0 has just been released, what do I need to do right now? Upgrade as soon as you can in preparation for Synapse 1.0.0. @@ -126,14 +203,13 @@ other servers know how to find it. The easiest way to do this is with a .well-known/matrix/server URI on the webroot of the domain to advertise your server. For instance, if you ran -"matrixhosting.com" and you were hosting a Matrix server for example.com, you -would ask example.com to create a file at: +"matrixhosting.com" and you were hosting a Matrix server for `example.com`, you +would ask `example.com` to create a file at +`https://example.com/.well-known/matrix/server` with contents: -`https://example.com/.well-known/matrix/server` - -with contents: - -`{"m.server": "example.matrixhosting.com:8448"}` +```json +{"m.server": "example.matrixhosting.com:8448"} +``` ...which would tell servers trying to connect to example.com to instead connect to example.matrixhosting.com on port 8448. You would then configure Synapse @@ -231,7 +307,7 @@ We no longer actively recommend against using a reverse proxy. Many admins will find it easier to direct federation traffic to a reverse-proxy and manage their own TLS certificates, and this is a supported configuration. -### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy? +### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy? Practically speaking, this is no longer necessary. From 3bd9daf4b86ed811c9ced95a4adb9aa58b681399 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 5 Feb 2019 18:33:02 +0000 Subject: [PATCH 54/96] v0.99.0 --- CHANGES.md | 71 +++++++++------------------------------- changelog.d/4547.misc | 1 - changelog.d/4557.misc | 1 - changelog.d/4558.feature | 1 - changelog.d/4562.misc | 1 - changelog.d/4564.bugfix | 1 - changelog.d/4566.feature | 1 - changelog.d/4572.misc | 1 - debian/changelog | 6 ++++ synapse/__init__.py | 2 +- 10 files changed, 23 insertions(+), 63 deletions(-) delete mode 100644 changelog.d/4547.misc delete mode 100644 changelog.d/4557.misc delete mode 100644 changelog.d/4558.feature delete mode 100644 changelog.d/4562.misc delete mode 100644 changelog.d/4564.bugfix delete mode 100644 changelog.d/4566.feature delete mode 100644 changelog.d/4572.misc diff --git a/CHANGES.md b/CHANGES.md index 66991340a4..e330aea9e3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,52 +1,5 @@ -Synapse 0.99.0rc4 (2019-02-01) -============================== - -Internal Changes ----------------- - -- Update federation routing logic to check .well-known before SRV ([\#4539](https://github.com/matrix-org/synapse/issues/4539)) -- Improve performance of handling servers with invalid .well-known ([\#4542](https://github.com/matrix-org/synapse/issues/4542)) -- Treat an invalid .well-known file the same as an absent one ([\#4544](https://github.com/matrix-org/synapse/issues/4544)) - - -Synapse 0.99.0rc3 (2019-01-31) -============================== - -Bugfixes --------- - -- Fix infinite loop when an event is redacted in a v3 room ([\#4535](https://github.com/matrix-org/synapse/issues/4535)) - - -Improved Documentation ----------------------- - -- Update debian installation instructions ([\#4526](https://github.com/matrix-org/synapse/issues/4526)) - - -Internal Changes ----------------- - -- Add some debug for membership syncing issues ([\#4538](https://github.com/matrix-org/synapse/issues/4538)) - - -Synapse 0.99.0rc2 (2019-01-30) -============================== - -Bugfixes --------- - -- Fix bug when rejecting remote invites. ([\#4527](https://github.com/matrix-org/synapse/issues/4527)) -- Fix incorrect rendering of server capabilities. ([81b7e7eed](https://github.com/matrix-org/synapse/commit/81b7e7eed323f55d6550e7a270a9dc2c4c7b0fe0)) - -Improved Documentation ----------------------- - -- Add documentation on enabling ACME support when upgrading to v0.99. ([\#4528](https://github.com/matrix-org/synapse/issues/4528)) - - -Synapse 0.99.0rc1 (2019-01-30) -============================== +Synapse 0.99.0 (2019-02-05) +=========================== Synapse v0.99.x is a precursor to the upcoming Synapse v1.0 release. It contains foundational changes to room architecture and the federation security model necessary to support the upcoming r0 release of the Server to Server API. @@ -54,15 +7,15 @@ Features -------- - Synapse's cipher string has been updated to require ECDH key exchange. Configuring and generating dh_params is no longer required, and they will be ignored. ([\#4229](https://github.com/matrix-org/synapse/issues/4229)) -- Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). ([\#4384](https://github.com/matrix-org/synapse/issues/4384), [\#4492](https://github.com/matrix-org/synapse/issues/4492), [\#4525](https://github.com/matrix-org/synapse/issues/4525)) -- Implement MSC1708 (.well-known routing for server-server federation) ([\#4408](https://github.com/matrix-org/synapse/issues/4408), [\#4409](https://github.com/matrix-org/synapse/issues/4409), [\#4426](https://github.com/matrix-org/synapse/issues/4426), [\#4427](https://github.com/matrix-org/synapse/issues/4427), [\#4428](https://github.com/matrix-org/synapse/issues/4428), [\#4464](https://github.com/matrix-org/synapse/issues/4464), [\#4468](https://github.com/matrix-org/synapse/issues/4468), [\#4487](https://github.com/matrix-org/synapse/issues/4487), [\#4488](https://github.com/matrix-org/synapse/issues/4488), [\#4489](https://github.com/matrix-org/synapse/issues/4489), [\#4497](https://github.com/matrix-org/synapse/issues/4497), [\#4511](https://github.com/matrix-org/synapse/issues/4511), [\#4516](https://github.com/matrix-org/synapse/issues/4516), [\#4520](https://github.com/matrix-org/synapse/issues/4520), [\#4521](https://github.com/matrix-org/synapse/issues/4521)) +- Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). ([\#4384](https://github.com/matrix-org/synapse/issues/4384), [\#4492](https://github.com/matrix-org/synapse/issues/4492), [\#4525](https://github.com/matrix-org/synapse/issues/4525), [\#4572](https://github.com/matrix-org/synapse/issues/4572), [\#4564](https://github.com/matrix-org/synapse/issues/4564), [\#4566](https://github.com/matrix-org/synapse/issues/4566), [\#4547](https://github.com/matrix-org/synapse/issues/4547), [\#4557](https://github.com/matrix-org/synapse/issues/4557)) +- Implement MSC1708 (.well-known routing for server-server federation) ([\#4408](https://github.com/matrix-org/synapse/issues/4408), [\#4409](https://github.com/matrix-org/synapse/issues/4409), [\#4426](https://github.com/matrix-org/synapse/issues/4426), [\#4427](https://github.com/matrix-org/synapse/issues/4427), [\#4428](https://github.com/matrix-org/synapse/issues/4428), [\#4464](https://github.com/matrix-org/synapse/issues/4464), [\#4468](https://github.com/matrix-org/synapse/issues/4468), [\#4487](https://github.com/matrix-org/synapse/issues/4487), [\#4488](https://github.com/matrix-org/synapse/issues/4488), [\#4489](https://github.com/matrix-org/synapse/issues/4489), [\#4497](https://github.com/matrix-org/synapse/issues/4497), [\#4511](https://github.com/matrix-org/synapse/issues/4511), [\#4516](https://github.com/matrix-org/synapse/issues/4516), [\#4520](https://github.com/matrix-org/synapse/issues/4520), [\#4521](https://github.com/matrix-org/synapse/issues/4521), [\#4539](https://github.com/matrix-org/synapse/issues/4539), [\#4542](https://github.com/matrix-org/synapse/issues/4542), [\#4544](https://github.com/matrix-org/synapse/issues/4544)) - Search now includes results from predecessor rooms after a room upgrade. ([\#4415](https://github.com/matrix-org/synapse/issues/4415)) - Config option to disable requesting MSISDN on registration. ([\#4423](https://github.com/matrix-org/synapse/issues/4423)) - Add a metric for tracking event stream position of the user directory. ([\#4445](https://github.com/matrix-org/synapse/issues/4445)) -- Support exposing server capabilities in CS API (MSC1753, MSC1804) ([\#4472](https://github.com/matrix-org/synapse/issues/4472)) -- Add support for room version 3 ([\#4483](https://github.com/matrix-org/synapse/issues/4483), [\#4499](https://github.com/matrix-org/synapse/issues/4499), [\#4515](https://github.com/matrix-org/synapse/issues/4515), [\#4523](https://github.com/matrix-org/synapse/issues/4523)) +- Support exposing server capabilities in CS API (MSC1753, MSC1804) ([\#4472](https://github.com/matrix-org/synapse/issues/4472), [81b7e7eed](https://github.com/matrix-org/synapse/commit/81b7e7eed323f55d6550e7a270a9dc2c4c7b0fe0))) +- Add support for room version 3 ([\#4483](https://github.com/matrix-org/synapse/issues/4483), [\#4499](https://github.com/matrix-org/synapse/issues/4499), [\#4515](https://github.com/matrix-org/synapse/issues/4515), [\#4523](https://github.com/matrix-org/synapse/issues/4523), [\#4535](https://github.com/matrix-org/synapse/issues/4535)) - Synapse will now reload TLS certificates from disk upon SIGHUP. ([\#4495](https://github.com/matrix-org/synapse/issues/4495), [\#4524](https://github.com/matrix-org/synapse/issues/4524)) - +- The matrixdotorg/synapse Docker images now use Python 3 by default. ([\#4558](https://github.com/matrix-org/synapse/issues/4558)) Bugfixes -------- @@ -71,7 +24,7 @@ Bugfixes - Fix typo in ALL_USER_TYPES definition to ensure type is a tuple ([\#4392](https://github.com/matrix-org/synapse/issues/4392)) - Fix high CPU usage due to remote devicelist updates ([\#4397](https://github.com/matrix-org/synapse/issues/4397)) - Fix potential bug where creating or joining a room could fail ([\#4404](https://github.com/matrix-org/synapse/issues/4404)) -- Fix bug when rejecting remote invites ([\#4405](https://github.com/matrix-org/synapse/issues/4405)) +- Fix bug when rejecting remote invites ([\#4405](https://github.com/matrix-org/synapse/issues/4405), [\#4527](https://github.com/matrix-org/synapse/issues/4527)) - Fix incorrect logcontexts after a Deferred was cancelled ([\#4407](https://github.com/matrix-org/synapse/issues/4407)) - Ensure encrypted room state is persisted across room upgrades. ([\#4411](https://github.com/matrix-org/synapse/issues/4411)) - Copy over whether a room is a direct message and any associated room tags on room upgrade. ([\#4412](https://github.com/matrix-org/synapse/issues/4412)) @@ -89,6 +42,12 @@ Deprecations and Removals - Synapse no longer generates self-signed TLS certificates when generating a configuration file. ([\#4509](https://github.com/matrix-org/synapse/issues/4509)) +Improved Documentation +---------------------- + +- Update debian installation instructions ([\#4526](https://github.com/matrix-org/synapse/issues/4526)) + + Internal Changes ---------------- @@ -111,6 +70,8 @@ Internal Changes - Make it possible to set the log level for tests via an environment variable ([\#4506](https://github.com/matrix-org/synapse/issues/4506)) - Reduce the log level of linearizer lock acquirement to DEBUG. ([\#4507](https://github.com/matrix-org/synapse/issues/4507)) - Fix code to comply with linting in PyFlakes 3.7.1. ([\#4519](https://github.com/matrix-org/synapse/issues/4519)) +- Add some debug for membership syncing issues ([\#4538](https://github.com/matrix-org/synapse/issues/4538)) +- Docker: only copy what we need to the build image ([\#4562](https://github.com/matrix-org/synapse/issues/4562)) Synapse 0.34.1.1 (2019-01-11) diff --git a/changelog.d/4547.misc b/changelog.d/4547.misc deleted file mode 100644 index b6e421d095..0000000000 --- a/changelog.d/4547.misc +++ /dev/null @@ -1 +0,0 @@ -Add docs for ACME setup to README. \ No newline at end of file diff --git a/changelog.d/4557.misc b/changelog.d/4557.misc deleted file mode 100644 index 7ebfb23f43..0000000000 --- a/changelog.d/4557.misc +++ /dev/null @@ -1 +0,0 @@ -Fix comment typo in TLS section of config diff --git a/changelog.d/4558.feature b/changelog.d/4558.feature deleted file mode 100644 index 40724d1c3f..0000000000 --- a/changelog.d/4558.feature +++ /dev/null @@ -1 +0,0 @@ -The matrixdotorg/synapse Docker images now use Python 3 by default. \ No newline at end of file diff --git a/changelog.d/4562.misc b/changelog.d/4562.misc deleted file mode 100644 index f7185fa768..0000000000 --- a/changelog.d/4562.misc +++ /dev/null @@ -1 +0,0 @@ -Docker: only copy what we need to the build image diff --git a/changelog.d/4564.bugfix b/changelog.d/4564.bugfix deleted file mode 100644 index 78e78504b4..0000000000 --- a/changelog.d/4564.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix default ACME config for py2 diff --git a/changelog.d/4566.feature b/changelog.d/4566.feature deleted file mode 100644 index 11fc07476e..0000000000 --- a/changelog.d/4566.feature +++ /dev/null @@ -1 +0,0 @@ -enable ACME support in the docker image diff --git a/changelog.d/4572.misc b/changelog.d/4572.misc deleted file mode 100644 index ea5d49b706..0000000000 --- a/changelog.d/4572.misc +++ /dev/null @@ -1 +0,0 @@ -FAQ to help server admins configure TLS certs in 0.99.0 diff --git a/debian/changelog b/debian/changelog index e6c174e02d..04b5d69053 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (0.99.0) stable; urgency=medium + + * New synapse release 0.99.0 + + -- Synapse Packaging team Tue, 5 Feb 2019 18:25:00 +0000 + matrix-synapse-py3 (0.34.1.1++1) stable; urgency=medium * Update conflicts specifications to allow smoother transition from matrix-synapse. diff --git a/synapse/__init__.py b/synapse/__init__.py index 54745af283..048d6e572f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.0rc4" +__version__ = "0.99.0" From b05dd4ac06e88797477ef1991d1b2e44583afb9a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 5 Feb 2019 18:59:57 +0000 Subject: [PATCH 55/96] faq cleanups --- UPGRADE.rst | 2 +- docs/MSC1711_certificates_FAQ.md | 54 +++++++++++--------------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index 3643477250..d869b7111b 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -57,7 +57,7 @@ will need to replace any self-signed certificates with those verified by a root CA. Information on how to do so can be found at `the ACME docs `_. -For more information on configuring TLS certificates see the `FAQ `_ +For more information on configuring TLS certificates see the `FAQ `_. Upgrading to v0.34.0 ==================== diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index efe6330647..eee37d9457 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -1,4 +1,4 @@ -# MSC 1711 Certificates FAQ +# MSC1711 Certificates FAQ The goal of Synapse 0.99.0 is to act as a stepping stone to Synapse 1.0.0. It supports the r0.1 release of the server to server specification, but is @@ -165,11 +165,8 @@ You can do this with a `.well-known` file as follows: ### Synapse 0.99.0 has just been released, what do I need to do right now? -Upgrade as soon as you can in preparation for Synapse 1.0.0. - -### How do I upgrade? - -Follow the upgrade notes here [UPGRADE.rst](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst) +Upgrade as soon as you can in preparation for Synapse 1.0.0, and update your +TLS certificates as [above](#configuring-certificates-for-compatibility-with-synapse-100). ### What will happen if I do not set up a valid federation certificate immediately? @@ -186,39 +183,24 @@ homeserver will not be able to federate with any Synapse >= 1.0.0 ### When do I need a SRV record or .well-known URI? If your homeserver listens on the default federation port (8448), and your -server_name points to the host that your homeserver runs on, you do not need an -SRV record or .well-known/matrix/server URI.\ -For instance, if you registered example.com and pointed its DNS A record at a +`server_name` points to the host that your homeserver runs on, you do not need an +SRV record or `.well-known/matrix/server` URI. + +For instance, if you registered `example.com` and pointed its DNS A record at a fresh Upcloud VPS or similar, you could install Synapse 0.99 on that host, -giving it a server_name of example.com, and it would automatically generate a +giving it a server_name of `example.com`, and it would automatically generate a valid TLS certificate for you via Let's Encrypt and no SRV record or -.well-known URI would be needed. +`.well-known` URI would be needed. This is the common case, although you can add an SRV record or -.well-known/matrix/server URI for completeness if you wish. +`.well-known/matrix/server` URI for completeness if you wish. -**However**, if your server does not listen on port 8448, or if your server_name +**However**, if your server does not listen on port 8448, or if your `server_name` does not point to the host that your homeserver runs on, you will need to let other servers know how to find it. -The easiest way to do this is with a .well-known/matrix/server URI on the -webroot of the domain to advertise your server. For instance, if you ran -"matrixhosting.com" and you were hosting a Matrix server for `example.com`, you -would ask `example.com` to create a file at -`https://example.com/.well-known/matrix/server` with contents: - -```json -{"m.server": "example.matrixhosting.com:8448"} -``` - -...which would tell servers trying to connect to example.com to instead connect -to example.matrixhosting.com on port 8448. You would then configure Synapse -with a server_name of "example.com", but generate a TLS certificate for -example.matrixhosting.com. - -As an alternative, you can still use an SRV DNS record for the delegation, but -this will require you to have a certificate for the matrix domain (example.com -in this example). See "Can I still use an SRV record?". +In this case, you should see ["If you do have an SRV record +currently"](#if-you-do-have-an-srv-record-currently) above. ### Can I still use an SRV record? @@ -244,13 +226,13 @@ also need to use a .well-known URI instead. However, see also "I have created a ### I have created a .well-known URI. Do I still need an SRV record? -As of Synapse 0.99, Synapse will first check for the existence of a .well-known -URL and follow any delegation it suggests. It will only then check for the +As of Synapse 0.99, Synapse will first check for the existence of a `.well-known` +URI and follow any delegation it suggests. It will only then check for the existence of an SRV record. That means that the SRV record will often be redundant. However, you should remember that there may still be older versions of Synapse in the federation -which do not understand .well-known URIs, so if you removed your SRV record you +which do not understand `.well-known` URIs, so if you removed your SRV record you would no longer be able to federate with them. It is therefore best to leave the SRV record in place for now. Synapse 0.34 and @@ -332,5 +314,5 @@ where TLS is configured. ### How do I tell Synapse to reload my keys/certificates after I replace them? Synapse will reload the keys and certificates when it receives a SIGHUP - for -example kill -HUP $(cat homeserver.pid). Alternatively, simply restart Synapse, -though this will result in downtime while it restarts. +example `kill -HUP $(cat homeserver.pid)`. Alternatively, simply restart +Synapse, though this will result in downtime while it restarts. From d8e63846e23537d534f94df1e4294326964bb780 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 6 Feb 2019 09:41:54 +0000 Subject: [PATCH 56/96] Fix docker upload job to push -py2 images (#4576) --- .circleci/config.yml | 2 ++ changelog.d/4576.misc | 1 + 2 files changed, 3 insertions(+) create mode 100644 changelog.d/4576.misc diff --git a/.circleci/config.yml b/.circleci/config.yml index ee72ad7a14..137747dae3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,7 @@ jobs: - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 . - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - run: docker push matrixdotorg/synapse:${CIRCLE_TAG} + - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py2 - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3 dockerhubuploadlatest: machine: true @@ -17,6 +18,7 @@ jobs: - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 . - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - run: docker push matrixdotorg/synapse:latest + - run: docker push matrixdotorg/synapse:latest-py2 - run: docker push matrixdotorg/synapse:latest-py3 sytestpy2: docker: diff --git a/changelog.d/4576.misc b/changelog.d/4576.misc new file mode 100644 index 0000000000..94b1ade2e3 --- /dev/null +++ b/changelog.d/4576.misc @@ -0,0 +1 @@ +Fix docker upload job to push -py2 images. From 664c81e8b7525bfaa5c3a7620f2f831ade0754a2 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 17:47:22 -0500 Subject: [PATCH 57/96] return proper error codes for some 404s --- synapse/handlers/e2e_room_keys.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 42b040375f..c5d7bf0c29 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -267,7 +267,7 @@ class E2eRoomKeysHandler(object): version(str): Optional; if None gives the most recent version otherwise a historical one. Raises: - StoreError: code 404 if the requested backup version doesn't exist + NotFoundError: if the requested backup version doesn't exist Returns: A deferred of a info dict that gives the info about the new version. @@ -279,7 +279,13 @@ class E2eRoomKeysHandler(object): """ with (yield self._upload_linearizer.queue(user_id)): - res = yield self.store.get_e2e_room_keys_version_info(user_id, version) + try: + res = yield self.store.get_e2e_room_keys_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise defer.returnValue(res) @defer.inlineCallbacks @@ -290,8 +296,14 @@ class E2eRoomKeysHandler(object): user_id(str): the user whose current backup version we're deleting version(str): the version id of the backup being deleted Raises: - StoreError: code 404 if this backup version doesn't exist + NotFoundError: if this backup version doesn't exist """ with (yield self._upload_linearizer.queue(user_id)): - yield self.store.delete_e2e_room_keys_version(user_id, version) + try: + yield self.store.delete_e2e_room_keys_version(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise From 82486371738a130322f3a1829bc4b49f79c1a3e4 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 17:57:10 -0500 Subject: [PATCH 58/96] add new endpoint to update backup versions --- synapse/handlers/e2e_room_keys.py | 34 ++++++++++++++++++++++- synapse/rest/client/v2_alpha/room_keys.py | 33 ++++++++++++++++++++++ synapse/storage/e2e_room_keys.py | 21 ++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index c5d7bf0c29..e0e5ece747 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -19,7 +19,8 @@ from six import iteritems from twisted.internet import defer -from synapse.api.errors import NotFoundError, RoomKeysVersionError, StoreError +from synapse.api.errors import Codes, NotFoundError, RoomKeysVersionError, \ + StoreError, SynapseError from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) @@ -307,3 +308,34 @@ class E2eRoomKeysHandler(object): raise NotFoundError("Unknown backup version") else: raise + + @defer.inlineCallbacks + def update_version(self, user_id, version, version_info): + """Update the info about a given version of the user's backup + + Args: + user_id(str): the user whose current backup version we're updating + version(str): the backup version we're updating + version_info(dict): the new information about the backup + Raises: + NotFoundError: if the requested backup version doesn't exist + Returns: + A deferred of an empty dict. + """ + try: + old_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise + if old_info["algorithm"] != version_info["algorithm"]: + raise SynapseError( + 400, + "Algorithm does not match", + Codes.INVALID_PARAM + ) + + yield self.store.update_e2e_room_keys_version(user_id, version, version_info) + + defer.returnValue({}) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index ab3f1bd21a..1c39d2af1c 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -380,6 +380,39 @@ class RoomKeysVersionServlet(RestServlet): ) defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_PUT(self, request, version): + """ + Update the information about a given version of the user's room_keys backup. + + POST /room_keys/version/12345 HTTP/1.1 + Content-Type: application/json + { + "algorithm": "m.megolm_backup.v1", + "auth_data": { + "public_key": "abcdefg", + "signatures": { + "ed25519:something": "hijklmnop" + } + } + } + + HTTP/1.1 200 OK + Content-Type: application/json + {} + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + info = parse_json_object_from_request(request) + + if version is None: + raise SynapseError(400, "No version specified to update", Codes.MISSING_PARAM) + + yield self.e2e_room_keys_handler.update_version( + user_id, version, info + ) + defer.returnValue((200, {})) + def register_servlets(hs, http_server): RoomKeysServlet(hs).register(http_server) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 45cebe61d1..9a3aec759e 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -298,6 +298,27 @@ class EndToEndRoomKeyStore(SQLBaseStore): "create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn ) + def update_e2e_room_keys_version(self, user_id, version, info): + """Update a given backup version + + Args: + user_id(str): the user whose backup version we're updating + version(str): the version ID of the backup version we're updating + info(dict): the new backup version info to store + """ + + return self._simple_update( + table="e2e_room_keys_versions", + keyvalues={ + "user_id": user_id, + "version": version, + }, + updatevalues={ + "auth_data": json.dumps(info["auth_data"]), + }, + desc="update_e2e_room_keys_version" + ) + def delete_e2e_room_keys_version(self, user_id, version=None): """Delete a given backup version of the user's room keys. Doesn't delete their actual key data. From 9ff620a518739bfe7417bac20365e4acf9c5906e Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 21:32:52 -0500 Subject: [PATCH 59/96] fix import to make isort happy --- synapse/handlers/e2e_room_keys.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index e0e5ece747..6c43c9db76 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -19,8 +19,13 @@ from six import iteritems from twisted.internet import defer -from synapse.api.errors import Codes, NotFoundError, RoomKeysVersionError, \ - StoreError, SynapseError +from synapse.api.errors import ( + Codes, + NotFoundError, + RoomKeysVersionError, + StoreError, + SynapseError +) from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) From 51b73be63bd6931018f98ea00f6b723c6196219a Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 21:39:56 -0500 Subject: [PATCH 60/96] add changelog entry --- changelog.d/4580.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4580.feature diff --git a/changelog.d/4580.feature b/changelog.d/4580.feature new file mode 100644 index 0000000000..a2a5a77dbe --- /dev/null +++ b/changelog.d/4580.feature @@ -0,0 +1 @@ +Add ability to update backup versions \ No newline at end of file From d9e424bf649c1d4ba7b9da7fd64db84cda389e11 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 22:18:41 -0500 Subject: [PATCH 61/96] re-try to make isort happy --- synapse/handlers/e2e_room_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 6c43c9db76..546050f8e7 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -24,7 +24,7 @@ from synapse.api.errors import ( NotFoundError, RoomKeysVersionError, StoreError, - SynapseError + SynapseError, ) from synapse.util.async_helpers import Linearizer From 9b7aa543d9fc90d3a754ab129a619cfff6a71355 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 7 Feb 2019 18:46:02 +0000 Subject: [PATCH 62/96] clarify option 1 --- docs/MSC1711_certificates_FAQ.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index eee37d9457..414af96ef3 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -112,7 +112,7 @@ _matrix._tcp.example.com. IN SRV 10 5 443 customer.example.net. In this situation, you have two choices for how to proceed: -#### Option 1: give Synapse a certificate for your matrix domain +#### Option 1: give Synapse (or a reverse-proxy) a certificate for your matrix domain Synapse 1.0 will expect your server to present a TLS certificate for your `server_name` (`example.com` in the above example). You can achieve this by From c17b128b837f58cb59ba75803e32c8a720cf8501 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 7 Feb 2019 19:18:08 +0000 Subject: [PATCH 63/96] Update ACME docs to include port instructions (#4578) --- changelog.d/4578.misc | 1 + docs/ACME.md | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 changelog.d/4578.misc diff --git a/changelog.d/4578.misc b/changelog.d/4578.misc new file mode 100644 index 0000000000..d1c006bb6b --- /dev/null +++ b/changelog.d/4578.misc @@ -0,0 +1 @@ +Add port configuration information to ACME instructions. \ No newline at end of file diff --git a/docs/ACME.md b/docs/ACME.md index 8fb2bd66a9..e555c7c939 100644 --- a/docs/ACME.md +++ b/docs/ACME.md @@ -41,10 +41,10 @@ placed in Synapse's config directory without the need for any ACME setup. The main steps for enabling ACME support in short summary are: -1. Allow Synapse to listen on port 80 with authbind, or forward it from a reverse-proxy. -1. Set `acme:enabled` to `true` in homeserver.yaml. +1. Allow Synapse to listen for incoming ACME challenges. +1. Enable ACME support in `homeserver.yaml`. 1. Move your old certificates (files `example.com.tls.crt` and `example.com.tls.key` out of the way if they currently exist at the paths specified in `homeserver.yaml`. -1. Restart Synapse +1. Restart Synapse. Detailed instructions for each step are provided below. @@ -71,7 +71,7 @@ location /.well-known/acme-challenge { } ``` -For Apache, add the following to your existing webserver config:: +For Apache, add the following to your existing webserver config: ``` ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge @@ -79,6 +79,14 @@ ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-cha Make sure to restart/reload your webserver after making changes. +Now make the relevant changes in `homeserver.yaml` to enable ACME support: + +``` +acme: + enabled: true + port: 8009 +``` + #### Authbind @@ -102,24 +110,20 @@ sudo touch /etc/authbind/byport/80 sudo chmod 777 /etc/authbind/byport/80 ``` -When Synapse is started, use the following syntax:: +When Synapse is started, use the following syntax: ``` authbind --deep ``` -### Config file editing - -Once Synapse is able to listen on port 80 for ACME challenge -requests, it must be told to perform ACME provisioning by setting `enabled` -to true under the `acme` section in `homeserver.yaml`: +Make the relevant changes in `homeserver.yaml` to enable ACME support: ``` acme: enabled: true ``` -### Starting synapse +### (Re)starting synapse Ensure that the certificate paths specified in `homeserver.yaml` (`tls_certificate_path` and `tls_private_key_path`) do not currently point to any files. Synapse will not provision certificates if files exist, as it does not want to overwrite existing certificates. From 9285d5c2ce897cf71bff42eca2cfd59e04e1b056 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Feb 2019 17:49:53 +0000 Subject: [PATCH 64/96] Update MSC1711 FAQ to be explicit about well-known A surprising number of people are using the well-known method, and are simply copying the example configuration. This is problematic as the example includes an explicit port, which causes inbound federation requests to have the HTTP Host header include the port, upsetting some reverse proxies. Given that, we update the well-known example to be more explicit about the various ways you can set it up, and the consequence of using an explict port. --- docs/MSC1711_certificates_FAQ.md | 38 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index eee37d9457..a3a36d222e 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -107,10 +107,10 @@ hosted at a target domain of `customer.example.net`. Currently you should have an SRV record which looks like: ``` -_matrix._tcp.example.com. IN SRV 10 5 443 customer.example.net. +_matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net. ``` -In this situation, you have two choices for how to proceed: +In this situation, you have three choices for how to proceed: #### Option 1: give Synapse a certificate for your matrix domain @@ -125,10 +125,16 @@ doing one of the following: * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the `server_name` domain to your Synapse instance, or: - * Set up a reverse-proxy on port 8448 on the `server_name` domain, which - forwards to Synapse. Once it is set up, you can remove the SRV record. -#### Option 2: add a .well-known file to delegate your matrix traffic +### Option 2: run Synapse behind a reverse proxy + +If you have an existing reverse proxy set up with correct TLS certificates for +your domain, you can simply route all traffic through the reverse proxy by +updating the SRV record appropriately (or removing it, if the proxy listens on +8448). + + +#### Option 3: add a .well-known file to delegate your matrix traffic This will allow you to keep Synapse on a separate domain, without having to give it a certificate for the matrix domain. @@ -151,15 +157,25 @@ You can do this with a `.well-known` file as follows: `https:///.well-known/matrix/server` with contents: ```json - {"m.server": ":"} + {"m.server": ""} ``` - In the above example, `https://example.com/.well-known/matrix/server` - should have the contents: + where the target server name is resolved as usual (i.e. SRV lookup, falling + back to talking to port 8448). + + In the above example, where synapse is listening on port 8000, + `https://example.com/.well-known/matrix/server` should have `m.server` set to one of: + + 1. `customer.example.net` ─ with a SRV record on + `_matrix._tcp.customer.example.com` pointing to port 8000, or: + + 2. `customer.example.net` ─ updating synapse to listen on the default port + 8448, or: + + 3. `customer.example.net:8000` ─ ensuring that if there is a reverse proxy + on `customer.example.net:8000` it correctly handles HTTP requests with + Host header set to `customer.example.net:8000`. - ```json - {"m.server": "customer.example.net:443"} - ``` ## FAQ From 43e16ea3bccad4231675c6e94631267921c4837d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Feb 2019 18:01:29 +0000 Subject: [PATCH 65/96] Newsfile --- changelog.d/4584.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4584.misc diff --git a/changelog.d/4584.misc b/changelog.d/4584.misc new file mode 100644 index 0000000000..4dec2e2b5c --- /dev/null +++ b/changelog.d/4584.misc @@ -0,0 +1 @@ +Update MSC1711 FAQ to calrify .well-known usage From 7cadc4c918c207a574ea15bd1e3793d8a48b7beb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 7 Feb 2019 19:29:20 +0000 Subject: [PATCH 66/96] cleanups --- docs/MSC1711_certificates_FAQ.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 579c5dffce..0a781d00e3 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -112,7 +112,7 @@ _matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net. In this situation, you have three choices for how to proceed: -#### Option 1: give Synapse (or a reverse-proxy) a certificate for your matrix domain +#### Option 1: give Synapse a certificate for your matrix domain Synapse 1.0 will expect your server to present a TLS certificate for your `server_name` (`example.com` in the above example). You can achieve this by @@ -123,8 +123,7 @@ doing one of the following: and `tls_private_key_path`, or: * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the - `server_name` domain to your Synapse instance, or: - + `server_name` domain to your Synapse instance. ### Option 2: run Synapse behind a reverse proxy @@ -133,7 +132,6 @@ your domain, you can simply route all traffic through the reverse proxy by updating the SRV record appropriately (or removing it, if the proxy listens on 8448). - #### Option 3: add a .well-known file to delegate your matrix traffic This will allow you to keep Synapse on a separate domain, without having to From acb2ac5863d34e2d01fa53598aecda274dc2fb26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Feb 2019 19:30:32 +0000 Subject: [PATCH 67/96] Update MSC1711 FAQ to be explicit about well-known (#4584) A surprising number of people are using the well-known method, and are simply copying the example configuration. This is problematic as the example includes an explicit port, which causes inbound federation requests to have the HTTP Host header include the port, upsetting some reverse proxies. Given that, we update the well-known example to be more explicit about the various ways you can set it up, and the consequence of using an explict port. --- changelog.d/4584.misc | 1 + docs/MSC1711_certificates_FAQ.md | 40 +++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 changelog.d/4584.misc diff --git a/changelog.d/4584.misc b/changelog.d/4584.misc new file mode 100644 index 0000000000..4dec2e2b5c --- /dev/null +++ b/changelog.d/4584.misc @@ -0,0 +1 @@ +Update MSC1711 FAQ to calrify .well-known usage diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 414af96ef3..0a781d00e3 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -107,12 +107,12 @@ hosted at a target domain of `customer.example.net`. Currently you should have an SRV record which looks like: ``` -_matrix._tcp.example.com. IN SRV 10 5 443 customer.example.net. +_matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net. ``` -In this situation, you have two choices for how to proceed: +In this situation, you have three choices for how to proceed: -#### Option 1: give Synapse (or a reverse-proxy) a certificate for your matrix domain +#### Option 1: give Synapse a certificate for your matrix domain Synapse 1.0 will expect your server to present a TLS certificate for your `server_name` (`example.com` in the above example). You can achieve this by @@ -123,12 +123,16 @@ doing one of the following: and `tls_private_key_path`, or: * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the - `server_name` domain to your Synapse instance, or: + `server_name` domain to your Synapse instance. - * Set up a reverse-proxy on port 8448 on the `server_name` domain, which - forwards to Synapse. Once it is set up, you can remove the SRV record. +### Option 2: run Synapse behind a reverse proxy -#### Option 2: add a .well-known file to delegate your matrix traffic +If you have an existing reverse proxy set up with correct TLS certificates for +your domain, you can simply route all traffic through the reverse proxy by +updating the SRV record appropriately (or removing it, if the proxy listens on +8448). + +#### Option 3: add a .well-known file to delegate your matrix traffic This will allow you to keep Synapse on a separate domain, without having to give it a certificate for the matrix domain. @@ -151,15 +155,25 @@ You can do this with a `.well-known` file as follows: `https:///.well-known/matrix/server` with contents: ```json - {"m.server": ":"} + {"m.server": ""} ``` - In the above example, `https://example.com/.well-known/matrix/server` - should have the contents: + where the target server name is resolved as usual (i.e. SRV lookup, falling + back to talking to port 8448). + + In the above example, where synapse is listening on port 8000, + `https://example.com/.well-known/matrix/server` should have `m.server` set to one of: + + 1. `customer.example.net` ─ with a SRV record on + `_matrix._tcp.customer.example.com` pointing to port 8000, or: + + 2. `customer.example.net` ─ updating synapse to listen on the default port + 8448, or: + + 3. `customer.example.net:8000` ─ ensuring that if there is a reverse proxy + on `customer.example.net:8000` it correctly handles HTTP requests with + Host header set to `customer.example.net:8000`. - ```json - {"m.server": "customer.example.net:443"} - ``` ## FAQ From afae8442b56d2e2466812916654396341038c38c Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 8 Feb 2019 01:32:45 -0500 Subject: [PATCH 68/96] make sure version is in body and wrap in linearizer queue also add tests --- synapse/handlers/e2e_room_keys.py | 37 ++++++++---- synapse/rest/client/v2_alpha/room_keys.py | 3 +- tests/handlers/test_e2e_room_keys.py | 72 +++++++++++++++++++++++ 3 files changed, 100 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 546050f8e7..7bc174070e 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -327,20 +327,35 @@ class E2eRoomKeysHandler(object): Returns: A deferred of an empty dict. """ - try: - old_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) - except StoreError as e: - if e.code == 404: - raise NotFoundError("Unknown backup version") - else: - raise - if old_info["algorithm"] != version_info["algorithm"]: + if "version" not in version_info: raise SynapseError( 400, - "Algorithm does not match", + "Missing version in body", + Codes.MISSING_PARAM + ) + if version_info["version"] != version: + raise SynapseError( + 400, + "Version in body does not match", Codes.INVALID_PARAM ) + with (yield self._upload_linearizer.queue(user_id)): + try: + old_info = yield self.store.get_e2e_room_keys_version_info( + user_id, version + ) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise + if old_info["algorithm"] != version_info["algorithm"]: + raise SynapseError( + 400, + "Algorithm does not match", + Codes.INVALID_PARAM + ) - yield self.store.update_e2e_room_keys_version(user_id, version, version_info) + yield self.store.update_e2e_room_keys_version(user_id, version, version_info) - defer.returnValue({}) + defer.returnValue({}) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 1c39d2af1c..220a0de30b 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -394,7 +394,8 @@ class RoomKeysVersionServlet(RestServlet): "signatures": { "ed25519:something": "hijklmnop" } - } + }, + "version": "42" } HTTP/1.1 200 OK diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index c8994f416e..1c49bbbc3c 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -125,6 +125,78 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): "auth_data": "second_version_auth_data", }) + @defer.inlineCallbacks + def test_update_version(self): + """Check that we can update versions. + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": version + }) + self.assertDictEqual(res, {}) + + # check we can retrieve it as the current version + res = yield self.handler.get_version_info(self.local_user) + self.assertDictEqual(res, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": version + }) + + @defer.inlineCallbacks + def test_update_missing_version(self): + """Check that we get a 404 on updating nonexistent versions + """ + res = None + try: + yield self.handler.update_version(self.local_user, "1", { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": "1" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + @defer.inlineCallbacks + def test_update_bad_version(self): + """Check that we get a 400 if the version in the body is missing or + doesn't match + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = None + try: + yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 400) + + res = None + try: + yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": "incorrect" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 400) + @defer.inlineCallbacks def test_delete_missing_version(self): """Check that we get a 404 on deleting nonexistent versions From 9cd33d2f4bc14a165f693e2d3dfe231179e968e8 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 8 Feb 2019 17:25:57 +0000 Subject: [PATCH 69/96] Deduplicate some code in synapse.app (#4567) --- changelog.d/4567.misc | 1 + synapse/app/_base.py | 63 ++++++++++++++++++++++++++++++++ synapse/app/appservice.py | 7 +--- synapse/app/client_reader.py | 13 +------ synapse/app/event_creator.py | 13 +------ synapse/app/federation_reader.py | 13 +------ synapse/app/federation_sender.py | 12 +----- synapse/app/frontend_proxy.py | 13 +------ synapse/app/homeserver.py | 54 ++------------------------- synapse/app/media_repository.py | 13 +------ synapse/app/pusher.py | 3 +- synapse/app/synchrotron.py | 7 +--- synapse/app/user_dir.py | 13 +------ synapse/config/logger.py | 16 +++----- 14 files changed, 83 insertions(+), 158 deletions(-) create mode 100644 changelog.d/4567.misc diff --git a/changelog.d/4567.misc b/changelog.d/4567.misc new file mode 100644 index 0000000000..96a2e0aefc --- /dev/null +++ b/changelog.d/4567.misc @@ -0,0 +1 @@ +Reduce duplication of ``synapse.app`` code. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5b97a54d45..3cbb003035 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -15,7 +15,9 @@ import gc import logging +import signal import sys +import traceback import psutil from daemonize import Daemonize @@ -23,11 +25,25 @@ from daemonize import Daemonize from twisted.internet import error, reactor from synapse.app import check_bind_error +from synapse.crypto import context_factory from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit logger = logging.getLogger(__name__) +_sighup_callbacks = [] + + +def register_sighup(func): + """ + Register a function to be called when a SIGHUP occurs. + + Args: + func (function): Function to be called when sent a SIGHUP signal. + Will be called with a single argument, the homeserver. + """ + _sighup_callbacks.append(func) + def start_worker_reactor(appname, config): """ Run the reactor in the main process @@ -189,3 +205,50 @@ def listen_ssl( logger.info("Synapse now listening on port %d (TLS)", port) return r + + +def refresh_certificate(hs): + """ + Refresh the TLS certificates that Synapse is using by re-reading them from + disk and updating the TLS context factories to use them. + """ + logging.info("Loading certificate from disk...") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + hs.config + ) + logging.info("Certificate loaded.") + + +def start(hs, listeners=None): + """ + Start a Synapse server or worker. + + Args: + hs (synapse.server.HomeServer) + listeners (list[dict]): Listener configuration ('listeners' in homeserver.yaml) + """ + try: + # Set up the SIGHUP machinery. + if hasattr(signal, "SIGHUP"): + def handle_sighup(*args, **kwargs): + for i in _sighup_callbacks: + i(hs) + + signal.signal(signal.SIGHUP, handle_sighup) + + register_sighup(refresh_certificate) + + # Load the certificate from disk. + refresh_certificate(hs) + + # It is now safe to start your Synapse. + hs.start_listening(listeners) + hs.get_datastore().start_profiling() + except Exception: + traceback.print_exc(file=sys.stderr) + reactor = hs.get_reactor() + if reactor.running: + reactor.stop() + sys.exit(1) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 8559e141af..33107f56d1 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -168,12 +168,7 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) - - def start(): - ps.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ps, config.worker_listeners) _base.start_worker_reactor("synapse-appservice", config) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index f8a417cb60..a9d2147022 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -25,7 +25,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -173,17 +172,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-client-reader", config) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 656e0edc0f..b8e5196152 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -25,7 +25,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -194,17 +193,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-event-creator", config) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 3de2715132..42886dbfc1 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -26,7 +26,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-federation-reader", config) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index d944e0517f..a461442fdc 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -25,7 +25,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation import send_queue from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -192,17 +191,8 @@ def start(config_options): ) ss.setup() + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) _base.start_worker_reactor("synapse-federation-sender", config) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d9ef6edc3c..d5b954361d 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -26,7 +26,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseSite @@ -250,17 +249,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-frontend-proxy", config) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 250a17cef8..1a341568ac 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,6 @@ import gc import logging import os -import signal import sys import traceback @@ -28,7 +27,6 @@ from prometheus_client import Gauge from twisted.application import service from twisted.internet import defer, reactor -from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.resource import EncodingResourceWrapper, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File @@ -49,7 +47,6 @@ from synapse.app import _base from synapse.app._base import listen_ssl, listen_tcp, quit_with_error from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.additional_resource import AdditionalResource from synapse.http.server import RootRedirect @@ -241,10 +238,10 @@ class SynapseHomeServer(HomeServer): return resources - def start_listening(self): + def start_listening(self, listeners): config = self.get_config() - for listener in config.listeners: + for listener in listeners: if listener["type"] == "http": self._listening_services.extend( self._listener_http(config, listener) @@ -328,20 +325,11 @@ def setup(config_options): # generating config files and shouldn't try to continue. sys.exit(0) - sighup_callbacks = [] synapse.config.logger.setup_logging( config, - use_worker_options=False, - register_sighup=sighup_callbacks.append + use_worker_options=False ) - def handle_sighup(*args, **kwargs): - for i in sighup_callbacks: - i(*args, **kwargs) - - if hasattr(signal, "SIGHUP"): - signal.signal(signal.SIGHUP, handle_sighup) - events.USE_FROZEN_DICTS = config.use_frozen_dicts database_engine = create_engine(config.database_config) @@ -377,31 +365,6 @@ def setup(config_options): hs.setup() - def refresh_certificate(*args): - """ - Refresh the TLS certificates that Synapse is using by re-reading them - from disk and updating the TLS context factories to use them. - """ - logging.info("Reloading certificate from disk...") - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - logging.info("Certificate reloaded.") - - logging.info("Updating context factories...") - for i in hs._listening_services: - if isinstance(i.factory, TLSMemoryBIOFactory): - i.factory = TLSMemoryBIOFactory( - hs.tls_server_context_factory, - False, - i.factory.wrappedFactory - ) - logging.info("Context factories updated.") - - sighup_callbacks.append(refresh_certificate) - @defer.inlineCallbacks def start(): try: @@ -425,18 +388,9 @@ def setup(config_options): ): yield acme.provision_certificate() - # Read the certificate from disk and build the context factories for - # TLS. - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) + _base.start(hs, config.listeners) - # It is now safe to start your Synapse. - hs.start_listening() hs.get_pusherpool().start() - hs.get_datastore().start_profiling() hs.get_datastore().start_doing_background_updates() except Exception as e: # If a DeferredList failed (like in listening on the ACME listener), diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 4ecf64031b..d4cc4e9443 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -26,7 +26,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-media-repository", config) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 83b0863f00..cbf0d67f51 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -224,11 +224,10 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) def start(): + _base.start(ps, config.worker_listeners) ps.get_pusherpool().start() - ps.get_datastore().start_profiling() reactor.callWhenRunning(start) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 0354e82bf8..9163b56d86 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -445,12 +445,7 @@ def start(config_options): ) ss.setup() - ss.start_listening(config.worker_listeners) - - def start(): - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-synchrotron", config) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 176d55a783..d1ab9512cd 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -26,7 +26,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -220,17 +219,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-user-dir", config) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index a795e39b1a..4b938053fb 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -15,7 +15,6 @@ import logging import logging.config import os -import signal import sys from string import Template @@ -24,6 +23,7 @@ import yaml from twisted.logger import STDLibLogObserver, globalLogBeginner import synapse +from synapse.app import _base as appbase from synapse.util.logcontext import LoggingContextFilter from synapse.util.versionstring import get_version_string @@ -127,7 +127,7 @@ class LoggingConfig(Config): ) -def setup_logging(config, use_worker_options=False, register_sighup=None): +def setup_logging(config, use_worker_options=False): """ Set up python logging Args: @@ -140,12 +140,6 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): register_sighup (func | None): Function to call to register a sighup handler. """ - if not register_sighup: - if getattr(signal, "SIGHUP"): - register_sighup = lambda x: signal.signal(signal.SIGHUP, x) - else: - register_sighup = lambda x: None - log_config = (config.worker_log_config if use_worker_options else config.log_config) log_file = (config.worker_log_file if use_worker_options @@ -187,7 +181,7 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): else: handler = logging.StreamHandler() - def sighup(signum, stack): + def sighup(*args): pass handler.setFormatter(formatter) @@ -200,14 +194,14 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): with open(log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) - def sighup(signum, stack): + def sighup(*args): # it might be better to use a file watcher or something for this. load_log_config() logging.info("Reloaded log config from %s due to SIGHUP", log_config) load_log_config() - register_sighup(sighup) + appbase.register_sighup(sighup) # make sure that the first thing we log is a thing we can grep backwards # for From 56710c7df55edc782f6562d520c4ebc839dfe500 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 8 Feb 2019 18:30:46 +0000 Subject: [PATCH 70/96] Fix 'no unique or exclusion constraint' error (#4591) Add more tables to the list of tables which need a background update to complete before we can upsert into them, which fixes a race against the background updates. --- changelog.d/4591.bugfix | 1 + synapse/storage/_base.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 changelog.d/4591.bugfix diff --git a/changelog.d/4591.bugfix b/changelog.d/4591.bugfix new file mode 100644 index 0000000000..628bbb6d81 --- /dev/null +++ b/changelog.d/4591.bugfix @@ -0,0 +1 @@ +Fix 'no unique or exclusion constraint' error diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4872ff55b6..e124161845 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -50,6 +50,21 @@ sql_query_timer = Histogram("synapse_storage_query_time", "sec", ["verb"]) sql_txn_timer = Histogram("synapse_storage_transaction_time", "sec", ["desc"]) +# Unique indexes which have been added in background updates. Maps from table name +# to the name of the background update which added the unique index to that table. +# +# This is used by the upsert logic to figure out which tables are safe to do a proper +# UPSERT on: until the relevant background update has completed, we +# have to emulate an upsert by locking the table. +# +UNIQUE_INDEX_BACKGROUND_UPDATES = { + "user_ips": "user_ips_device_unique_index", + "device_lists_remote_extremeties": "device_lists_remote_extremeties_unique_idx", + "device_lists_remote_cache": "device_lists_remote_cache_unique_idx", + "event_search": "event_search_event_id_idx", +} + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() @@ -194,7 +209,7 @@ class SQLBaseStore(object): self.database_engine = hs.database_engine # A set of tables that are not safe to use native upserts in. - self._unsafe_to_upsert_tables = {"user_ips"} + self._unsafe_to_upsert_tables = set(UNIQUE_INDEX_BACKGROUND_UPDATES.keys()) # We add the user_directory_search table to the blacklist on SQLite # because the existing search table does not have an index, making it @@ -230,12 +245,12 @@ class SQLBaseStore(object): ) updates = [x["update_name"] for x in updates] - # The User IPs table in schema #53 was missing a unique index, which we - # run as a background update. - if "user_ips_device_unique_index" not in updates: - self._unsafe_to_upsert_tables.discard("user_ips") + for table, update_name in UNIQUE_INDEX_BACKGROUND_UPDATES.items(): + if update_name not in updates: + logger.debug("Now safe to upsert in %s", table) + self._unsafe_to_upsert_tables.discard(table) - # If there's any tables left to check, reschedule to run. + # If there's any updates still running, reschedule to run. if updates: self._clock.call_later( 15.0, From 2dc2b6e9f1b85163c8ebf4d9062253a54d531f1e Mon Sep 17 00:00:00 2001 From: Valentin Anger Date: Fri, 8 Feb 2019 21:09:56 +0000 Subject: [PATCH 71/96] Allow "unavailable" presence status for /sync (#4592) * Allow "unavailable" presence status for /sync Closes #3772, closes #3779 Signed-off-by: Valentin Anger * Add changelog for PR 4592 --- changelog.d/4592.feature | 2 ++ synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/client/versions.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4592.feature diff --git a/changelog.d/4592.feature b/changelog.d/4592.feature new file mode 100644 index 0000000000..112005ded4 --- /dev/null +++ b/changelog.d/4592.feature @@ -0,0 +1,2 @@ +Allow the "unavailable" presence status for /sync. +This change makes Synapse compliant with r0.4.0 of the Client-Server specification. diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 0251146722..39d157a44b 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -75,7 +75,7 @@ class SyncRestServlet(RestServlet): """ PATTERNS = client_v2_patterns("/sync$") - ALLOWED_PRESENCE = set(["online", "offline"]) + ALLOWED_PRESENCE = set(["online", "offline", "unavailable"]) def __init__(self, hs): super(SyncRestServlet, self).__init__() diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 29e62bfcdd..27e7cbf3cc 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -38,6 +38,7 @@ class VersionsRestServlet(RestServlet): "r0.1.0", "r0.2.0", "r0.3.0", + "r0.4.0", ], # as per MSC1497: "unstable_features": { From 4ffd10f46d62f7bc5430d148260e3240b7a0598b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 11 Feb 2019 21:04:27 +1100 Subject: [PATCH 72/96] Be tolerant of blank TLS fingerprints config (#4589) --- changelog.d/4589.bugfix | 1 + synapse/config/tls.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4589.bugfix diff --git a/changelog.d/4589.bugfix b/changelog.d/4589.bugfix new file mode 100644 index 0000000000..d5783f46e8 --- /dev/null +++ b/changelog.d/4589.bugfix @@ -0,0 +1 @@ +Synapse is now tolerant of the tls_fingerprints option being None or not specified. diff --git a/synapse/config/tls.py b/synapse/config/tls.py index b5f2cfd9b7..81b3a659fe 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -45,7 +45,11 @@ class TlsConfig(Config): self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) - self._original_tls_fingerprints = config["tls_fingerprints"] + self._original_tls_fingerprints = config.get("tls_fingerprints", []) + + if self._original_tls_fingerprints is None: + self._original_tls_fingerprints = [] + self.tls_fingerprints = list(self._original_tls_fingerprints) self.no_tls = config.get("no_tls", False) From 6e2a5aa050fc132a7dee6b3e33a7a368207d7e5a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 11 Feb 2019 21:36:26 +1100 Subject: [PATCH 73/96] ACME Reprovisioning (#4522) --- changelog.d/4522.feature | 1 + synapse/app/_base.py | 19 ++++++++++ synapse/app/homeserver.py | 79 +++++++++++++++++++++++++++------------ synapse/config/tls.py | 12 +++++- synapse/server.py | 3 ++ 5 files changed, 89 insertions(+), 25 deletions(-) create mode 100644 changelog.d/4522.feature diff --git a/changelog.d/4522.feature b/changelog.d/4522.feature new file mode 100644 index 0000000000..ef18daf60b --- /dev/null +++ b/changelog.d/4522.feature @@ -0,0 +1 @@ +Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3cbb003035..62c633146f 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -23,6 +23,7 @@ import psutil from daemonize import Daemonize from twisted.internet import error, reactor +from twisted.protocols.tls import TLSMemoryBIOFactory from synapse.app import check_bind_error from synapse.crypto import context_factory @@ -220,6 +221,24 @@ def refresh_certificate(hs): ) logging.info("Certificate loaded.") + if hs._listening_services: + logging.info("Updating context factories...") + for i in hs._listening_services: + # When you listenSSL, it doesn't make an SSL port but a TCP one with + # a TLS wrapping factory around the factory you actually want to get + # requests. This factory attribute is public but missing from + # Twisted's documentation. + if isinstance(i.factory, TLSMemoryBIOFactory): + # We want to replace TLS factories with a new one, with the new + # TLS configuration. We do this by reaching in and pulling out + # the wrappedFactory, and then re-wrapping it. + i.factory = TLSMemoryBIOFactory( + hs.tls_server_context_factory, + False, + i.factory.wrappedFactory + ) + logging.info("Context factories updated.") + def start(hs, listeners=None): """ diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d1cab07bb6..b4476bf16e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -83,7 +83,6 @@ def gz_wrap(r): class SynapseHomeServer(HomeServer): DATASTORE_CLASS = DataStore - _listening_services = [] def _listener_http(self, config, listener_config): port = listener_config["port"] @@ -376,42 +375,73 @@ def setup(config_options): hs.setup() + @defer.inlineCallbacks + def do_acme(): + """ + Reprovision an ACME certificate, if it's required. + + Returns: + Deferred[bool]: Whether the cert has been updated. + """ + acme = hs.get_acme_handler() + + # Check how long the certificate is active for. + cert_days_remaining = hs.config.is_disk_cert_valid( + allow_self_signed=False + ) + + # We want to reprovision if cert_days_remaining is None (meaning no + # certificate exists), or the days remaining number it returns + # is less than our re-registration threshold. + provision = False + + if (cert_days_remaining is None): + provision = True + + if cert_days_remaining > hs.config.acme_reprovision_threshold: + provision = True + + if provision: + yield acme.provision_certificate() + + defer.returnValue(provision) + + @defer.inlineCallbacks + def reprovision_acme(): + """ + Provision a certificate from ACME, if required, and reload the TLS + certificate if it's renewed. + """ + reprovisioned = yield do_acme() + if reprovisioned: + _base.refresh_certificate(hs) + @defer.inlineCallbacks def start(): try: - # Check if the certificate is still valid. - cert_days_remaining = hs.config.is_disk_cert_valid() - + # Run the ACME provisioning code, if it's enabled. if hs.config.acme_enabled: - # If ACME is enabled, we might need to provision a certificate - # before starting. acme = hs.get_acme_handler() - # Start up the webservices which we will respond to ACME - # challenges with. + # challenges with, and then provision. yield acme.start_listening() + yield do_acme() - # We want to reprovision if cert_days_remaining is None (meaning no - # certificate exists), or the days remaining number it returns - # is less than our re-registration threshold. - if (cert_days_remaining is None) or ( - not cert_days_remaining > hs.config.acme_reprovision_threshold - ): - yield acme.provision_certificate() + # Check if it needs to be reprovisioned every day. + hs.get_clock().looping_call( + reprovision_acme, + 24 * 60 * 60 * 1000 + ) _base.start(hs, config.listeners) hs.get_pusherpool().start() hs.get_datastore().start_doing_background_updates() - except Exception as e: - # If a DeferredList failed (like in listening on the ACME listener), - # we need to print the subfailure explicitly. - if isinstance(e, defer.FirstError): - e.subFailure.printTraceback(sys.stderr) - sys.exit(1) - - # Something else went wrong when starting. Print it and bail out. + except Exception: + # Print the exception and bail out. traceback.print_exc(file=sys.stderr) + if reactor.running: + reactor.stop() sys.exit(1) reactor.callWhenRunning(start) @@ -420,7 +450,8 @@ def setup(config_options): class SynapseService(service.Service): - """A twisted Service class that will start synapse. Used to run synapse + """ + A twisted Service class that will start synapse. Used to run synapse via twistd and a .tac. """ def __init__(self, config): diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 81b3a659fe..9fcc79816d 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -64,10 +64,14 @@ class TlsConfig(Config): self.tls_certificate = None self.tls_private_key = None - def is_disk_cert_valid(self): + def is_disk_cert_valid(self, allow_self_signed=True): """ Is the certificate we have on disk valid, and if so, for how long? + Args: + allow_self_signed (bool): Should we allow the certificate we + read to be self signed? + Returns: int: Days remaining of certificate validity. None: No certificate exists. @@ -88,6 +92,12 @@ class TlsConfig(Config): logger.exception("Failed to parse existing certificate off disk!") raise + if not allow_self_signed: + if tls_certificate.get_subject() == tls_certificate.get_issuer(): + raise ValueError( + "TLS Certificate is self signed, and this is not permitted" + ) + # YYYYMMDDhhmmssZ -- in UTC expires_on = datetime.strptime( tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ" diff --git a/synapse/server.py b/synapse/server.py index 6c52101616..a2cf8a91cd 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -112,6 +112,8 @@ class HomeServer(object): Attributes: config (synapse.config.homeserver.HomeserverConfig): + _listening_services (list[twisted.internet.tcp.Port]): TCP ports that + we are listening on to provide HTTP services. """ __metaclass__ = abc.ABCMeta @@ -196,6 +198,7 @@ class HomeServer(object): self._reactor = reactor self.hostname = hostname self._building = {} + self._listening_services = [] self.clock = Clock(reactor) self.distributor = Distributor() From a126f86eec53cd4807a1300bea885122f8559944 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 11 Feb 2019 11:30:37 +0000 Subject: [PATCH 74/96] Transfer Server ACLs on room upgrade --- synapse/handlers/room.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 5e40e9ea46..f9af1f0046 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -284,6 +284,7 @@ class RoomCreationHandler(BaseHandler): (EventTypes.GuestAccess, ""), (EventTypes.RoomAvatar, ""), (EventTypes.Encryption, ""), + (EventTypes.ServerACL, ""), ) old_room_state_ids = yield self.store.get_filtered_current_state_ids( From eff204221748c7cfa0c88162f6fd1758f03a2f4a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 11 Feb 2019 11:41:57 +0000 Subject: [PATCH 75/96] Changelog --- changelog.d/4608.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4608.bugfix diff --git a/changelog.d/4608.bugfix b/changelog.d/4608.bugfix new file mode 100644 index 0000000000..e331a362c4 --- /dev/null +++ b/changelog.d/4608.bugfix @@ -0,0 +1 @@ +Transfer Server ACLs on room upgrade. \ No newline at end of file From c475275926aeee906b76621444468280d5bf569b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 11:44:28 +0000 Subject: [PATCH 76/96] Clarifications for reverse proxy docs (#4607) Factor out the reverse proxy info to a separate file, add some more info on reverse-proxying the federation port. --- INSTALL.md | 5 +- README.rst | 52 +----------------- changelog.d/4607.misc | 1 + docs/MSC1711_certificates_FAQ.md | 22 +++++--- docs/reverse_proxy.rst | 94 ++++++++++++++++++++++++++++++++ docs/workers.rst | 5 +- 6 files changed, 117 insertions(+), 62 deletions(-) create mode 100644 changelog.d/4607.misc create mode 100644 docs/reverse_proxy.rst diff --git a/INSTALL.md b/INSTALL.md index cbe4bda120..e496a13b21 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -359,8 +359,9 @@ Synapse v1.0. Instructions for having Synapse automatically provision and renew If you would like to use your own certificates, you can do so by changing `tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`; -alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, -both ports are the same in the default configuration. +alternatively, you can use a reverse proxy. See +[docs/reverse_proxy.rst](docs/reverse_proxy.rst) for information on configuring +a reverse proxy. ## Registering a user diff --git a/README.rst b/README.rst index e666b3b427..bc7cb5f784 100644 --- a/README.rst +++ b/README.rst @@ -263,6 +263,8 @@ So, things to check are: (it should be ``_matrix._tcp.``), and that the port and hostname it specifies are reachable from outside your network. +.. TODO: add a note about forgetting ``nocanon`` on a reverse-proxy config + Running a Demo Federation of Synapses ------------------------------------- @@ -290,7 +292,6 @@ The advantages of Postgres include: For information on how to install and use PostgreSQL, please see `docs/postgres.rst `_. - .. _reverse-proxy: Using a reverse proxy with Synapse @@ -304,54 +305,7 @@ It is recommended to put a reverse proxy such as doing so is that it means that you can expose the default https port (443) to Matrix clients without needing to run Synapse with root privileges. -The most important thing to know here is that Matrix clients and other Matrix -servers do not necessarily need to connect to your server via the same -port. Indeed, clients will use port 443 by default, whereas servers default to -port 8448. Where these are different, we refer to the 'client port' and the -'federation port'. - -All Matrix endpoints begin with ``/_matrix``, so an example nginx -configuration for forwarding client connections to Synapse might look like:: - - server { - listen 443 ssl; - listen [::]:443 ssl; - server_name matrix.example.com; - - location /_matrix { - proxy_pass http://localhost:8008; - proxy_set_header X-Forwarded-For $remote_addr; - } - } - -an example Caddy configuration might look like:: - - matrix.example.com { - proxy /_matrix http://localhost:8008 { - transparent - } - } - -and an example Apache configuration might look like:: - - - SSLEngine on - ServerName matrix.example.com; - - - ProxyPass http://127.0.0.1:8008/_matrix nocanon - ProxyPassReverse http://127.0.0.1:8008/_matrix - - - -You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true`` -for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are -recorded correctly. - -Having done so, you can then use ``https://matrix.example.com`` (instead of -``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to -Synapse from a client`_. - +For information on configuring one, see ``_. Identity Servers ================ diff --git a/changelog.d/4607.misc b/changelog.d/4607.misc new file mode 100644 index 0000000000..160a824378 --- /dev/null +++ b/changelog.d/4607.misc @@ -0,0 +1 @@ +Clarifications for reverse proxy docs diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 0a781d00e3..2c52b0d517 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -42,9 +42,9 @@ imminent Matrix 1.0 release, you can also see our * It used to work just fine, why are you breaking everything? * Can I manage my own certificates rather than having Synapse renew certificates itself? - * Do you still recommend against using a reverse-proxy on the federation port? + * Do you still recommend against using a reverse proxy on the federation port? * Do I still need to give my TLS certificates to Synapse if I am using a - reverse-proxy? + reverse proxy? * Do I need the same certificate for the client and federation port? * How do I tell Synapse to reload my keys/certificates after I replace them? @@ -132,6 +132,9 @@ your domain, you can simply route all traffic through the reverse proxy by updating the SRV record appropriately (or removing it, if the proxy listens on 8448). +See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a +reverse proxy. + #### Option 3: add a .well-known file to delegate your matrix traffic This will allow you to keep Synapse on a separate domain, without having to @@ -297,17 +300,20 @@ attempt to obtain certificates from Let's Encrypt if you configure it to do so.The only requirement is that there is a valid TLS cert present for federation end points. -### Do you still recommend against using a reverse-proxy on the federation port? +### Do you still recommend against using a reverse proxy on the federation port? We no longer actively recommend against using a reverse proxy. Many admins will -find it easier to direct federation traffic to a reverse-proxy and manage their +find it easier to direct federation traffic to a reverse proxy and manage their own TLS certificates, and this is a supported configuration. +See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a +reverse proxy. + ### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy? Practically speaking, this is no longer necessary. -If you are using a reverse-proxy for all of your TLS traffic, then you can set +If you are using a reverse proxy for all of your TLS traffic, then you can set `no_tls: True`. In that case, the only reason Synapse needs the certificate is to populate a legacy 'tls_fingerprints' field in the federation API. This is ignored by Synapse 0.99.0 and later, and the only time pre-0.99 Synapses will @@ -321,9 +327,9 @@ this, you can give it any TLS certificate at all. This will be fixed soon. ### Do I need the same certificate for the client and federation port? -No. There is nothing stopping you doing so, particularly if you are using a -reverse-proxy. However, Synapse will use the same certificate on any ports -where TLS is configured. +No. There is nothing stopping you from using different certificates, +particularly if you are using a reverse proxy. However, Synapse will use the +same certificate on any ports where TLS is configured. ### How do I tell Synapse to reload my keys/certificates after I replace them? diff --git a/docs/reverse_proxy.rst b/docs/reverse_proxy.rst new file mode 100644 index 0000000000..d8aaac8a08 --- /dev/null +++ b/docs/reverse_proxy.rst @@ -0,0 +1,94 @@ +Using a reverse proxy with Synapse +================================== + +It is recommended to put a reverse proxy such as +`nginx `_, +`Apache `_, +`Caddy `_ or +`HAProxy `_ in front of Synapse. One advantage of +doing so is that it means that you can expose the default https port (443) to +Matrix clients without needing to run Synapse with root privileges. + +**NOTE**: Your reverse proxy must not 'canonicalise' or 'normalise' the +requested URI in any way (for example, by decoding ``%xx`` escapes). Beware +that Apache *will* canonicalise URIs unless you specifify ``nocanon``. + +When setting up a reverse proxy, remember that Matrix clients and other Matrix +servers do not necessarily need to connect to your server via the same server +name or port. Indeed, clients will use port 443 by default, whereas servers +default to port 8448. Where these are different, we refer to the 'client port' +and the 'federation port'. See `Setting up federation +<../README.rst#setting-up-federation>`_ for more details of the algorithm used for +federation connections. + +Let's assume that we expect clients to connect to our server at +``https://matrix.example.com``, and other servers to connect at +``https://example.com:8448``. Here are some example configurations: + +* nginx:: + + server { + listen 443 ssl; + listen [::]:443 ssl; + server_name matrix.example.com; + + location /_matrix { + proxy_pass http://localhost:8008; + proxy_set_header X-Forwarded-For $remote_addr; + } + } + + server { + listen 8448 ssl default_server; + listen [::]:8448 ssl default_server; + server_name example.com; + + location / { + proxy_pass http://localhost:8008; + proxy_set_header X-Forwarded-For $remote_addr; + } + } + +* Caddy:: + + matrix.example.com { + proxy /_matrix http://localhost:8008 { + transparent + } + } + + example.com:8448 { + proxy / http://localhost:8008 { + transparent + } + } + +* Apache (note the ``nocanon`` options here!):: + + + SSLEngine on + ServerName matrix.example.com; + + + ProxyPass http://127.0.0.1:8008/_matrix nocanon + ProxyPassReverse http://127.0.0.1:8008/_matrix + + + + + SSLEngine on + ServerName example.com; + + + ProxyPass http://127.0.0.1:8008/_matrix nocanon + ProxyPassReverse http://127.0.0.1:8008/_matrix + + + +You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true`` +for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are +recorded correctly. + +Having done so, you can then use ``https://matrix.example.com`` (instead of +``https://matrix.example.com:8448``) as the "Custom server" when connecting to +Synapse from a client. diff --git a/docs/workers.rst b/docs/workers.rst index 101e950020..dd3a84ba0d 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -26,9 +26,8 @@ Configuration To make effective use of the workers, you will need to configure an HTTP reverse-proxy such as nginx or haproxy, which will direct incoming requests to the correct worker, or to the main synapse instance. Note that this includes -requests made to the federation port. The caveats regarding running a -reverse-proxy on the federation port still apply (see -https://github.com/matrix-org/synapse/blob/master/README.rst#reverse-proxying-the-federation-port). +requests made to the federation port. See ``_ for +information on setting up a reverse proxy. To enable workers, you need to add two replication listeners to the master synapse, e.g.:: From 24b7f3916d514f570985b6ebc3936938bb5adf45 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 12:50:30 +0000 Subject: [PATCH 77/96] Clean up default listener configuration (#4586) Rearrange the comments to try to clarify them, and expand on what some of it means. Use a sensible default 'bind_addresses' setting. For the insecure port, only bind to localhost, and enable x_forwarded, since apparently it's for use behind a load-balancer. --- changelog.d/4586.misc | 1 + synapse/config/server.py | 129 ++++++++++++++++++++++++--------------- 2 files changed, 82 insertions(+), 48 deletions(-) create mode 100644 changelog.d/4586.misc diff --git a/changelog.d/4586.misc b/changelog.d/4586.misc new file mode 100644 index 0000000000..37af371ccf --- /dev/null +++ b/changelog.d/4586.misc @@ -0,0 +1 @@ +Clean up default listener configuration diff --git a/synapse/config/server.py b/synapse/config/server.py index f0a60cc712..ce0458195c 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -24,6 +24,14 @@ from ._base import Config, ConfigError logger = logging.Logger(__name__) +# by default, we attempt to listen on both '::' *and* '0.0.0.0' because some OSes +# (Windows, macOS, other BSD/Linux where net.ipv6.bindv6only is set) will only listen +# on IPv6 when '::' is set. +# +# We later check for errors when binding to 0.0.0.0 and ignore them if :: is also in +# in the list. +DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0'] + class ServerConfig(Config): @@ -124,10 +132,13 @@ class ServerConfig(Config): bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) + # if bind_address was specified, add it to the list of addresses if bind_address: bind_addresses.append(bind_address) - elif not bind_addresses: - bind_addresses.append('') + + # if we still have an empty list of addresses, use the default list + if not bind_addresses: + bind_addresses.extend(DEFAULT_BIND_ADDRESSES) if not self.web_client_location: _warn_if_webclient_configured(self.listeners) @@ -295,76 +306,98 @@ class ServerConfig(Config): # List of ports that Synapse should listen on, their purpose and their # configuration. + # + # Options for each listener include: + # + # port: the TCP port to bind to + # + # bind_addresses: a list of local addresses to listen on. The default is + # 'all local interfaces'. + # + # type: the type of listener. Normally 'http', but other valid options are: + # 'manhole' (see docs/manhole.md), + # 'metrics' (see docs/metrics-howto.rst), + # 'replication' (see docs/workers.rst). + # + # tls: set to true to enable TLS for this listener. Will use the TLS + # key/cert specified in tls_private_key_path / tls_certificate_path. + # + # x_forwarded: Only valid for an 'http' listener. Set to true to use the + # X-Forwarded-For header as the client IP. Useful when Synapse is + # behind a reverse-proxy. + # + # resources: Only valid for an 'http' listener. A list of resources to host + # on this port. Options for each resource are: + # + # names: a list of names of HTTP resources. See below for a list of + # valid resource names. + # + # compress: set to true to enable HTTP comression for this resource. + # + # additional_resources: Only valid for an 'http' listener. A map of + # additional endpoints which should be loaded via dynamic modules. + # + # Valid resource names are: + # + # client: the client-server API (/_matrix/client). Also implies 'media' and + # 'static'. + # + # consent: user consent forms (/_matrix/consent). See + # docs/consent_tracking.md. + # + # federation: the server-server API (/_matrix/federation). Also implies + # 'media', 'keys', 'openid' + # + # keys: the key discovery API (/_matrix/keys). + # + # media: the media API (/_matrix/media). + # + # metrics: the metrics interface. See docs/metrics-howto.rst. + # + # openid: OpenID authentication. + # + # replication: the HTTP replication API (/_synapse/replication). See + # docs/workers.rst. + # + # static: static resources under synapse/static (/_matrix/static). (Mostly + # useful for 'fallback authentication'.) + # + # webclient: A web client. Requires web_client_location to be set. + # listeners: - # Main HTTPS listener + # Main HTTPS listener. # For when matrix traffic is sent directly to synapse. - - - # The port to listen for HTTPS requests on. - port: %(bind_port)s - - # Local addresses to listen on. - # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6 - # addresses by default. For most other OSes, this will only listen - # on IPv6. - bind_addresses: - - '::' - - '0.0.0.0' - - # This is a 'http' listener, allows us to specify 'resources'. + - port: %(bind_port)s type: http - tls: true - # Use the X-Forwarded-For (XFF) header as the client IP and not the - # actual client IP. - x_forwarded: false - # List of HTTP resources to serve on this listener. resources: - - - # List of resources to host on this listener. - names: - - client # The client-server APIs, both v1 and v2 - # - webclient # A web client. Requires web_client_location to be set. - - # Should synapse compress HTTP responses to clients that support it? - # This should be disabled if running synapse behind a load balancer - # that can do automatic compression. + - names: [client] compress: true - - - names: [federation] # Federation APIs + - names: [federation] compress: false - # # If federation is disabled synapse can still expose the open ID endpoint - # # to allow integrations to authenticate users - # - names: [openid] - # compress: false - - # optional list of additional endpoints which can be loaded via - # dynamic modules + # example addional_resources: + # # additional_resources: # "/_matrix/my/custom/endpoint": # module: my_module.CustomRequestHandler # config: {} - # Unsecure HTTP listener, - # For when matrix traffic passes through loadbalancer that unwraps TLS. + # Unsecure HTTP listener + # For when matrix traffic passes through a reverse-proxy that unwraps TLS. - port: %(unsecure_port)s tls: false - bind_addresses: ['::', '0.0.0.0'] + bind_addresses: ['::1', '127.0.0.1'] type: http - - x_forwarded: false + x_forwarded: true resources: - names: [client] compress: true - names: [federation] compress: false - # # If federation is disabled synapse can still expose the open ID endpoint - # # to allow integrations to authenticate users - # - names: [openid] - # compress: false # Turn on the twisted ssh manhole service on localhost on the given # port. From 5d27730a73d01bd3ff7eb42a21f2f024493c5b89 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 18:03:30 +0000 Subject: [PATCH 78/96] Move ClientTLSOptionsFactory init out of refresh_certificates (#4611) It's nothing to do with refreshing the certificates. No idea why it was here. --- changelog.d/4611.misc | 1 + synapse/app/_base.py | 3 --- synapse/http/matrixfederationclient.py | 4 ++-- synapse/server.py | 6 +++++- tests/http/test_fedclient.py | 4 +--- 5 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 changelog.d/4611.misc diff --git a/changelog.d/4611.misc b/changelog.d/4611.misc new file mode 100644 index 0000000000..d2e0a05daa --- /dev/null +++ b/changelog.d/4611.misc @@ -0,0 +1 @@ +Move ClientTLSOptionsFactory init out of refresh_certificates diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 62c633146f..e1fc1afd5b 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -216,9 +216,6 @@ def refresh_certificate(hs): logging.info("Loading certificate from disk...") hs.config.read_certificate_from_disk() hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - hs.config - ) logging.info("Certificate loaded.") if hs._listening_services: diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5ee4d528d2..3c24bf3805 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -168,7 +168,7 @@ class MatrixFederationHttpClient(object): requests. """ - def __init__(self, hs): + def __init__(self, hs, tls_client_options_factory): self.hs = hs self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname @@ -176,7 +176,7 @@ class MatrixFederationHttpClient(object): self.agent = MatrixFederationAgent( hs.get_reactor(), - hs.tls_client_options_factory, + tls_client_options_factory, ) self.clock = hs.get_clock() self._store = hs.get_datastore() diff --git a/synapse/server.py b/synapse/server.py index a2cf8a91cd..8615b67ad4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -31,6 +31,7 @@ from synapse.api.filtering import Filtering from synapse.api.ratelimiting import Ratelimiter from synapse.appservice.api import ApplicationServiceApi from synapse.appservice.scheduler import ApplicationServiceScheduler +from synapse.crypto import context_factory from synapse.crypto.keyring import Keyring from synapse.events.builder import EventBuilderFactory from synapse.events.spamcheck import SpamChecker @@ -367,7 +368,10 @@ class HomeServer(object): return PusherPool(self) def build_http_client(self): - return MatrixFederationHttpClient(self) + tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + self.config + ) + return MatrixFederationHttpClient(self, tls_client_options_factory) def build_db_pool(self): name = self.db_config["name"] diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py index 018c77ebcd..b03b37affe 100644 --- a/tests/http/test_fedclient.py +++ b/tests/http/test_fedclient.py @@ -43,13 +43,11 @@ def check_logcontext(context): class FederationClientTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): - hs = self.setup_test_homeserver(reactor=reactor, clock=clock) - hs.tls_client_options_factory = None return hs def prepare(self, reactor, clock, homeserver): - self.cl = MatrixFederationHttpClient(self.hs) + self.cl = MatrixFederationHttpClient(self.hs, None) self.reactor.lookups["testserv"] = "1.2.3.4" def test_client_get(self): From 086f6f27d409520e71556cad4707cb2f70476e20 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:00:41 +0000 Subject: [PATCH 79/96] Logging improvements around TLS certs Log which file we're reading keys and certs from, and refactor the code a bit in preparation for other work --- changelog.d/4615.misc | 1 + synapse/app/_base.py | 6 ++--- synapse/config/tls.py | 54 ++++++++++++++++++++++++++++--------------- 3 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 changelog.d/4615.misc diff --git a/changelog.d/4615.misc b/changelog.d/4615.misc new file mode 100644 index 0000000000..c7266fcfc7 --- /dev/null +++ b/changelog.d/4615.misc @@ -0,0 +1 @@ +Logging improvements around TLS certs diff --git a/synapse/app/_base.py b/synapse/app/_base.py index e1fc1afd5b..6d72de1daa 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -213,13 +213,11 @@ def refresh_certificate(hs): Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. """ - logging.info("Loading certificate from disk...") hs.config.read_certificate_from_disk() hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) - logging.info("Certificate loaded.") if hs._listening_services: - logging.info("Updating context factories...") + logger.info("Updating context factories...") for i in hs._listening_services: # When you listenSSL, it doesn't make an SSL port but a TCP one with # a TLS wrapping factory around the factory you actually want to get @@ -234,7 +232,7 @@ def refresh_certificate(hs): False, i.factory.wrappedFactory ) - logging.info("Context factories updated.") + logger.info("Context factories updated.") def start(hs, listeners=None): diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 9fcc79816d..76d2add4fe 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -25,7 +25,7 @@ from OpenSSL import crypto from synapse.config._base import Config -logger = logging.getLogger() +logger = logging.getLogger(__name__) class TlsConfig(Config): @@ -110,20 +110,10 @@ class TlsConfig(Config): """ Read the certificates from disk. """ - self.tls_certificate = self.read_tls_certificate(self.tls_certificate_file) - - # Check if it is self-signed, and issue a warning if so. - if self.tls_certificate.get_issuer() == self.tls_certificate.get_subject(): - warnings.warn( - ( - "Self-signed TLS certificates will not be accepted by Synapse 1.0. " - "Please either provide a valid certificate, or use Synapse's ACME " - "support to provision one." - ) - ) + self.tls_certificate = self.read_tls_certificate() if not self.no_tls: - self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file) + self.tls_private_key = self.read_tls_private_key() self.tls_fingerprints = list(self._original_tls_fingerprints) @@ -250,10 +240,38 @@ class TlsConfig(Config): % locals() ) - def read_tls_certificate(self, cert_path): - cert_pem = self.read_file(cert_path, "tls_certificate") - return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + def read_tls_certificate(self): + """Reads the TLS certificate from the configured file, and returns it - def read_tls_private_key(self, private_key_path): - private_key_pem = self.read_file(private_key_path, "tls_private_key") + Also checks if it is self-signed, and warns if so + + Returns: + OpenSSL.crypto.X509: the certificate + """ + cert_path = self.tls_certificate_file + logger.info("Loading TLS certificate from %s", cert_path) + cert_pem = self.read_file(cert_path, "tls_certificate_path") + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + # Check if it is self-signed, and issue a warning if so. + if cert.get_issuer() == cert.get_subject(): + warnings.warn( + ( + "Self-signed TLS certificates will not be accepted by Synapse 1.0. " + "Please either provide a valid certificate, or use Synapse's ACME " + "support to provision one." + ) + ) + + return cert + + def read_tls_private_key(self): + """Reads the TLS private key from the configured file, and returns it + + Returns: + OpenSSL.crypto.PKey: the private key + """ + private_key_path = self.tls_private_key_file + logger.info("Loading TLS key from %s", private_key_path) + private_key_pem = self.read_file(private_key_path, "tls_private_key_path") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) From 2129dd1a023d1e221dab8753be3fbd7024963634 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:13:53 +0000 Subject: [PATCH 80/96] Fail cleanly if listener config lacks a 'port' ... otherwise we would fail with a mysterious KeyError or something later. --- changelog.d/4616.misc | 1 + synapse/config/server.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/4616.misc diff --git a/changelog.d/4616.misc b/changelog.d/4616.misc new file mode 100644 index 0000000000..ee79e208e3 --- /dev/null +++ b/changelog.d/4616.misc @@ -0,0 +1 @@ +Fail cleanly if listener config lacks a 'port' diff --git a/synapse/config/server.py b/synapse/config/server.py index ce0458195c..eed9d7c81e 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -129,6 +129,11 @@ class ServerConfig(Config): self.listeners = config.get("listeners", []) for listener in self.listeners: + if not isinstance(listener.get("port", None), int): + raise ConfigError( + "Listener configuration is lacking a valid 'port' option" + ) + bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) From 9645728619828fda050fa08aaa25628f5db5d775 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:30:59 +0000 Subject: [PATCH 81/96] Don't create server contexts when TLS is disabled we aren't going to use them anyway. --- changelog.d/4617.misc | 1 + synapse/app/_base.py | 5 +++++ synapse/crypto/context_factory.py | 4 +--- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4617.misc diff --git a/changelog.d/4617.misc b/changelog.d/4617.misc new file mode 100644 index 0000000000..6d751865c9 --- /dev/null +++ b/changelog.d/4617.misc @@ -0,0 +1 @@ +Don't create server contexts when TLS is disabled diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 6d72de1daa..6b3cb61ae9 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -214,6 +214,11 @@ def refresh_certificate(hs): disk and updating the TLS context factories to use them. """ hs.config.read_certificate_from_disk() + + if hs.config.no_tls: + # nothing else to do here + return + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) if hs._listening_services: diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 286ad80100..85f2848fb1 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -43,9 +43,7 @@ class ServerContextFactory(ContextFactory): logger.exception("Failed to enable elliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate_chain_file(config.tls_certificate_file) - - if not config.no_tls: - context.use_privatekey(config.tls_private_key) + context.use_privatekey(config.tls_private_key) # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ context.set_cipher_list( From 4fddf8fc77496d9bb3b5fa8835f0e5ba9a5a9926 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 17:57:58 +0000 Subject: [PATCH 82/96] Infer no_tls from presence of TLS listeners Rather than have to specify `no_tls` explicitly, infer whether we need to load the TLS keys etc from whether we have any TLS-enabled listeners. --- changelog.d/4613.feature | 1 + changelog.d/4615.feature | 1 + changelog.d/4615.misc | 1 - changelog.d/4617.feature | 1 + changelog.d/4617.misc | 1 - synapse/app/_base.py | 2 +- synapse/app/homeserver.py | 5 ----- synapse/config/homeserver.py | 2 +- synapse/config/server.py | 23 ++++++++++++++++++++--- synapse/config/tls.py | 10 ++-------- 10 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 changelog.d/4613.feature create mode 100644 changelog.d/4615.feature delete mode 100644 changelog.d/4615.misc create mode 100644 changelog.d/4617.feature delete mode 100644 changelog.d/4617.misc diff --git a/changelog.d/4613.feature b/changelog.d/4613.feature new file mode 100644 index 0000000000..098f906af2 --- /dev/null +++ b/changelog.d/4613.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4615.feature b/changelog.d/4615.feature new file mode 100644 index 0000000000..098f906af2 --- /dev/null +++ b/changelog.d/4615.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4615.misc b/changelog.d/4615.misc deleted file mode 100644 index c7266fcfc7..0000000000 --- a/changelog.d/4615.misc +++ /dev/null @@ -1 +0,0 @@ -Logging improvements around TLS certs diff --git a/changelog.d/4617.feature b/changelog.d/4617.feature new file mode 100644 index 0000000000..098f906af2 --- /dev/null +++ b/changelog.d/4617.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4617.misc b/changelog.d/4617.misc deleted file mode 100644 index 6d751865c9..0000000000 --- a/changelog.d/4617.misc +++ /dev/null @@ -1 +0,0 @@ -Don't create server contexts when TLS is disabled diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 6b3cb61ae9..50fd17c0be 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -215,7 +215,7 @@ def refresh_certificate(hs): """ hs.config.read_certificate_from_disk() - if hs.config.no_tls: + if not hs.config.has_tls_listener(): # nothing else to do here return diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b4476bf16e..dbd98d394f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -90,11 +90,6 @@ class SynapseHomeServer(HomeServer): tls = listener_config.get("tls", False) site_tag = listener_config.get("tag", port) - if tls and config.no_tls: - raise ConfigError( - "Listener on port %i has TLS enabled, but no_tls is set" % (port,), - ) - resources = {} for res in listener_config["resources"]: for name in res["names"]: diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 5aad062c36..727fdc54d8 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -42,7 +42,7 @@ from .voip import VoipConfig from .workers import WorkerConfig -class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, +class HomeServerConfig(ServerConfig, TlsConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig, AppServiceConfig, KeyConfig, SAML2Config, CasConfig, diff --git a/synapse/config/server.py b/synapse/config/server.py index eed9d7c81e..767897c419 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -126,14 +126,22 @@ class ServerConfig(Config): self.public_baseurl += '/' self.start_pushers = config.get("start_pushers", True) - self.listeners = config.get("listeners", []) - - for listener in self.listeners: + self.listeners = [] + for listener in config.get("listeners", []): if not isinstance(listener.get("port", None), int): raise ConfigError( "Listener configuration is lacking a valid 'port' option" ) + if listener.setdefault("tls", False): + # no_tls is not really supported any more, but let's grandfather it in + # here. + if config.get("no_tls", False): + logger.info( + "Ignoring TLS-enabled listener on port %i due to no_tls" + ) + continue + bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) @@ -145,6 +153,8 @@ class ServerConfig(Config): if not bind_addresses: bind_addresses.extend(DEFAULT_BIND_ADDRESSES) + self.listeners.append(listener) + if not self.web_client_location: _warn_if_webclient_configured(self.listeners) @@ -152,6 +162,9 @@ class ServerConfig(Config): bind_port = config.get("bind_port") if bind_port: + if config.get("no_tls", False): + raise ConfigError("no_tls is incompatible with bind_port") + self.listeners = [] bind_host = config.get("bind_host", "") gzip_responses = config.get("gzip_responses", True) @@ -198,6 +211,7 @@ class ServerConfig(Config): "port": manhole, "bind_addresses": ["127.0.0.1"], "type": "manhole", + "tls": False, }) metrics_port = config.get("metrics_port") @@ -223,6 +237,9 @@ class ServerConfig(Config): _check_resource_config(self.listeners) + def has_tls_listener(self): + return any(l["tls"] for l in self.listeners) + def default_config(self, server_name, data_dir_path, **kwargs): _, bind_port = parse_and_validate_server_name(server_name) if bind_port is not None: diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 76d2add4fe..e37a41eff4 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -51,7 +51,6 @@ class TlsConfig(Config): self._original_tls_fingerprints = [] self.tls_fingerprints = list(self._original_tls_fingerprints) - self.no_tls = config.get("no_tls", False) # This config option applies to non-federation HTTP clients # (e.g. for talking to recaptcha, identity servers, and such) @@ -141,6 +140,8 @@ class TlsConfig(Config): return ( """\ + ## TLS ## + # PEM-encoded X509 certificate for TLS. # This certificate, as of Synapse 1.0, will need to be a valid and verifiable # certificate, signed by a recognised Certificate Authority. @@ -201,13 +202,6 @@ class TlsConfig(Config): # # reprovision_threshold: 30 - # If your server runs behind a reverse-proxy which terminates TLS connections - # (for both client and federation connections), it may be useful to disable - # All TLS support for incoming connections. Setting no_tls to True will - # do so (and avoid the need to give synapse a TLS private key). - # - # no_tls: True - # List of allowed TLS fingerprints for this server to publish along # with the signing keys for this server. Other matrix servers that # make HTTPS requests to this server will check that the TLS From 0ca290865350212e1834730c918973162a3067f4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 22:01:27 +0000 Subject: [PATCH 83/96] fix tests --- synapse/config/tls.py | 2 +- tests/config/test_tls.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e37a41eff4..86e6eb80db 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -111,7 +111,7 @@ class TlsConfig(Config): """ self.tls_certificate = self.read_tls_certificate() - if not self.no_tls: + if self.has_tls_listener(): self.tls_private_key = self.read_tls_private_key() self.tls_fingerprints = list(self._original_tls_fingerprints) diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py index 4ccaf35603..d8fd18a9cb 100644 --- a/tests/config/test_tls.py +++ b/tests/config/test_tls.py @@ -20,6 +20,11 @@ from synapse.config.tls import TlsConfig from tests.unittest import TestCase +class TestConfig(TlsConfig): + def has_tls_listener(self): + return False + + class TLSConfigTests(TestCase): def test_warn_self_signed(self): @@ -55,11 +60,10 @@ s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg= config = { "tls_certificate_path": os.path.join(config_dir, "cert.pem"), - "no_tls": True, "tls_fingerprints": [] } - t = TlsConfig() + t = TestConfig() t.read_config(config) t.read_certificate_from_disk() From 91f8cd3307fee502c6bfb064f825952b05c4617b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 22:15:08 +0000 Subject: [PATCH 84/96] Remove redundant entries from docker config * no_tls is now redundant (#4613) * we don't need a dummy cert any more (#4618) --- changelog.d/4619.misc | 1 + docker/conf/dummy.tls.crt | 17 ----------------- docker/conf/homeserver.yaml | 8 +------- 3 files changed, 2 insertions(+), 24 deletions(-) create mode 100644 changelog.d/4619.misc delete mode 100644 docker/conf/dummy.tls.crt diff --git a/changelog.d/4619.misc b/changelog.d/4619.misc new file mode 100644 index 0000000000..886fedf198 --- /dev/null +++ b/changelog.d/4619.misc @@ -0,0 +1 @@ +Remove redundant entries from docker config diff --git a/docker/conf/dummy.tls.crt b/docker/conf/dummy.tls.crt deleted file mode 100644 index 8e3b1a9aaa..0000000000 --- a/docker/conf/dummy.tls.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICnTCCAYUCAgPoMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMMCWxvY2FsaG9z -dDAeFw0xOTAxMTUwMDQxNTBaFw0yOTAxMTIwMDQxNTBaMBQxEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMKqm81/8j5d -R1s7VZ8ueg12gJrPVCCAOkp0UnuC/ZlXhN0HTvnhQ+B0IlSgB4CcQZyf4jnA6o4M -rwSc7VX0MPE9x/idoA0g/0WoC6tsxugOrvbzCw8Tv+fnXglm6uVc7aFPfx69wU3q -lUHGD/8jtEoHxmCG177Pt2lHAfiVLBAyMQGtETzxt/yAfkloaybe316qoljgK5WK -cokdAt9G84EEqxNeEnx5FG3Vc100bAqJS4GvQlFgtF9KFEqZKEyB1yKBpPMDfPIS -V9hIV0gswSmYI8dpyBlGf5lPElY68ZGABmOQgr0RI5qHK/h28OpFPE0q3v4AMHgZ -I36wii4NrAUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfD8kcpZ+dn08xh1qtKtp -X+/YNZaOBIeVdlCzfoZKNblSFAFD/jCfObNJYvZMUQ8NX2UtEJp1lTA6m7ltSsdY -gpC2k1VD8iN+ooXklJmL0kxc7UUqho8I0l9vn35h+lhLF0ihT6XfZVi/lDHWl+4G -rG+v9oxvCSCWrNWLearSlFPtQQ8xPtOE0nLwfXtOI/H/2kOuC38ihaIWM4jjbWXK -E/ksgUfuDv0mFiwf1YdBF5/M3/qOowqzU8HgMJ3WoT/9Po5Ya1pWc+3BcxxytUDf -XdMu0tWHKX84tZxLcR1nZHzluyvFFM8xNtLi9xV0Z7WbfT76V0C/ulEOybGInYsv -nQ== ------END CERTIFICATE----- diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index f07d5c1001..babd5bef9e 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -2,13 +2,7 @@ ## TLS ## -{% if SYNAPSE_NO_TLS %} -no_tls: True - -# workaround for https://github.com/matrix-org/synapse/issues/4554 -tls_certificate_path: "/conf/dummy.tls.crt" - -{% else %} +{% if not SYNAPSE_NO_TLS %} tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" From dfc846a316a92a750133ed04f3625da70f7944d7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 12 Feb 2019 10:37:00 +0000 Subject: [PATCH 85/96] fix self-signed cert notice from generate-config fixes #4620 --- changelog.d/4625.bugfix | 1 + synapse/config/_base.py | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 changelog.d/4625.bugfix diff --git a/changelog.d/4625.bugfix b/changelog.d/4625.bugfix new file mode 100644 index 0000000000..3dc0ecf24c --- /dev/null +++ b/changelog.d/4625.bugfix @@ -0,0 +1 @@ +fix self-signed cert notice from generate-config. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 5858fb92b4..5aec43b702 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -257,7 +257,7 @@ class Config(object): "--keys-directory", metavar="DIRECTORY", help="Used with 'generate-*' options to specify where files such as" - " certs and signing keys should be stored in, unless explicitly" + " signing keys should be stored, unless explicitly" " specified in the config.", ) config_parser.add_argument( @@ -313,16 +313,11 @@ class Config(object): print( ( "A config file has been generated in %r for server name" - " %r with corresponding SSL keys and self-signed" - " certificates. Please review this file and customise it" + " %r. Please review this file and customise it" " to your needs." ) % (config_path, server_name) ) - print( - "If this server name is incorrect, you will need to" - " regenerate the SSL certificates" - ) return else: print( From 32b781bfe20034052d71558c0ed9b0df511a1f77 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:51:31 +0000 Subject: [PATCH 86/96] Fix error when loading cert if tls is disabled (#4618) If TLS is disabled, it should not be an error if no cert is given. Fixes #4554. --- changelog.d/4618.bugfix | 1 + synapse/app/_base.py | 5 ++-- synapse/config/tls.py | 57 +++++++++++++++++++++++++++++----------- tests/config/test_tls.py | 2 +- 4 files changed, 47 insertions(+), 18 deletions(-) create mode 100644 changelog.d/4618.bugfix diff --git a/changelog.d/4618.bugfix b/changelog.d/4618.bugfix new file mode 100644 index 0000000000..22115a020e --- /dev/null +++ b/changelog.d/4618.bugfix @@ -0,0 +1 @@ +Fix failure to start when not TLS certificate was given even if TLS was disabled. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 50fd17c0be..5b0ca312e2 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -213,12 +213,13 @@ def refresh_certificate(hs): Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. """ - hs.config.read_certificate_from_disk() if not hs.config.has_tls_listener(): - # nothing else to do here + # attempt to reload the certs for the good of the tls_fingerprints + hs.config.read_certificate_from_disk(require_cert_and_key=False) return + hs.config.read_certificate_from_disk(require_cert_and_key=True) hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) if hs._listening_services: diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 86e6eb80db..57f117a14d 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -23,7 +23,7 @@ from unpaddedbase64 import encode_base64 from OpenSSL import crypto -from synapse.config._base import Config +from synapse.config._base import Config, ConfigError logger = logging.getLogger(__name__) @@ -45,6 +45,19 @@ class TlsConfig(Config): self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) + + if self.has_tls_listener(): + if not self.tls_certificate_file: + raise ConfigError( + "tls_certificate_path must be specified if TLS-enabled listeners are " + "configured." + ) + if not self.tls_private_key_file: + raise ConfigError( + "tls_certificate_path must be specified if TLS-enabled listeners are " + "configured." + ) + self._original_tls_fingerprints = config.get("tls_fingerprints", []) if self._original_tls_fingerprints is None: @@ -105,26 +118,40 @@ class TlsConfig(Config): days_remaining = (expires_on - now).days return days_remaining - def read_certificate_from_disk(self): + def read_certificate_from_disk(self, require_cert_and_key): """ - Read the certificates from disk. - """ - self.tls_certificate = self.read_tls_certificate() + Read the certificates and private key from disk. - if self.has_tls_listener(): + Args: + require_cert_and_key (bool): set to True to throw an error if the certificate + and key file are not given + """ + if require_cert_and_key: self.tls_private_key = self.read_tls_private_key() + self.tls_certificate = self.read_tls_certificate() + elif self.tls_certificate_file: + # we only need the certificate for the tls_fingerprints. Reload it if we + # can, but it's not a fatal error if we can't. + try: + self.tls_certificate = self.read_tls_certificate() + except Exception as e: + logger.info( + "Unable to read TLS certificate (%s). Ignoring as no " + "tls listeners enabled.", e, + ) self.tls_fingerprints = list(self._original_tls_fingerprints) - # Check that our own certificate is included in the list of fingerprints - # and include it if it is not. - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, self.tls_certificate - ) - sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) - sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) - if sha256_fingerprint not in sha256_fingerprints: - self.tls_fingerprints.append({u"sha256": sha256_fingerprint}) + if self.tls_certificate: + # Check that our own certificate is included in the list of fingerprints + # and include it if it is not. + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, self.tls_certificate + ) + sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) + sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) + if sha256_fingerprint not in sha256_fingerprints: + self.tls_fingerprints.append({u"sha256": sha256_fingerprint}) def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py index d8fd18a9cb..c260d3359f 100644 --- a/tests/config/test_tls.py +++ b/tests/config/test_tls.py @@ -65,7 +65,7 @@ s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg= t = TestConfig() t.read_config(config) - t.read_certificate_from_disk() + t.read_certificate_from_disk(require_cert_and_key=False) warnings = self.flushWarnings() self.assertEqual(len(warnings), 1) From a4ce91396bda0c6a6e3a2392355f8297cc97071b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:52:08 +0000 Subject: [PATCH 87/96] Disable TLS by default (#4614) --- changelog.d/4614.feature | 1 + synapse/config/server.py | 50 ++++++++++++++++++++-------------------- synapse/config/tls.py | 6 ++--- 3 files changed, 29 insertions(+), 28 deletions(-) create mode 100644 changelog.d/4614.feature diff --git a/changelog.d/4614.feature b/changelog.d/4614.feature new file mode 100644 index 0000000000..18e16dbc7b --- /dev/null +++ b/changelog.d/4614.feature @@ -0,0 +1 @@ +The default configuration no longer requires TLS certificates. diff --git a/synapse/config/server.py b/synapse/config/server.py index 767897c419..c5c3aac8ed 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -387,28 +387,24 @@ class ServerConfig(Config): # webclient: A web client. Requires web_client_location to be set. # listeners: - # Main HTTPS listener. - # For when matrix traffic is sent directly to synapse. - - port: %(bind_port)s - type: http - tls: true + # TLS-enabled listener: for when matrix traffic is sent directly to synapse. + # + # Disabled by default. To enable it, uncomment the following. (Note that you + # will also need to give Synapse a TLS key and certificate: see the TLS section + # below.) + # + # - port: %(bind_port)s + # type: http + # tls: true + # resources: + # - names: [client, federation] - # List of HTTP resources to serve on this listener. - resources: - - names: [client] - compress: true - - names: [federation] - compress: false - - # example addional_resources: - # - # additional_resources: - # "/_matrix/my/custom/endpoint": - # module: my_module.CustomRequestHandler - # config: {} - - # Unsecure HTTP listener - # For when matrix traffic passes through a reverse-proxy that unwraps TLS. + # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy + # that unwraps TLS. + # + # If you plan to use a reverse proxy, please see + # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst. + # - port: %(unsecure_port)s tls: false bind_addresses: ['::1', '127.0.0.1'] @@ -416,18 +412,22 @@ class ServerConfig(Config): x_forwarded: true resources: - - names: [client] - compress: true - - names: [federation] + - names: [client, federation] compress: false + # example additonal_resources: + # + # additional_resources: + # "/_matrix/my/custom/endpoint": + # module: my_module.CustomRequestHandler + # config: {} + # Turn on the twisted ssh manhole service on localhost on the given # port. # - port: 9000 # bind_addresses: ['::1', '127.0.0.1'] # type: manhole - # Homeserver blocking # # How to reach the server admin, used in ResourceLimitError diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 57f117a14d..5fb3486db1 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -176,10 +176,10 @@ class TlsConfig(Config): # See 'ACME support' below to enable auto-provisioning this certificate via # Let's Encrypt. # - tls_certificate_path: "%(tls_certificate_path)s" + # tls_certificate_path: "%(tls_certificate_path)s" # PEM-encoded private key for TLS - tls_private_key_path: "%(tls_private_key_path)s" + # tls_private_key_path: "%(tls_private_key_path)s" # ACME support: This will configure Synapse to request a valid TLS certificate # for your configured `server_name` via Let's Encrypt. @@ -204,7 +204,7 @@ class TlsConfig(Config): # acme: # ACME support is disabled by default. Uncomment the following line - # to enable it. + # (and tls_certificate_path and tls_private_key_path above) to enable it. # # enabled: true From 2418b91bb7d64bfe572647565a6f1c80e82e1f5a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:53:28 +0000 Subject: [PATCH 88/96] README updates (#4621) Lots of updates to the README/INSTALL.md. Fixes #4601. --- INSTALL.md | 40 ++++++++++++------ README.rst | 98 +++++++++++++++++++++---------------------- changelog.d/4621.misc | 1 + 3 files changed, 78 insertions(+), 61 deletions(-) create mode 100644 changelog.d/4621.misc diff --git a/INSTALL.md b/INSTALL.md index e496a13b21..fb6a5e4e99 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -350,18 +350,34 @@ Once you have installed synapse as above, you will need to configure it. ## TLS certificates -The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is -configured without TLS; it should be behind a reverse proxy for TLS/SSL -termination on port 443 which in turn should be used for clients. Port 8448 -is configured to use TLS for Federation with a self-signed or verified -certificate, but please be aware that a valid certificate will be required in -Synapse v1.0. Instructions for having Synapse automatically provision and renew federation certificates through ACME can be found at [ACME.md](docs/ACME.md). +The default configuration exposes a single HTTP port: http://localhost:8008. It +is suitable for local testing, but for any practical use, you will either need +to enable a reverse proxy, or configure Synapse to expose an HTTPS port. -If you would like to use your own certificates, you can do so by changing -`tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`; -alternatively, you can use a reverse proxy. See -[docs/reverse_proxy.rst](docs/reverse_proxy.rst) for information on configuring -a reverse proxy. +For information on using a reverse proxy, see +[docs/reverse_proxy.rst](docs/reverse_proxy.rst). + +To configure Synapse to expose an HTTPS port, you will need to edit +`homeserver.yaml`. + +First, under the `listeners` section, uncomment the configuration for the +TLS-enabled listener. (Remove the hash sign (`#`) and space at the start of +each line). The relevant lines are like this: + +``` + - port: 8448 + type: http + tls: true + resources: + - names: [client, federation] +``` + +You will also need to uncomment the `tls_certificate_path` and +`tls_private_key_path` lines under the `TLS` section. You can either point +these settings at an existing certificate and key, or you can enable Synapse's +built-in ACME (Let's Encrypt) support. Instructions for having Synapse +automatically provision and renew federation certificates through ACME can be +found at [ACME.md](docs/ACME.md). ## Registering a user @@ -375,7 +391,7 @@ users. This can be done as follows: ``` $ source ~/synapse/env/bin/activate $ synctl start # if not already running -$ register_new_matrix_user -c homeserver.yaml https://localhost:8448 +$ register_new_matrix_user -c homeserver.yaml http://localhost:8008 New user localpart: erikj Password: Confirm password: diff --git a/README.rst b/README.rst index bc7cb5f784..9a7c04b55e 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,6 @@ via IRC bridge at irc://irc.freenode.net/matrix. Synapse is currently in rapid development, but as of version 0.5 we believe it is sufficiently stable to be run as an internet-facing service for real usage! - About Matrix ============ @@ -88,18 +87,20 @@ Connecting to Synapse from a client =================================== The easiest way to try out your new Synapse installation is by connecting to it -from a web client. The easiest option is probably the one at -https://riot.im/app. You will need to specify a "Custom server" when you log on -or register: set this to ``https://domain.tld`` if you setup a reverse proxy -following the recommended setup, or ``https://localhost:8448`` - remember to specify the -port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity -server as the default - see `Identity servers`_.) +from a web client. -If using port 8448 you will run into errors if you are using a self-signed -certificate. To overcome this, simply go to ``https://localhost:8448`` -directly with your browser and accept the presented certificate. You can then -go back in your web client and proceed further. Valid federation certificates -should not have this problem. +Unless you are running a test instance of Synapse on your local machine, in +general, you will need to enable TLS support before you can successfully +connect from a client: see ``_. + +An easy way to get started is to login or register via Riot at +https://riot.im/app/#/login or https://riot.im/app/#/register respectively. +You will need to change the server you are logging into from ``matrix.org`` +and instead specify a Homeserver URL of ``https://:8448`` +(or just ``https://`` if you are using a reverse proxy). +(Leave the identity server as the default - see `Identity servers`_.) +If you prefer to use another client, refer to our +`client breakdown `_. If all goes well you should at least be able to log in, create a room, and start sending messages. @@ -174,9 +175,30 @@ Separately, Synapse may leak file handles if inbound HTTP requests get stuck during processing - e.g. blocked behind a lock or talking to a remote server etc. This is best diagnosed by matching up the 'Received request' and 'Processed request' log lines and looking for any 'Processed request' lines which take more than -a few seconds to execute. Please let us know at #matrix-dev:matrix.org if +a few seconds to execute. Please let us know at #synapse:matrix.org if you see this failure mode so we can help debug it, however. +Help!! Synapse eats all my RAM! +------------------------------- + +Synapse's architecture is quite RAM hungry currently - we deliberately +cache a lot of recent room data and metadata in RAM in order to speed up +common requests. We'll improve this in future, but for now the easiest +way to either reduce the RAM usage (at the risk of slowing things down) +is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment +variable. The default is 0.5, which can be decreased to reduce RAM usage +in memory constrained enviroments, or increased if performance starts to +degrade. + +Using `libjemalloc `_ can also yield a significant +improvement in overall amount, and especially in terms of giving back RAM +to the OS. To use it, the library must simply be put in the LD_PRELOAD +environment variable when launching Synapse. On Debian, this can be done +by installing the ``libjemalloc1`` package and adding this line to +``/etc/default/matrix-synapse``:: + + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 + Upgrading an existing Synapse ============================= @@ -196,12 +218,12 @@ Federation is the process by which users on different servers can participate in the same room. For this to work, those other servers must be able to contact yours to send messages. -The ``server_name`` in your -``homeserver.yaml`` file determines the way that other servers will reach -yours. By default, they will treat it as a hostname and try to connect to -port 8448. This is easy to set up and will work with the default configuration, -provided you set the ``server_name`` to match your machine's public DNS -hostname. +The ``server_name`` in your ``homeserver.yaml`` file determines the way that +other servers will reach yours. By default, they will treat it as a hostname +and try to connect to port 8448. This is easy to set up and will work with the +default configuration, provided you set the ``server_name`` to match your +machine's public DNS hostname, and give Synapse a TLS certificate which is +valid for your ``server_name``. For a more flexible configuration, you can set up a DNS SRV record. This allows you to run your server on a machine that might not have the same name as your @@ -243,11 +265,8 @@ largest boxes pause for thought.) Troubleshooting --------------- -You can use the federation tester to check if your homeserver is all set: -``https://matrix.org/federationtester/api/report?server_name=`` -If any of the attributes under "checks" is false, federation won't work. -There is also a nicer interface available from a community member at -``_. +You can use the `federation tester `_ to +check if your homeserver is all set. The typical failure mode with federation is that when you try to join a room, it is rejected with "401: Unauthorized". Generally this means that other @@ -263,7 +282,10 @@ So, things to check are: (it should be ``_matrix._tcp.``), and that the port and hostname it specifies are reachable from outside your network. -.. TODO: add a note about forgetting ``nocanon`` on a reverse-proxy config +Another common problem is that people on other servers can't join rooms that +you invite them to. This can be caused by an incorrectly-configured reverse +proxy: see ``_ for instructions on how to correctly +configure a reverse proxy. Running a Demo Federation of Synapses ------------------------------------- @@ -363,7 +385,7 @@ Synapse Development Before setting up a development environment for synapse, make sure you have the system dependencies (such as the python header files) installed - see -`Installing from source`_. +`Installing from source `_. To check out a synapse for development, clone the git repo into a working directory of your choice:: @@ -374,7 +396,7 @@ directory of your choice:: Synapse has a number of external dependencies, that are easiest to install using pip and a virtualenv:: - virtualenv -p python2.7 env + virtualenv -p python3 env source env/bin/activate python -m pip install -e .[all] @@ -416,25 +438,3 @@ sphinxcontrib-napoleon:: Building internal API documentation:: python setup.py build_sphinx - - -Help!! Synapse eats all my RAM! -=============================== - -Synapse's architecture is quite RAM hungry currently - we deliberately -cache a lot of recent room data and metadata in RAM in order to speed up -common requests. We'll improve this in future, but for now the easiest -way to either reduce the RAM usage (at the risk of slowing things down) -is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment -variable. The default is 0.5, which can be decreased to reduce RAM usage -in memory constrained enviroments, or increased if performance starts to -degrade. - -Using `libjemalloc `_ can also yield a significant -improvement in overall amount, and especially in terms of giving back RAM -to the OS. To use it, the library must simply be put in the LD_PRELOAD -environment variable when launching Synapse. On Debian, this can be done -by installing the ``libjemalloc1`` package and adding this line to -``/etc/default/matrix-synapse``:: - - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 diff --git a/changelog.d/4621.misc b/changelog.d/4621.misc new file mode 100644 index 0000000000..60e45cb70c --- /dev/null +++ b/changelog.d/4621.misc @@ -0,0 +1 @@ +README updates From 362d80b7706c7bcd57ea5ee3e8e38b31c7d1ad8b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:28:08 +0000 Subject: [PATCH 89/96] Reduce user_ips bloat during dedupe background update The background update to remove duplicate rows naively deleted and reinserted the duplicates. For large tables with a large number of duplicates this causes a lot of bloat (with postgres), as the inserted rows are appended to the table, since deleted rows will not be overwritten until a VACUUM has happened. This should hopefully also help ensure that the query in the last batch uses the correct index, as inserting a large number of new rows without analyzing will upset the query planner. --- synapse/storage/client_ips.py | 63 +++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 091d7116c5..a20cc8231f 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -167,12 +167,16 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): clause = "? <= last_seen AND last_seen < ?" args = (begin_last_seen, end_last_seen) + # (Note: The DISTINCT in the inner query is important to ensure that + # the COUNT(*) is accurate, otherwise double counting may happen due + # to the join effectively being a cross product) txn.execute( """ SELECT user_id, access_token, ip, - MAX(device_id), MAX(user_agent), MAX(last_seen) + MAX(device_id), MAX(user_agent), MAX(last_seen), + COUNT(*) FROM ( - SELECT user_id, access_token, ip + SELECT DISTINCT user_id, access_token, ip FROM user_ips WHERE {} ) c @@ -186,7 +190,60 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): # We've got some duplicates for i in res: - user_id, access_token, ip, device_id, user_agent, last_seen = i + user_id, access_token, ip, device_id, user_agent, last_seen, count = i + + # We want to delete the duplicates so we end up with only a + # single row. + # + # The naive way of doing this would be just to delete all rows + # and reinsert a constructed row. However, if there are a lot of + # duplicate rows this can cause the table to grow a lot, which + # can be problematic in two ways: + # 1. If user_ips is already large then this can cause the + # table to rapidly grow, potentially filling the disk. + # 2. Reinserting a lot of rows can confuse the table + # statistics for postgres, causing it to not use the + # correct indices for the query above, resulting in a full + # table scan. This is incredibly slow for large tables and + # can kill database performance. (This seems to mainly + # happen for the last query where the clause is simply `? < + # last_seen`) + # + # So instead we want to delete all but *one* of the duplicate + # rows. That is hard to do reliably, so we cheat and do a two + # step process: + # 1. Delete all rows with a last_seen strictly less than the + # max last_seen. This hopefully results in deleting all but + # one row the majority of the time, but there may be + # duplicate last_seen + # 2. If multiple rows remain, we fall back to the naive method + # and simply delete all rows and reinsert. + # + # Note that this relies on no new duplicate rows being inserted, + # but if that is happening then this entire process is futile + # anyway. + + # Do step 1: + + txn.execute( + """ + DELETE FROM user_ips + WHERE user_id = ? AND access_token = ? AND ip = ? AND last_seen < ? + """, + (user_id, access_token, ip, last_seen) + ) + if txn.rowcount == count - 1: + # We deleted all but one of the duplicate rows, i.e. there + # is exactly one remaining and so there is nothing left to + # do. + continue + elif txn.rowcount >= count: + raise Exception( + "We deleted more duplicate rows from 'user_ips' than expected", + ) + + # The previous step didn't delete enough rows, so we fallback to + # step 2: # Drop all the duplicates txn.execute( From 218cc071c5ef027361eddc72ef1863b43b2cdd67 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:37:00 +0000 Subject: [PATCH 90/96] Newsfile --- changelog.d/4626.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4626.misc diff --git a/changelog.d/4626.misc b/changelog.d/4626.misc new file mode 100644 index 0000000000..f1a57dcf9a --- /dev/null +++ b/changelog.d/4626.misc @@ -0,0 +1 @@ +Improve 'user_ips' table deduplication background update From 483ba85c7a1a8ee9b7eebcc5c07d522c71229c9f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:55:27 +0000 Subject: [PATCH 91/96] Analyze user_ips before running deduplication Due to the table locks taken out by the naive upsert, the table statistics may be out of date. During deduplication it is important that the correct index is used as otherwise a full table scan may be incorrectly used, which can end up thrashing the database badly. --- synapse/storage/client_ips.py | 24 +++++++++++++++++++ .../schema/delta/53/user_ips_index.sql | 10 +++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 091d7116c5..6f81406269 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -65,6 +65,11 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): columns=["last_seen"], ) + self.register_background_update_handler( + "user_ips_analyze", + self._analyze_user_ip, + ) + self.register_background_update_handler( "user_ips_remove_dupes", self._remove_user_ip_dupes, @@ -108,6 +113,25 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): yield self._end_background_update("user_ips_drop_nonunique_index") defer.returnValue(1) + @defer.inlineCallbacks + def _analyze_user_ip(self, progress, batch_size): + # Background update to analyze user_ips table before we run the + # deduplication background update. The table may not have been analyzed + # for ages due to the table locks. + # + # This will lock out the naive upserts to user_ips while it happens, but + # the analyze should be quick (28GB table takes ~10s) + def user_ips_analyze(txn): + txn.execute("ANALYZE user_ips") + + end_last_seen = yield self.runInteraction( + "user_ips_analyze", user_ips_analyze + ) + + yield self._end_background_update("user_ips_analyze") + + defer.returnValue(1) + @defer.inlineCallbacks def _remove_user_ip_dupes(self, progress, batch_size): # This works function works by scanning the user_ips table in batches diff --git a/synapse/storage/schema/delta/53/user_ips_index.sql b/synapse/storage/schema/delta/53/user_ips_index.sql index 4ca346c111..b812c5794f 100644 --- a/synapse/storage/schema/delta/53/user_ips_index.sql +++ b/synapse/storage/schema/delta/53/user_ips_index.sql @@ -13,9 +13,13 @@ * limitations under the License. */ --- delete duplicates + -- analyze user_ips, to help ensure the correct indices are used INSERT INTO background_updates (update_name, progress_json) VALUES - ('user_ips_remove_dupes', '{}'); + ('user_ips_analyze', '{}'); + +-- delete duplicates +INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES + ('user_ips_remove_dupes', '{}', 'user_ips_analyze'); -- add a new unique index to user_ips table INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES @@ -23,4 +27,4 @@ INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES -- drop the old original index INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES - ('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index'); \ No newline at end of file + ('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index'); From b2327eb9cb2c03037acd58b320063ec34968ea81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:58:36 +0000 Subject: [PATCH 92/96] Newsfile --- changelog.d/4627.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4627.misc diff --git a/changelog.d/4627.misc b/changelog.d/4627.misc new file mode 100644 index 0000000000..f1a57dcf9a --- /dev/null +++ b/changelog.d/4627.misc @@ -0,0 +1 @@ +Improve 'user_ips' table deduplication background update From 495ea92350c4a3f6bd92cded5da939326b858d9b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 12:40:42 +0000 Subject: [PATCH 93/96] Fix pep8 --- synapse/storage/client_ips.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 6f81406269..cc23d7cdbe 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -124,7 +124,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): def user_ips_analyze(txn): txn.execute("ANALYZE user_ips") - end_last_seen = yield self.runInteraction( + yield self.runInteraction( "user_ips_analyze", user_ips_analyze ) From b18cd25e42711d99c62a098203f59f4e12f3fca7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:05:31 +0000 Subject: [PATCH 94/96] Fixup changelog entries --- changelog.d/4626.bugfix | 1 + changelog.d/4626.misc | 1 - changelog.d/4627.bugfix | 1 + changelog.d/4627.misc | 1 - 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4626.bugfix delete mode 100644 changelog.d/4626.misc create mode 100644 changelog.d/4627.bugfix delete mode 100644 changelog.d/4627.misc diff --git a/changelog.d/4626.bugfix b/changelog.d/4626.bugfix new file mode 100644 index 0000000000..cc71df44ca --- /dev/null +++ b/changelog.d/4626.bugfix @@ -0,0 +1 @@ +Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4626.misc b/changelog.d/4626.misc deleted file mode 100644 index f1a57dcf9a..0000000000 --- a/changelog.d/4626.misc +++ /dev/null @@ -1 +0,0 @@ -Improve 'user_ips' table deduplication background update diff --git a/changelog.d/4627.bugfix b/changelog.d/4627.bugfix new file mode 100644 index 0000000000..cc71df44ca --- /dev/null +++ b/changelog.d/4627.bugfix @@ -0,0 +1 @@ +Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4627.misc b/changelog.d/4627.misc deleted file mode 100644 index f1a57dcf9a..0000000000 --- a/changelog.d/4627.misc +++ /dev/null @@ -1 +0,0 @@ -Improve 'user_ips' table deduplication background update From d2fa7b7e99a50b47b73de7e673fbe12a6cc12adc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:22:25 +0000 Subject: [PATCH 95/96] Update changelog and version --- CHANGES.md | 46 ++++++++++++++++++++++++++++++++++++++++ changelog.d/3902.feature | 1 - changelog.d/4420.feature | 1 - changelog.d/4462.misc | 1 - changelog.d/4513.misc | 1 - changelog.d/4522.feature | 1 - changelog.d/4530.bugfix | 1 - changelog.d/4546.bugfix | 1 - changelog.d/4567.misc | 1 - changelog.d/4576.misc | 1 - changelog.d/4578.misc | 1 - changelog.d/4580.feature | 1 - changelog.d/4584.misc | 1 - changelog.d/4586.misc | 1 - changelog.d/4589.bugfix | 1 - changelog.d/4591.bugfix | 1 - changelog.d/4592.feature | 2 -- changelog.d/4607.misc | 1 - changelog.d/4608.bugfix | 1 - changelog.d/4611.misc | 1 - changelog.d/4613.feature | 1 - changelog.d/4614.feature | 1 - changelog.d/4615.feature | 1 - changelog.d/4616.misc | 1 - changelog.d/4617.feature | 1 - changelog.d/4618.bugfix | 1 - changelog.d/4619.misc | 1 - changelog.d/4621.misc | 1 - changelog.d/4625.bugfix | 1 - changelog.d/4626.bugfix | 1 - changelog.d/4627.bugfix | 1 - synapse/__init__.py | 2 +- 32 files changed, 47 insertions(+), 32 deletions(-) delete mode 100644 changelog.d/3902.feature delete mode 100644 changelog.d/4420.feature delete mode 100644 changelog.d/4462.misc delete mode 100644 changelog.d/4513.misc delete mode 100644 changelog.d/4522.feature delete mode 100644 changelog.d/4530.bugfix delete mode 100644 changelog.d/4546.bugfix delete mode 100644 changelog.d/4567.misc delete mode 100644 changelog.d/4576.misc delete mode 100644 changelog.d/4578.misc delete mode 100644 changelog.d/4580.feature delete mode 100644 changelog.d/4584.misc delete mode 100644 changelog.d/4586.misc delete mode 100644 changelog.d/4589.bugfix delete mode 100644 changelog.d/4591.bugfix delete mode 100644 changelog.d/4592.feature delete mode 100644 changelog.d/4607.misc delete mode 100644 changelog.d/4608.bugfix delete mode 100644 changelog.d/4611.misc delete mode 100644 changelog.d/4613.feature delete mode 100644 changelog.d/4614.feature delete mode 100644 changelog.d/4615.feature delete mode 100644 changelog.d/4616.misc delete mode 100644 changelog.d/4617.feature delete mode 100644 changelog.d/4618.bugfix delete mode 100644 changelog.d/4619.misc delete mode 100644 changelog.d/4621.misc delete mode 100644 changelog.d/4625.bugfix delete mode 100644 changelog.d/4626.bugfix delete mode 100644 changelog.d/4627.bugfix diff --git a/CHANGES.md b/CHANGES.md index e330aea9e3..5120631fb6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,49 @@ +Synapse 0.99.1rc1 (2019-02-12) +============================== + +Features +-------- + +- Include m.room.encryption on invites by default ([\#3902](https://github.com/matrix-org/synapse/issues/3902)) +- Federation OpenID listener resource can now be activated even if federation is disabled ([\#4420](https://github.com/matrix-org/synapse/issues/4420)) +- Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. ([\#4522](https://github.com/matrix-org/synapse/issues/4522)) +- Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580)) +- Allow the "unavailable" presence status for /sync. + This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592)) +- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617)) +- The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614)) + + +Bugfixes +-------- + +- Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530)) +- Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546)) +- Synapse is now tolerant of the tls_fingerprints option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) +- Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591)) +- Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608)) +- Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618)) +- fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) +- Fix performance of 'user_ips' table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) + + +Internal Changes +---------------- + +- Change the user directory state query to use a filtered call to the db instead of a generic one. ([\#4462](https://github.com/matrix-org/synapse/issues/4462)) +- Reject federation transactions if they include more than 50 PDUs or 100 EDUs. ([\#4513](https://github.com/matrix-org/synapse/issues/4513)) +- Reduce duplication of ``synapse.app`` code. ([\#4567](https://github.com/matrix-org/synapse/issues/4567)) +- Fix docker upload job to push -py2 images. ([\#4576](https://github.com/matrix-org/synapse/issues/4576)) +- Add port configuration information to ACME instructions. ([\#4578](https://github.com/matrix-org/synapse/issues/4578)) +- Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584)) +- Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586)) +- Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607)) +- Move ClientTLSOptionsFactory init out of refresh_certificates ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) +- Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616)) +- Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619)) +- README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621)) + + Synapse 0.99.0 (2019-02-05) =========================== diff --git a/changelog.d/3902.feature b/changelog.d/3902.feature deleted file mode 100644 index eb8d9f2393..0000000000 --- a/changelog.d/3902.feature +++ /dev/null @@ -1 +0,0 @@ -Include m.room.encryption on invites by default diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature deleted file mode 100644 index 05e777c624..0000000000 --- a/changelog.d/4420.feature +++ /dev/null @@ -1 +0,0 @@ -Federation OpenID listener resource can now be activated even if federation is disabled diff --git a/changelog.d/4462.misc b/changelog.d/4462.misc deleted file mode 100644 index 03a4d7ae1c..0000000000 --- a/changelog.d/4462.misc +++ /dev/null @@ -1 +0,0 @@ -Change the user directory state query to use a filtered call to the db instead of a generic one. \ No newline at end of file diff --git a/changelog.d/4513.misc b/changelog.d/4513.misc deleted file mode 100644 index 1f64a96465..0000000000 --- a/changelog.d/4513.misc +++ /dev/null @@ -1 +0,0 @@ -Reject federation transactions if they include more than 50 PDUs or 100 EDUs. \ No newline at end of file diff --git a/changelog.d/4522.feature b/changelog.d/4522.feature deleted file mode 100644 index ef18daf60b..0000000000 --- a/changelog.d/4522.feature +++ /dev/null @@ -1 +0,0 @@ -Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. diff --git a/changelog.d/4530.bugfix b/changelog.d/4530.bugfix deleted file mode 100644 index d010af927e..0000000000 --- a/changelog.d/4530.bugfix +++ /dev/null @@ -1 +0,0 @@ -Copy over room federation ability on room upgrade. \ No newline at end of file diff --git a/changelog.d/4546.bugfix b/changelog.d/4546.bugfix deleted file mode 100644 index 056f2848ed..0000000000 --- a/changelog.d/4546.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix noisy "twisted.internet.task.TaskStopped" errors in logs diff --git a/changelog.d/4567.misc b/changelog.d/4567.misc deleted file mode 100644 index 96a2e0aefc..0000000000 --- a/changelog.d/4567.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce duplication of ``synapse.app`` code. diff --git a/changelog.d/4576.misc b/changelog.d/4576.misc deleted file mode 100644 index 94b1ade2e3..0000000000 --- a/changelog.d/4576.misc +++ /dev/null @@ -1 +0,0 @@ -Fix docker upload job to push -py2 images. diff --git a/changelog.d/4578.misc b/changelog.d/4578.misc deleted file mode 100644 index d1c006bb6b..0000000000 --- a/changelog.d/4578.misc +++ /dev/null @@ -1 +0,0 @@ -Add port configuration information to ACME instructions. \ No newline at end of file diff --git a/changelog.d/4580.feature b/changelog.d/4580.feature deleted file mode 100644 index a2a5a77dbe..0000000000 --- a/changelog.d/4580.feature +++ /dev/null @@ -1 +0,0 @@ -Add ability to update backup versions \ No newline at end of file diff --git a/changelog.d/4584.misc b/changelog.d/4584.misc deleted file mode 100644 index 4dec2e2b5c..0000000000 --- a/changelog.d/4584.misc +++ /dev/null @@ -1 +0,0 @@ -Update MSC1711 FAQ to calrify .well-known usage diff --git a/changelog.d/4586.misc b/changelog.d/4586.misc deleted file mode 100644 index 37af371ccf..0000000000 --- a/changelog.d/4586.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up default listener configuration diff --git a/changelog.d/4589.bugfix b/changelog.d/4589.bugfix deleted file mode 100644 index d5783f46e8..0000000000 --- a/changelog.d/4589.bugfix +++ /dev/null @@ -1 +0,0 @@ -Synapse is now tolerant of the tls_fingerprints option being None or not specified. diff --git a/changelog.d/4591.bugfix b/changelog.d/4591.bugfix deleted file mode 100644 index 628bbb6d81..0000000000 --- a/changelog.d/4591.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 'no unique or exclusion constraint' error diff --git a/changelog.d/4592.feature b/changelog.d/4592.feature deleted file mode 100644 index 112005ded4..0000000000 --- a/changelog.d/4592.feature +++ /dev/null @@ -1,2 +0,0 @@ -Allow the "unavailable" presence status for /sync. -This change makes Synapse compliant with r0.4.0 of the Client-Server specification. diff --git a/changelog.d/4607.misc b/changelog.d/4607.misc deleted file mode 100644 index 160a824378..0000000000 --- a/changelog.d/4607.misc +++ /dev/null @@ -1 +0,0 @@ -Clarifications for reverse proxy docs diff --git a/changelog.d/4608.bugfix b/changelog.d/4608.bugfix deleted file mode 100644 index e331a362c4..0000000000 --- a/changelog.d/4608.bugfix +++ /dev/null @@ -1 +0,0 @@ -Transfer Server ACLs on room upgrade. \ No newline at end of file diff --git a/changelog.d/4611.misc b/changelog.d/4611.misc deleted file mode 100644 index d2e0a05daa..0000000000 --- a/changelog.d/4611.misc +++ /dev/null @@ -1 +0,0 @@ -Move ClientTLSOptionsFactory init out of refresh_certificates diff --git a/changelog.d/4613.feature b/changelog.d/4613.feature deleted file mode 100644 index 098f906af2..0000000000 --- a/changelog.d/4613.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4614.feature b/changelog.d/4614.feature deleted file mode 100644 index 18e16dbc7b..0000000000 --- a/changelog.d/4614.feature +++ /dev/null @@ -1 +0,0 @@ -The default configuration no longer requires TLS certificates. diff --git a/changelog.d/4615.feature b/changelog.d/4615.feature deleted file mode 100644 index 098f906af2..0000000000 --- a/changelog.d/4615.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4616.misc b/changelog.d/4616.misc deleted file mode 100644 index ee79e208e3..0000000000 --- a/changelog.d/4616.misc +++ /dev/null @@ -1 +0,0 @@ -Fail cleanly if listener config lacks a 'port' diff --git a/changelog.d/4617.feature b/changelog.d/4617.feature deleted file mode 100644 index 098f906af2..0000000000 --- a/changelog.d/4617.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4618.bugfix b/changelog.d/4618.bugfix deleted file mode 100644 index 22115a020e..0000000000 --- a/changelog.d/4618.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix failure to start when not TLS certificate was given even if TLS was disabled. diff --git a/changelog.d/4619.misc b/changelog.d/4619.misc deleted file mode 100644 index 886fedf198..0000000000 --- a/changelog.d/4619.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant entries from docker config diff --git a/changelog.d/4621.misc b/changelog.d/4621.misc deleted file mode 100644 index 60e45cb70c..0000000000 --- a/changelog.d/4621.misc +++ /dev/null @@ -1 +0,0 @@ -README updates diff --git a/changelog.d/4625.bugfix b/changelog.d/4625.bugfix deleted file mode 100644 index 3dc0ecf24c..0000000000 --- a/changelog.d/4625.bugfix +++ /dev/null @@ -1 +0,0 @@ -fix self-signed cert notice from generate-config. diff --git a/changelog.d/4626.bugfix b/changelog.d/4626.bugfix deleted file mode 100644 index cc71df44ca..0000000000 --- a/changelog.d/4626.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4627.bugfix b/changelog.d/4627.bugfix deleted file mode 100644 index cc71df44ca..0000000000 --- a/changelog.d/4627.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix performance of 'user_ips' table deduplication background update diff --git a/synapse/__init__.py b/synapse/__init__.py index 048d6e572f..c211cb4e6f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.0" +__version__ = "0.99.1rc1" From 19818d66af9d67731e19020efae3a244d276b2c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:25:05 +0000 Subject: [PATCH 96/96] Fixup changelog --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5120631fb6..cc629978c2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,12 +19,12 @@ Bugfixes - Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530)) - Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546)) -- Synapse is now tolerant of the tls_fingerprints option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) +- Synapse is now tolerant of the `tls_fingerprints` option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) - Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591)) - Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608)) - Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618)) -- fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) -- Fix performance of 'user_ips' table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) +- Fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) +- Fix performance of `user_ips` table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) Internal Changes @@ -38,7 +38,7 @@ Internal Changes - Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584)) - Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586)) - Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607)) -- Move ClientTLSOptionsFactory init out of refresh_certificates ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) +- Move ClientTLSOptionsFactory init out of `refresh_certificates` ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) - Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616)) - Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619)) - README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621))