From f726f2dc6c398eb5fdf9d2dc0c56e96d930537ba Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 15 Oct 2018 22:21:45 +1100 Subject: [PATCH 01/22] version bump --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 43c5821ade..ea07fda14a 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.33.6" +__version__ = "0.33.7rc1" From 4e50fe3edb94125d496f2f77da0d653054e3ac3c Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 15 Oct 2018 22:22:49 +1100 Subject: [PATCH 02/22] update changelog --- changelog.d/4033.bugfix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/changelog.d/4033.bugfix b/changelog.d/4033.bugfix index 4e9e38cdcf..05d54d9524 100644 --- a/changelog.d/4033.bugfix +++ b/changelog.d/4033.bugfix @@ -1,2 +1 @@ -Synapse now starts on Python 3.7. -_All_ workers now start on Python 3. +Synapse now starts on Python 3.7. \ No newline at end of file From 24bc15eab46992faff9976ddfe9b15c611eab600 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 15 Oct 2018 22:23:13 +1100 Subject: [PATCH 03/22] update changelog --- CHANGES.md | 28 ++++++++++++++++++++++++++++ changelog.d/3995.bugfix | 1 - changelog.d/3996.bugfix | 1 - changelog.d/3997.bugfix | 1 - changelog.d/3999.bugfix | 1 - changelog.d/4008.misc | 1 - changelog.d/4017.misc | 1 - changelog.d/4019.feature | 1 - changelog.d/4022.misc | 1 - changelog.d/4027.bugfix | 1 - changelog.d/4033.bugfix | 1 - 11 files changed, 28 insertions(+), 10 deletions(-) delete mode 100644 changelog.d/3995.bugfix delete mode 100644 changelog.d/3996.bugfix delete mode 100644 changelog.d/3997.bugfix delete mode 100644 changelog.d/3999.bugfix delete mode 100644 changelog.d/4008.misc delete mode 100644 changelog.d/4017.misc delete mode 100644 changelog.d/4019.feature delete mode 100644 changelog.d/4022.misc delete mode 100644 changelog.d/4027.bugfix delete mode 100644 changelog.d/4033.bugfix diff --git a/CHANGES.md b/CHANGES.md index 048b9f95db..6c80a86461 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,31 @@ +Synapse 0.33.7rc1 (2018-10-15) +============================== + +Features +-------- + +- Add support for end-to-end key backup (MSC1687) ([\#4019](https://github.com/matrix-org/synapse/issues/4019)) + + +Bugfixes +-------- + +- Fix bug in event persistence logic which caused 'NoneType is not iterable' ([\#3995](https://github.com/matrix-org/synapse/issues/3995)) +- Fix exception in background metrics collection ([\#3996](https://github.com/matrix-org/synapse/issues/3996)) +- Fix exception handling in fetching remote profiles ([\#3997](https://github.com/matrix-org/synapse/issues/3997)) +- Fix handling of rejected threepid invites ([\#3999](https://github.com/matrix-org/synapse/issues/3999)) +- Workers now start on Python 3. ([\#4027](https://github.com/matrix-org/synapse/issues/4027)) +- Synapse now starts on Python 3.7. ([\#4033](https://github.com/matrix-org/synapse/issues/4033)) + + +Internal Changes +---------------- + +- Log exceptions in looping calls ([\#4008](https://github.com/matrix-org/synapse/issues/4008)) +- Optimisation for serving federation requests ([\#4017](https://github.com/matrix-org/synapse/issues/4017)) +- Add metric to count number of non-empty sync responses ([\#4022](https://github.com/matrix-org/synapse/issues/4022)) + + Synapse 0.33.6 (2018-10-04) =========================== diff --git a/changelog.d/3995.bugfix b/changelog.d/3995.bugfix deleted file mode 100644 index 2adc36756b..0000000000 --- a/changelog.d/3995.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug in event persistence logic which caused 'NoneType is not iterable' \ No newline at end of file diff --git a/changelog.d/3996.bugfix b/changelog.d/3996.bugfix deleted file mode 100644 index a056485ea1..0000000000 --- a/changelog.d/3996.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix exception in background metrics collection diff --git a/changelog.d/3997.bugfix b/changelog.d/3997.bugfix deleted file mode 100644 index b060ee8c18..0000000000 --- a/changelog.d/3997.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix exception handling in fetching remote profiles diff --git a/changelog.d/3999.bugfix b/changelog.d/3999.bugfix deleted file mode 100644 index dc3b2caffa..0000000000 --- a/changelog.d/3999.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix handling of rejected threepid invites diff --git a/changelog.d/4008.misc b/changelog.d/4008.misc deleted file mode 100644 index 5730210054..0000000000 --- a/changelog.d/4008.misc +++ /dev/null @@ -1 +0,0 @@ -Log exceptions in looping calls diff --git a/changelog.d/4017.misc b/changelog.d/4017.misc deleted file mode 100644 index b1ceb06560..0000000000 --- a/changelog.d/4017.misc +++ /dev/null @@ -1 +0,0 @@ -Optimisation for serving federation requests \ No newline at end of file diff --git a/changelog.d/4019.feature b/changelog.d/4019.feature deleted file mode 100644 index 49e066d269..0000000000 --- a/changelog.d/4019.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for end-to-end key backup (MSC1687) diff --git a/changelog.d/4022.misc b/changelog.d/4022.misc deleted file mode 100644 index 5b0e136795..0000000000 --- a/changelog.d/4022.misc +++ /dev/null @@ -1 +0,0 @@ -Add metric to count number of non-empty sync responses diff --git a/changelog.d/4027.bugfix b/changelog.d/4027.bugfix deleted file mode 100644 index c9bbff3f68..0000000000 --- a/changelog.d/4027.bugfix +++ /dev/null @@ -1 +0,0 @@ -Workers now start on Python 3. diff --git a/changelog.d/4033.bugfix b/changelog.d/4033.bugfix deleted file mode 100644 index 05d54d9524..0000000000 --- a/changelog.d/4033.bugfix +++ /dev/null @@ -1 +0,0 @@ -Synapse now starts on Python 3.7. \ No newline at end of file From b8a5b0097c4fc88a43c5c5f9785720d38299d3d8 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 16 Oct 2018 10:44:49 +0100 Subject: [PATCH 04/22] Various cleanups in the federation client code (#4031) - Improve logging: log things in the right order, include destination and txids in all log lines, don't log successful responses twice - Fix the docstring on TransportLayerClient.send_transaction - Don't use treq.request, which is overcomplicated for our purposes: just use a twisted.web.client.Agent. - simplify the logic for setting up the bodyProducer - fix bytes/str confusions --- changelog.d/4031.misc | 1 + synapse/federation/transaction_queue.py | 27 ++++----- synapse/federation/transport/client.py | 19 +++--- synapse/http/matrixfederationclient.py | 78 +++++++++++++------------ 4 files changed, 64 insertions(+), 61 deletions(-) create mode 100644 changelog.d/4031.misc diff --git a/changelog.d/4031.misc b/changelog.d/4031.misc new file mode 100644 index 0000000000..60be8b59fd --- /dev/null +++ b/changelog.d/4031.misc @@ -0,0 +1 @@ +Various cleanups in the federation client code diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 98b5950800..3fdd63be95 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -633,14 +633,6 @@ class TransactionQueue(object): transaction, json_data_cb ) code = 200 - - if response: - for e_id, r in response.get("pdus", {}).items(): - if "error" in r: - logger.warn( - "Transaction returned error for %s: %s", - e_id, r, - ) except HttpResponseException as e: code = e.code response = e.response @@ -657,19 +649,24 @@ class TransactionQueue(object): destination, txn_id, code ) - logger.debug("TX [%s] Sent transaction", destination) - logger.debug("TX [%s] Marking as delivered...", destination) - yield self.transaction_actions.delivered( transaction, code, response ) - logger.debug("TX [%s] Marked as delivered", destination) + logger.debug("TX [%s] {%s} Marked as delivered", destination, txn_id) - if code != 200: + if code == 200: + for e_id, r in response.get("pdus", {}).items(): + if "error" in r: + logger.warn( + "TX [%s] {%s} Remote returned error for %s: %s", + destination, txn_id, e_id, r, + ) + else: for p in pdus: - logger.info( - "Failed to send event %s to %s", p.event_id, destination + logger.warn( + "TX [%s] {%s} Failed to send event %s", + destination, txn_id, p.event_id, ) success = False diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 2ab973d6c8..edba5a9808 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -143,9 +143,17 @@ class TransportLayerClient(object): transaction (Transaction) Returns: - Deferred: Results of the deferred is a tuple in the form of - (response_code, response_body) where the response_body is a - python dict decoded from json + Deferred: Succeeds when we get a 2xx HTTP response. The result + will be the decoded JSON body. + + Fails with ``HTTPRequestException`` if we get an HTTP response + code >= 300. + + Fails with ``NotRetryingDestination`` if we are not yet ready + to retry this server. + + Fails with ``FederationDeniedError`` if this destination + is not on our federation whitelist """ logger.debug( "send_data dest=%s, txid=%s", @@ -170,11 +178,6 @@ class TransportLayerClient(object): backoff_on_404=True, # If we get a 404 the other side has gone ) - logger.debug( - "send_data dest=%s, txid=%s, got response: 200", - transaction.destination, transaction.transaction_id, - ) - defer.returnValue(response) @defer.inlineCallbacks diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 14b12cd1c4..fcc02fc77d 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -195,7 +195,7 @@ class MatrixFederationHttpClient(object): ) self.clock = hs.get_clock() self._store = hs.get_datastore() - self.version_string = hs.version_string.encode('ascii') + self.version_string_bytes = hs.version_string.encode('ascii') self.default_timeout = 60 def schedule(x): @@ -261,8 +261,8 @@ class MatrixFederationHttpClient(object): ignore_backoff=ignore_backoff, ) - method = request.method - destination = request.destination + method_bytes = request.method.encode("ascii") + destination_bytes = request.destination.encode("ascii") path_bytes = request.path.encode("ascii") if request.query: query_bytes = encode_query_args(request.query) @@ -270,8 +270,8 @@ class MatrixFederationHttpClient(object): query_bytes = b"" headers_dict = { - "User-Agent": [self.version_string], - "Host": [request.destination], + b"User-Agent": [self.version_string_bytes], + b"Host": [destination_bytes], } with limiter: @@ -282,50 +282,51 @@ class MatrixFederationHttpClient(object): else: retries_left = MAX_SHORT_RETRIES - url = urllib.parse.urlunparse(( - b"matrix", destination.encode("ascii"), + url_bytes = urllib.parse.urlunparse(( + b"matrix", destination_bytes, path_bytes, None, query_bytes, b"", - )).decode('ascii') + )) + url_str = url_bytes.decode('ascii') - http_url = urllib.parse.urlunparse(( + url_to_sign_bytes = urllib.parse.urlunparse(( b"", b"", path_bytes, None, query_bytes, b"", - )).decode('ascii') + )) while True: try: json = request.get_json() if json: - data = encode_canonical_json(json) - headers_dict["Content-Type"] = ["application/json"] + headers_dict[b"Content-Type"] = [b"application/json"] self.sign_request( - destination, method, http_url, headers_dict, json + destination_bytes, method_bytes, url_to_sign_bytes, + headers_dict, json, ) - else: - data = None - self.sign_request(destination, method, http_url, headers_dict) - - logger.info( - "{%s} [%s] Sending request: %s %s", - request.txn_id, destination, method, url - ) - - if data: + data = encode_canonical_json(json) producer = FileBodyProducer( BytesIO(data), - cooperator=self._cooperator + cooperator=self._cooperator, ) else: producer = None + self.sign_request( + destination_bytes, method_bytes, url_to_sign_bytes, + headers_dict, + ) - request_deferred = treq.request( - method, - url, + logger.info( + "{%s} [%s] Sending request: %s %s", + request.txn_id, request.destination, request.method, + url_str, + ) + + # we don't want all the fancy cookie and redirect handling that + # treq.request gives: just use the raw Agent. + request_deferred = self.agent.request( + method_bytes, + url_bytes, headers=Headers(headers_dict), - data=producer, - agent=self.agent, - reactor=self.hs.get_reactor(), - unbuffered=True + bodyProducer=producer, ) request_deferred = timeout_deferred( @@ -344,9 +345,9 @@ class MatrixFederationHttpClient(object): logger.warn( "{%s} [%s] Request failed: %s %s: %s", request.txn_id, - destination, - method, - url, + request.destination, + request.method, + url_str, _flatten_response_never_received(e), ) @@ -366,7 +367,7 @@ class MatrixFederationHttpClient(object): logger.debug( "{%s} [%s] Waiting %ss before re-sending...", request.txn_id, - destination, + request.destination, delay, ) @@ -378,7 +379,7 @@ class MatrixFederationHttpClient(object): logger.info( "{%s} [%s] Got response headers: %d %s", request.txn_id, - destination, + request.destination, response.code, response.phrase.decode('ascii', errors='replace'), ) @@ -411,8 +412,9 @@ class MatrixFederationHttpClient(object): destination_is must be non-None. method (bytes): The HTTP method of the request url_bytes (bytes): The URI path of the request - headers_dict (dict): Dictionary of request headers to append to - content (bytes): The body of the request + headers_dict (dict[bytes, list[bytes]]): Dictionary of request headers to + append to + content (object): The body of the request destination_is (bytes): As 'destination', but if the destination is an identity server From a94967bc5f3735375f12bf8275236eb2ea1cc66b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 16 Oct 2018 15:29:08 +0100 Subject: [PATCH 05/22] run the circle builds in docker containers Docker containers spin up faster than entire VMs. --- .circleci/config.yml | 89 ++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 41 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec3848b048..5395028426 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,99 +23,106 @@ jobs: - run: docker push matrixdotorg/synapse:latest - run: docker push matrixdotorg/synapse:latest-py3 sytestpy2: - machine: true + docker: + - image: matrixdotorg/sytest-synapsepy2 + working_directory: /src steps: - checkout - - run: docker pull matrixdotorg/sytest-synapsepy2 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2 + - run: /synapse_sytest.sh - store_artifacts: - path: ~/project/logs + path: /logs destination: logs - store_test_results: - path: logs + path: /logs sytestpy2postgres: - machine: true + docker: + - image: matrixdotorg/sytest-synapsepy2 + working_directory: /src steps: - checkout - - run: docker pull matrixdotorg/sytest-synapsepy2 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2 + - run: POSTGRES=1 /synapse_sytest.sh - store_artifacts: - path: ~/project/logs + path: /logs destination: logs - store_test_results: - path: logs + path: /logs sytestpy2merged: - machine: true + docker: + - image: matrixdotorg/sytest-synapsepy2 + working_directory: /src steps: - checkout - run: bash .circleci/merge_base_branch.sh - - run: docker pull matrixdotorg/sytest-synapsepy2 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2 + - run: /synapse_sytest.sh - store_artifacts: - path: ~/project/logs + path: /logs destination: logs - store_test_results: - path: logs - + path: /logs sytestpy2postgresmerged: - machine: true + docker: + - image: matrixdotorg/sytest-synapsepy2 + working_directory: /src steps: - checkout - run: bash .circleci/merge_base_branch.sh - - run: docker pull matrixdotorg/sytest-synapsepy2 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2 + - run: POSTGRES=1 /synapse_sytest.sh - store_artifacts: - path: ~/project/logs + path: /logs destination: logs - store_test_results: - path: logs + path: /logs sytestpy3: - machine: true + docker: + - image: matrixdotorg/sytest-synapsepy3 + working_directory: /src steps: - checkout - - run: docker pull matrixdotorg/sytest-synapsepy3 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3 + - run: /synapse_sytest.sh - store_artifacts: - path: ~/project/logs + path: /logs destination: logs - store_test_results: - path: logs + path: /logs sytestpy3postgres: - machine: true + docker: + - image: matrixdotorg/sytest-synapsepy3 + working_directory: /src steps: - checkout - - run: docker pull matrixdotorg/sytest-synapsepy3 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3 + - run: POSTGRES=1 /synapse_sytest.sh - store_artifacts: - path: ~/project/logs + path: /logs destination: logs - store_test_results: - path: logs + path: /logs sytestpy3merged: - machine: true + docker: + - image: matrixdotorg/sytest-synapsepy3 + working_directory: /src steps: - checkout - run: bash .circleci/merge_base_branch.sh - - run: docker pull matrixdotorg/sytest-synapsepy3 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3 + - run: /synapse_sytest.sh - store_artifacts: - path: ~/project/logs + path: /logs destination: logs - store_test_results: - path: logs + path: /logs sytestpy3postgresmerged: - machine: true + docker: + - image: matrixdotorg/sytest-synapsepy3 + working_directory: /src steps: - checkout - run: bash .circleci/merge_base_branch.sh - - run: docker pull matrixdotorg/sytest-synapsepy3 - - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3 + - run: POSTGRES=1 /synapse_sytest.sh - store_artifacts: - path: ~/project/logs + path: /logs destination: logs - store_test_results: - path: logs + path: /logs workflows: version: 2 From 017eb9d17acff2fd6f053f6e500eda0e26cc22eb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 16 Oct 2018 16:31:47 +0100 Subject: [PATCH 06/22] changelog --- changelog.d/4041.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4041.misc diff --git a/changelog.d/4041.misc b/changelog.d/4041.misc new file mode 100644 index 0000000000..8cce9daac9 --- /dev/null +++ b/changelog.d/4041.misc @@ -0,0 +1 @@ +Run the CircleCI builds in docker containers From 10405153c226b9aa2eccb0c4cf16de03cabde778 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 16 Oct 2018 16:47:26 +0100 Subject: [PATCH 07/22] Use wget rather than curl the docker image doesn't have wget --- .circleci/merge_base_branch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/merge_base_branch.sh b/.circleci/merge_base_branch.sh index 6b0bf3aa48..b2c8c40f4c 100755 --- a/.circleci/merge_base_branch.sh +++ b/.circleci/merge_base_branch.sh @@ -16,7 +16,7 @@ then GITBASE="develop" else # Get the reference, using the GitHub API - GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'` + GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'` fi # Show what we are before @@ -31,4 +31,4 @@ git fetch -u origin $GITBASE git merge --no-edit origin/$GITBASE # Show what we are after. -git show -s \ No newline at end of file +git show -s From fc0f13dd036cec4e41f5969d021d9dd10d6e5016 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 16 Oct 2018 20:37:16 +0100 Subject: [PATCH 08/22] Fix incorrect truncation in get_missing_events It's quite important that get_missing_events returns the *latest* events in the room; however we were pulling event ids out of the database until we got *at least* 10, and then taking the *earliest* of the results. We also shouldn't really be relying on depth, and should be checking the room_id. --- changelog.d/4045.bugfix | 1 + synapse/federation/federation_server.py | 8 +++--- synapse/federation/transport/server.py | 2 -- synapse/handlers/federation.py | 12 ++++---- synapse/storage/event_federation.py | 38 +++++++++++-------------- 5 files changed, 26 insertions(+), 35 deletions(-) create mode 100644 changelog.d/4045.bugfix diff --git a/changelog.d/4045.bugfix b/changelog.d/4045.bugfix new file mode 100644 index 0000000000..fa50eb5aff --- /dev/null +++ b/changelog.d/4045.bugfix @@ -0,0 +1 @@ +Fix bug which made get_missing_events return too few events \ No newline at end of file diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 819e8f7331..4efe95faa4 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -507,19 +507,19 @@ class FederationServer(FederationBase): @defer.inlineCallbacks @log_function def on_get_missing_events(self, origin, room_id, earliest_events, - latest_events, limit, min_depth): + latest_events, limit): with (yield self._server_linearizer.queue((origin, room_id))): origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, room_id) logger.info( "on_get_missing_events: earliest_events: %r, latest_events: %r," - " limit: %d, min_depth: %d", - earliest_events, latest_events, limit, min_depth + " limit: %d", + earliest_events, latest_events, limit, ) missing_events = yield self.handler.on_get_missing_events( - origin, room_id, earliest_events, latest_events, limit, min_depth + origin, room_id, earliest_events, latest_events, limit, ) if len(missing_events) < 5: diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 2f874b4838..7288d49074 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -560,7 +560,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet): @defer.inlineCallbacks def on_POST(self, origin, content, query, room_id): limit = int(content.get("limit", 10)) - min_depth = int(content.get("min_depth", 0)) earliest_events = content.get("earliest_events", []) latest_events = content.get("latest_events", []) @@ -569,7 +568,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet): room_id=room_id, earliest_events=earliest_events, latest_events=latest_events, - min_depth=min_depth, limit=limit, ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 45d955e6f5..cab57a8849 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -309,8 +309,8 @@ class FederationHandler(BaseHandler): if sent_to_us_directly: logger.warn( - "[%s %s] Failed to fetch %d prev events: rejecting", - room_id, event_id, len(prevs - seen), + "[%s %s] Rejecting: failed to fetch %d prev events: %s", + room_id, event_id, len(prevs - seen), shortstr(prevs - seen) ) raise FederationError( "ERROR", @@ -452,8 +452,8 @@ class FederationHandler(BaseHandler): latest |= seen logger.info( - "[%s %s]: Requesting %d prev_events: %s", - room_id, event_id, len(prevs - seen), shortstr(prevs - seen) + "[%s %s]: Requesting missing events between %s and %s", + room_id, event_id, shortstr(latest), event_id, ) # XXX: we set timeout to 10s to help workaround @@ -1852,7 +1852,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def on_get_missing_events(self, origin, room_id, earliest_events, - latest_events, limit, min_depth): + latest_events, limit): in_room = yield self.auth.check_host_in_room( room_id, origin @@ -1861,14 +1861,12 @@ class FederationHandler(BaseHandler): raise AuthError(403, "Host not in room.") limit = min(limit, 20) - min_depth = max(min_depth, 0) missing_events = yield self.store.get_missing_events( room_id=room_id, earliest_events=earliest_events, latest_events=latest_events, limit=limit, - min_depth=min_depth, ) missing_events = yield filter_events_for_server( diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 24345b20a6..3faca2a042 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -376,33 +376,25 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, @defer.inlineCallbacks def get_missing_events(self, room_id, earliest_events, latest_events, - limit, min_depth): + limit): ids = yield self.runInteraction( "get_missing_events", self._get_missing_events, - room_id, earliest_events, latest_events, limit, min_depth + room_id, earliest_events, latest_events, limit, ) - events = yield self._get_events(ids) - - events = sorted( - [ev for ev in events if ev.depth >= min_depth], - key=lambda e: e.depth, - ) - - defer.returnValue(events[:limit]) + defer.returnValue(events) def _get_missing_events(self, txn, room_id, earliest_events, latest_events, - limit, min_depth): + limit): - earliest_events = set(earliest_events) - front = set(latest_events) - earliest_events - - event_results = set() + seen_events = set(earliest_events) + front = set(latest_events) - seen_events + event_results = [] query = ( "SELECT prev_event_id FROM event_edges " - "WHERE event_id = ? AND is_state = ? " + "WHERE room_id = ? AND event_id = ? AND is_state = ? " "LIMIT ?" ) @@ -411,18 +403,20 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, for event_id in front: txn.execute( query, - (event_id, False, limit - len(event_results)) + (room_id, event_id, False, limit - len(event_results)) ) - for e_id, in txn: - new_front.add(e_id) + new_results = set(t[0] for t in txn) - seen_events - new_front -= earliest_events - new_front -= event_results + new_front |= new_results + seen_events |= new_results + event_results.extend(new_results) front = new_front - event_results |= new_front + # we built the list working backwards from latest_events; we now need to + # reverse it so that the events are approximately chronological. + event_results.reverse() return event_results From d6a7797dd1d76a86e6914c2ae562fa83eee4c57f Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 17 Oct 2018 13:04:55 +0100 Subject: [PATCH 09/22] Fix roomlist since tokens on Python 3 (#4046) Thanks @Half-Shot !!! --- changelog.d/4046.bugfix | 1 + synapse/handlers/room_list.py | 11 ++++++++-- synapse/python_dependencies.py | 2 +- tests/handlers/test_roomlist.py | 39 +++++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4046.bugfix create mode 100644 tests/handlers/test_roomlist.py diff --git a/changelog.d/4046.bugfix b/changelog.d/4046.bugfix new file mode 100644 index 0000000000..5046dd1ce3 --- /dev/null +++ b/changelog.d/4046.bugfix @@ -0,0 +1 @@ +Fix issue where Python 3 users couldn't paginate /publicRooms diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 38e1737ec9..dc88620885 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -16,7 +16,7 @@ import logging from collections import namedtuple -from six import iteritems +from six import PY3, iteritems from six.moves import range import msgpack @@ -444,9 +444,16 @@ class RoomListNextBatch(namedtuple("RoomListNextBatch", ( @classmethod def from_token(cls, token): + if PY3: + # The argument raw=False is only available on new versions of + # msgpack, and only really needed on Python 3. Gate it behind + # a PY3 check to avoid causing issues on Debian-packaged versions. + decoded = msgpack.loads(decode_base64(token), raw=False) + else: + decoded = msgpack.loads(decode_base64(token)) return RoomListNextBatch(**{ cls.REVERSE_KEY_DICT[key]: val - for key, val in msgpack.loads(decode_base64(token)).items() + for key, val in decoded.items() }) def to_token(self): diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2947f37f1a..f51184b50d 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -55,7 +55,7 @@ REQUIREMENTS = { "sortedcontainers>=1.4.4": ["sortedcontainers"], "pysaml2>=3.0.0": ["saml2"], "pymacaroons-pynacl>=0.9.3": ["pymacaroons"], - "msgpack-python>=0.3.0": ["msgpack"], + "msgpack-python>=0.4.2": ["msgpack"], "phonenumbers>=8.2.0": ["phonenumbers"], "six>=1.10": ["six"], diff --git a/tests/handlers/test_roomlist.py b/tests/handlers/test_roomlist.py new file mode 100644 index 0000000000..61eebb6985 --- /dev/null +++ b/tests/handlers/test_roomlist.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.handlers.room_list import RoomListNextBatch + +import tests.unittest +import tests.utils + + +class RoomListTestCase(tests.unittest.TestCase): + """ Tests RoomList's RoomListNextBatch. """ + + def setUp(self): + pass + + def test_check_read_batch_tokens(self): + batch_token = RoomListNextBatch( + stream_ordering="abcdef", + public_room_stream_id="123", + current_limit=20, + direction_is_forward=True, + ).to_token() + next_batch = RoomListNextBatch.from_token(batch_token) + self.assertEquals(next_batch.stream_ordering, "abcdef") + self.assertEquals(next_batch.public_room_stream_id, "123") + self.assertEquals(next_batch.current_limit, 20) + self.assertEquals(next_batch.direction_is_forward, True) From df33c164de7d73323814ac439a394173d662e366 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 17 Oct 2018 13:46:08 +0100 Subject: [PATCH 10/22] Only colourise synctl output when attached to tty --- synctl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synctl b/synctl index 09b64459b1..111b9c1816 100755 --- a/synctl +++ b/synctl @@ -48,7 +48,12 @@ def pid_running(pid): def write(message, colour=NORMAL, stream=sys.stdout): - if colour == NORMAL: + # Lets check if we're writing to a TTY before colouring + should_colour = False + if stream in (sys.stdout, sys.stderr): + should_colour = stream.isatty() + + if not should_colour or colour == NORMAL: stream.write(message + "\n") else: stream.write(colour + message + NORMAL + "\n") From 1af16acd4ca40b15183ae0a25ffcbc13c36c130d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 17 Oct 2018 13:48:09 +0100 Subject: [PATCH 11/22] Newsfile --- changelog.d/4049.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4049.misc diff --git a/changelog.d/4049.misc b/changelog.d/4049.misc new file mode 100644 index 0000000000..4370d9dfa6 --- /dev/null +++ b/changelog.d/4049.misc @@ -0,0 +1 @@ +Only colourise synctl output when attached to tty From f6a0a02a62db1a3030e5563d6454d01c4a76c38d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 17 Oct 2018 16:09:34 +0100 Subject: [PATCH 12/22] Fix bug where we raised StopIteration in a generator This made python 3.7 unhappy --- synapse/rest/media/v1/preview_url_resource.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index af01040a38..8c892ff187 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -596,10 +596,13 @@ def _iterate_over_text(tree, *tags_to_ignore): # to be returned. elements = iter([tree]) while True: - el = next(elements) + el = next(elements, None) + if el is None: + return + if isinstance(el, string_types): yield el - elif el is not None and el.tag not in tags_to_ignore: + elif el.tag not in tags_to_ignore: # el.text is the text before the first child, so we can immediately # return it if the text exists. if el.text: From 3a5d8d5891cc66658c40a796fcf9cf3f0f2e0916 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 17 Oct 2018 16:12:45 +0100 Subject: [PATCH 13/22] Newsfile --- changelog.d/4050.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4050.bugfix diff --git a/changelog.d/4050.bugfix b/changelog.d/4050.bugfix new file mode 100644 index 0000000000..3d1f6af847 --- /dev/null +++ b/changelog.d/4050.bugfix @@ -0,0 +1 @@ +Fix URL priewing to work in Python 3.7 From 1519572961d22aeaacbe9fccae7788f0f9e72b1c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 17 Oct 2018 15:44:34 +0100 Subject: [PATCH 14/22] Ship the email templates as package_data move the example email templates into the synapse package so that they can be used as package data, which should mean that all of the packaging mechanisms (pip, docker, debian, arch, etc) should now come with the example templates. In order to grandfather in people who relied on the templates being in the old place, check for that situation and fall back to using the defaults if the templates directory does not exist. --- MANIFEST.in | 2 +- changelog.d/4052.feature | 12 +++++++ docker/conf/homeserver.yaml | 4 ++- synapse/config/emailconfig.py | 33 +++++++++++++++++-- synapse/push/mailer.py | 5 ++- .../res}/templates/mail-Vector.css | 0 {res => synapse/res}/templates/mail.css | 0 {res => synapse/res}/templates/notif.html | 0 {res => synapse/res}/templates/notif.txt | 0 .../res}/templates/notif_mail.html | 0 {res => synapse/res}/templates/notif_mail.txt | 0 {res => synapse/res}/templates/room.html | 0 {res => synapse/res}/templates/room.txt | 0 13 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 changelog.d/4052.feature rename {res => synapse/res}/templates/mail-Vector.css (100%) rename {res => synapse/res}/templates/mail.css (100%) rename {res => synapse/res}/templates/notif.html (100%) rename {res => synapse/res}/templates/notif.txt (100%) rename {res => synapse/res}/templates/notif_mail.html (100%) rename {res => synapse/res}/templates/notif_mail.txt (100%) rename {res => synapse/res}/templates/room.html (100%) rename {res => synapse/res}/templates/room.txt (100%) diff --git a/MANIFEST.in b/MANIFEST.in index c6a37ac685..25cdf0a61b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,12 +12,12 @@ recursive-include synapse/storage/schema *.sql recursive-include synapse/storage/schema *.py recursive-include docs * -recursive-include res * recursive-include scripts * recursive-include scripts-dev * recursive-include synapse *.pyi recursive-include tests *.py +recursive-include synapse/res * recursive-include synapse/static *.css recursive-include synapse/static *.gif recursive-include synapse/static *.html diff --git a/changelog.d/4052.feature b/changelog.d/4052.feature new file mode 100644 index 0000000000..0c01f06880 --- /dev/null +++ b/changelog.d/4052.feature @@ -0,0 +1,12 @@ +Ship the example email templates as part of the package + +**Note**: if you deploy your Synapse instance from a git checkout or a github +snapshot URL, then this means that the example email templates will no longer +be installed in `res/templates`. If you have email notifications enabled, you +should ensure that `email.template_dir` is either configured to point at a +directory where you have installed customised templates, or leave it unset to +use the default templates. + +The configuration parser will try to detect the situation where +`email.template_dir` is incorrectly set to `res/templates` and do the right +thing, but will warn about this. \ No newline at end of file diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index cfe88788f2..a38b929f50 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -211,7 +211,9 @@ email: require_transport_security: False notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}" app_name: Matrix - template_dir: res/templates + # if template_dir is unset, uses the example templates that are part of + # the Synapse distribution. + #template_dir: res/templates notif_template_html: notif_mail.html notif_template_text: notif_mail.txt notif_for_new_users: True diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index fe156b6930..862f92c6ae 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -13,11 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + # This file can't be called email.py because if it is, we cannot: import email.utils +import logging +import os +import sys +import textwrap from ._base import Config +logger = logging.getLogger(__name__) + class EmailConfig(Config): def read_config(self, config): @@ -38,7 +46,6 @@ class EmailConfig(Config): "smtp_host", "smtp_port", "notif_from", - "template_dir", "notif_template_html", "notif_template_text", ] @@ -62,9 +69,27 @@ class EmailConfig(Config): self.email_smtp_host = email_config["smtp_host"] self.email_smtp_port = email_config["smtp_port"] self.email_notif_from = email_config["notif_from"] - self.email_template_dir = email_config["template_dir"] self.email_notif_template_html = email_config["notif_template_html"] self.email_notif_template_text = email_config["notif_template_text"] + + self.email_template_dir = email_config.get("template_dir") + + # backwards-compatibility hack + if ( + self.email_template_dir == "res/templates" + and not os.path.isfile( + os.path.join(self.email_template_dir, self.email_notif_template_text) + ) + ): + t = """\ +WARNING: The email notifier is configured to look for templates in '%s', but no templates +could be found there. We will fall back to using the example templates; to get rid of this +warning, leave 'email.template_dir' unset. +""" % (self.email_template_dir,) + + print(textwrap.fill(t, width=80) + "\n", file=sys.stderr) + self.email_template_dir = None + self.email_notif_for_new_users = email_config.get( "notif_for_new_users", True ) @@ -113,7 +138,9 @@ class EmailConfig(Config): # require_transport_security: False # notif_from: "Your Friendly %(app)s Home Server " # app_name: Matrix - # template_dir: res/templates + # # if template_dir is unset, uses the example templates that are part of + # # the Synapse distribution. + # #template_dir: res/templates # notif_template_html: notif_mail.html # notif_template_text: notif_mail.txt # notif_for_new_users: True diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 1a5a10d974..b9dcfee740 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -528,7 +528,10 @@ def load_jinja2_templates(config): """ logger.info("loading jinja2") - loader = jinja2.FileSystemLoader(config.email_template_dir) + if config.email_template_dir: + loader = jinja2.FileSystemLoader(config.email_template_dir) + else: + loader = jinja2.PackageLoader('synapse', 'res/templates') env = jinja2.Environment(loader=loader) env.filters["format_ts"] = format_ts_filter env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config) diff --git a/res/templates/mail-Vector.css b/synapse/res/templates/mail-Vector.css similarity index 100% rename from res/templates/mail-Vector.css rename to synapse/res/templates/mail-Vector.css diff --git a/res/templates/mail.css b/synapse/res/templates/mail.css similarity index 100% rename from res/templates/mail.css rename to synapse/res/templates/mail.css diff --git a/res/templates/notif.html b/synapse/res/templates/notif.html similarity index 100% rename from res/templates/notif.html rename to synapse/res/templates/notif.html diff --git a/res/templates/notif.txt b/synapse/res/templates/notif.txt similarity index 100% rename from res/templates/notif.txt rename to synapse/res/templates/notif.txt diff --git a/res/templates/notif_mail.html b/synapse/res/templates/notif_mail.html similarity index 100% rename from res/templates/notif_mail.html rename to synapse/res/templates/notif_mail.html diff --git a/res/templates/notif_mail.txt b/synapse/res/templates/notif_mail.txt similarity index 100% rename from res/templates/notif_mail.txt rename to synapse/res/templates/notif_mail.txt diff --git a/res/templates/room.html b/synapse/res/templates/room.html similarity index 100% rename from res/templates/room.html rename to synapse/res/templates/room.html diff --git a/res/templates/room.txt b/synapse/res/templates/room.txt similarity index 100% rename from res/templates/room.txt rename to synapse/res/templates/room.txt From c8f2c199910b70a5d94acf5597a4f3ba94cd3969 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 17 Oct 2018 16:56:22 +0100 Subject: [PATCH 15/22] Put the warning blob at the top of the file --- synapse/config/emailconfig.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py index 862f92c6ae..e2582cfecc 100644 --- a/synapse/config/emailconfig.py +++ b/synapse/config/emailconfig.py @@ -26,6 +26,12 @@ from ._base import Config logger = logging.getLogger(__name__) +TEMPLATE_DIR_WARNING = """\ +WARNING: The email notifier is configured to look for templates in '%(template_dir)s', +but no templates could be found there. We will fall back to using the example templates; +to get rid of this warning, leave 'email.template_dir' unset. +""" + class EmailConfig(Config): def read_config(self, config): @@ -81,12 +87,9 @@ class EmailConfig(Config): os.path.join(self.email_template_dir, self.email_notif_template_text) ) ): - t = """\ -WARNING: The email notifier is configured to look for templates in '%s', but no templates -could be found there. We will fall back to using the example templates; to get rid of this -warning, leave 'email.template_dir' unset. -""" % (self.email_template_dir,) - + t = TEMPLATE_DIR_WARNING % { + "template_dir": self.email_template_dir, + } print(textwrap.fill(t, width=80) + "\n", file=sys.stderr) self.email_template_dir = None From a5aea15a6b0f4c389a243bbb3ac375c9ed128146 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 17 Oct 2018 17:06:49 +0100 Subject: [PATCH 16/22] Assume isatty is always defined, and catch AttributeError. Also don't bother checking colour==Normal --- synctl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synctl b/synctl index 111b9c1816..4bd4c68b37 100755 --- a/synctl +++ b/synctl @@ -50,10 +50,14 @@ def pid_running(pid): def write(message, colour=NORMAL, stream=sys.stdout): # Lets check if we're writing to a TTY before colouring should_colour = False - if stream in (sys.stdout, sys.stderr): + try: should_colour = stream.isatty() + except AttributeError: + # Just in case `isatty` isn't defined on everything. The python + # docs are incredibly vague. + pass - if not should_colour or colour == NORMAL: + if not should_colour: stream.write(message + "\n") else: stream.write(colour + message + NORMAL + "\n") From 0fd2321629f4a06bec8b552299d3ced32a7a021c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 16 Oct 2018 20:37:16 +0100 Subject: [PATCH 17/22] Fix incorrect truncation in get_missing_events It's quite important that get_missing_events returns the *latest* events in the room; however we were pulling event ids out of the database until we got *at least* 10, and then taking the *earliest* of the results. We also shouldn't really be relying on depth, and should be checking the room_id. --- changelog.d/4045.bugfix | 1 + synapse/federation/federation_server.py | 8 +++--- synapse/federation/transport/server.py | 2 -- synapse/handlers/federation.py | 12 ++++---- synapse/storage/event_federation.py | 38 +++++++++++-------------- 5 files changed, 26 insertions(+), 35 deletions(-) create mode 100644 changelog.d/4045.bugfix diff --git a/changelog.d/4045.bugfix b/changelog.d/4045.bugfix new file mode 100644 index 0000000000..fa50eb5aff --- /dev/null +++ b/changelog.d/4045.bugfix @@ -0,0 +1 @@ +Fix bug which made get_missing_events return too few events \ No newline at end of file diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 819e8f7331..4efe95faa4 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -507,19 +507,19 @@ class FederationServer(FederationBase): @defer.inlineCallbacks @log_function def on_get_missing_events(self, origin, room_id, earliest_events, - latest_events, limit, min_depth): + latest_events, limit): with (yield self._server_linearizer.queue((origin, room_id))): origin_host, _ = parse_server_name(origin) yield self.check_server_matches_acl(origin_host, room_id) logger.info( "on_get_missing_events: earliest_events: %r, latest_events: %r," - " limit: %d, min_depth: %d", - earliest_events, latest_events, limit, min_depth + " limit: %d", + earliest_events, latest_events, limit, ) missing_events = yield self.handler.on_get_missing_events( - origin, room_id, earliest_events, latest_events, limit, min_depth + origin, room_id, earliest_events, latest_events, limit, ) if len(missing_events) < 5: diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 2f874b4838..7288d49074 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -560,7 +560,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet): @defer.inlineCallbacks def on_POST(self, origin, content, query, room_id): limit = int(content.get("limit", 10)) - min_depth = int(content.get("min_depth", 0)) earliest_events = content.get("earliest_events", []) latest_events = content.get("latest_events", []) @@ -569,7 +568,6 @@ class FederationGetMissingEventsServlet(BaseFederationServlet): room_id=room_id, earliest_events=earliest_events, latest_events=latest_events, - min_depth=min_depth, limit=limit, ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 45d955e6f5..cab57a8849 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -309,8 +309,8 @@ class FederationHandler(BaseHandler): if sent_to_us_directly: logger.warn( - "[%s %s] Failed to fetch %d prev events: rejecting", - room_id, event_id, len(prevs - seen), + "[%s %s] Rejecting: failed to fetch %d prev events: %s", + room_id, event_id, len(prevs - seen), shortstr(prevs - seen) ) raise FederationError( "ERROR", @@ -452,8 +452,8 @@ class FederationHandler(BaseHandler): latest |= seen logger.info( - "[%s %s]: Requesting %d prev_events: %s", - room_id, event_id, len(prevs - seen), shortstr(prevs - seen) + "[%s %s]: Requesting missing events between %s and %s", + room_id, event_id, shortstr(latest), event_id, ) # XXX: we set timeout to 10s to help workaround @@ -1852,7 +1852,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def on_get_missing_events(self, origin, room_id, earliest_events, - latest_events, limit, min_depth): + latest_events, limit): in_room = yield self.auth.check_host_in_room( room_id, origin @@ -1861,14 +1861,12 @@ class FederationHandler(BaseHandler): raise AuthError(403, "Host not in room.") limit = min(limit, 20) - min_depth = max(min_depth, 0) missing_events = yield self.store.get_missing_events( room_id=room_id, earliest_events=earliest_events, latest_events=latest_events, limit=limit, - min_depth=min_depth, ) missing_events = yield filter_events_for_server( diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 24345b20a6..3faca2a042 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -376,33 +376,25 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, @defer.inlineCallbacks def get_missing_events(self, room_id, earliest_events, latest_events, - limit, min_depth): + limit): ids = yield self.runInteraction( "get_missing_events", self._get_missing_events, - room_id, earliest_events, latest_events, limit, min_depth + room_id, earliest_events, latest_events, limit, ) - events = yield self._get_events(ids) - - events = sorted( - [ev for ev in events if ev.depth >= min_depth], - key=lambda e: e.depth, - ) - - defer.returnValue(events[:limit]) + defer.returnValue(events) def _get_missing_events(self, txn, room_id, earliest_events, latest_events, - limit, min_depth): + limit): - earliest_events = set(earliest_events) - front = set(latest_events) - earliest_events - - event_results = set() + seen_events = set(earliest_events) + front = set(latest_events) - seen_events + event_results = [] query = ( "SELECT prev_event_id FROM event_edges " - "WHERE event_id = ? AND is_state = ? " + "WHERE room_id = ? AND event_id = ? AND is_state = ? " "LIMIT ?" ) @@ -411,18 +403,20 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, for event_id in front: txn.execute( query, - (event_id, False, limit - len(event_results)) + (room_id, event_id, False, limit - len(event_results)) ) - for e_id, in txn: - new_front.add(e_id) + new_results = set(t[0] for t in txn) - seen_events - new_front -= earliest_events - new_front -= event_results + new_front |= new_results + seen_events |= new_results + event_results.extend(new_results) front = new_front - event_results |= new_front + # we built the list working backwards from latest_events; we now need to + # reverse it so that the events are approximately chronological. + event_results.reverse() return event_results From 52e3d3813baf82dc16aae42bdd108cf296c3ead7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 17 Oct 2018 17:40:01 +0100 Subject: [PATCH 18/22] prep changelog --- CHANGES.md | 25 +++++++++++++++++++++++++ changelog.d/4045.bugfix | 1 - changelog.d/4052.feature | 12 ------------ 3 files changed, 25 insertions(+), 13 deletions(-) delete mode 100644 changelog.d/4045.bugfix delete mode 100644 changelog.d/4052.feature diff --git a/CHANGES.md b/CHANGES.md index 6c80a86461..2598bfc333 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,28 @@ +Synapse 0.33.7rc2 (2018-10-17) +============================== + +**Warning**: This release removes the example email notification templates from `res/templates` (they are now internal to the python package). This should only affect you if you (a) deploy your Synapse instance from a git checkout or a github snapshot URL, and (b) have email notifications enabled. + +If you have email notifications enabled, you should ensure that +`email.template_dir` is either configured to point at a directory where you +have installed customised templates, or leave it unset to use the default +templates. + +The configuration parser will try to detect the situation where +`email.template_dir` is incorrectly set to `res/templates` and do the right +thing, but will warn about this. + +Features +-------- + +- Ship the example email templates as part of the package ([\#4052](https://github.com/matrix-org/synapse/issues/4052)) + +Bugfixes +-------- + +- Fix bug which made get_missing_events return too few events ([\#4045](https://github.com/matrix-org/synapse/issues/4045)) + + Synapse 0.33.7rc1 (2018-10-15) ============================== diff --git a/changelog.d/4045.bugfix b/changelog.d/4045.bugfix deleted file mode 100644 index fa50eb5aff..0000000000 --- a/changelog.d/4045.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug which made get_missing_events return too few events \ No newline at end of file diff --git a/changelog.d/4052.feature b/changelog.d/4052.feature deleted file mode 100644 index 0c01f06880..0000000000 --- a/changelog.d/4052.feature +++ /dev/null @@ -1,12 +0,0 @@ -Ship the example email templates as part of the package - -**Note**: if you deploy your Synapse instance from a git checkout or a github -snapshot URL, then this means that the example email templates will no longer -be installed in `res/templates`. If you have email notifications enabled, you -should ensure that `email.template_dir` is either configured to point at a -directory where you have installed customised templates, or leave it unset to -use the default templates. - -The configuration parser will try to detect the situation where -`email.template_dir` is incorrectly set to `res/templates` and do the right -thing, but will warn about this. \ No newline at end of file From c7d0f34a3c0217be4ca06d934b66fcdfc22a7219 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 17 Oct 2018 17:40:19 +0100 Subject: [PATCH 19/22] v0.33.7rc2 --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index ea07fda14a..a0c4ef6488 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.33.7rc1" +__version__ = "0.33.7rc2" From 8c2b8d7f0bfb5043c5e22f09fa69f01b4656d3d2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 17 Oct 2018 17:42:12 +0100 Subject: [PATCH 20/22] fix changelog formatting --- CHANGES.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2598bfc333..7de66d6ab4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,10 @@ Synapse 0.33.7rc2 (2018-10-17) ============================== -**Warning**: This release removes the example email notification templates from `res/templates` (they are now internal to the python package). This should only affect you if you (a) deploy your Synapse instance from a git checkout or a github snapshot URL, and (b) have email notifications enabled. +**Warning**: This release removes the example email notification templates from +`res/templates` (they are now internal to the python package). This should only +affect you if you (a) deploy your Synapse instance from a git checkout or a +github snapshot URL, and (b) have email notifications enabled. If you have email notifications enabled, you should ensure that `email.template_dir` is either configured to point at a directory where you From 926da4dda84178b83fa00813d31f1bceef25171f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 18 Oct 2018 14:57:32 +0100 Subject: [PATCH 21/22] 0.33.7 --- CHANGES.md | 7 +++++-- synapse/__init__.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7de66d6ab4..5f598559a0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -Synapse 0.33.7rc2 (2018-10-17) -============================== +Synapse 0.33.7 (2018-10-18) +=========================== **Warning**: This release removes the example email notification templates from `res/templates` (they are now internal to the python package). This should only @@ -15,6 +15,9 @@ The configuration parser will try to detect the situation where `email.template_dir` is incorrectly set to `res/templates` and do the right thing, but will warn about this. +Synapse 0.33.7rc2 (2018-10-17) +============================== + Features -------- diff --git a/synapse/__init__.py b/synapse/__init__.py index a0c4ef6488..1ddbbbebfb 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.33.7rc2" +__version__ = "0.33.7" From 03287c350eec760b17a4739274d84ca8dd86143a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 18 Oct 2018 15:09:34 +0100 Subject: [PATCH 22/22] remove redundant changelog file this change has been released --- changelog.d/4045.bugfix | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changelog.d/4045.bugfix diff --git a/changelog.d/4045.bugfix b/changelog.d/4045.bugfix deleted file mode 100644 index fa50eb5aff..0000000000 --- a/changelog.d/4045.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug which made get_missing_events return too few events \ No newline at end of file