From 2ef3f84945c3a6ef653d6f589c06e2d8882cca05 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 10 Jul 2018 21:14:22 +0200 Subject: [PATCH 01/16] change http links to https Especially useful for the debian repo, as this makes it easier to get the key in a secure way --- README.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 4fe54b0c90..84b8cf572e 100644 --- a/README.rst +++ b/README.rst @@ -71,7 +71,7 @@ We'd like to invite you to join #matrix:matrix.org (via https://matrix.org/docs/projects/try-matrix-now.html), run a homeserver, take a look at the `Matrix spec `_, and experiment with the `APIs `_ and `Client SDKs -`_. +`_. Thanks for using Matrix! @@ -283,7 +283,7 @@ Connecting to Synapse from a client The easiest way to try out your new Synapse installation is by connecting to it from a web client. The easiest option is probably the one at -http://riot.im/app. You will need to specify a "Custom server" when you log on +https://riot.im/app. You will need to specify a "Custom server" when you log on or register: set this to ``https://domain.tld`` if you setup a reverse proxy following the recommended setup, or ``https://localhost:8448`` - remember to specify the port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity @@ -329,7 +329,7 @@ Security Note ============= Matrix serves raw user generated data in some APIs - specifically the `content -repository endpoints `_. +repository endpoints `_. Whilst we have tried to mitigate against possible XSS attacks (e.g. https://github.com/matrix-org/synapse/pull/1021) we recommend running @@ -348,7 +348,7 @@ Platform-Specific Instructions Debian ------ -Matrix provides official Debian packages via apt from http://matrix.org/packages/debian/. +Matrix provides official Debian packages via apt from https://matrix.org/packages/debian/. Note that these packages do not include a client - choose one from https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :) @@ -524,7 +524,7 @@ Troubleshooting Running ----------------------- If synapse fails with ``missing "sodium.h"`` crypto errors, you may need -to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for +to manually upgrade PyNaCL, as synapse uses NaCl (https://nacl.cr.yp.to/) for encryption and digital signatures. Unfortunately PyNACL currently has a few issues (https://github.com/pyca/pynacl/issues/53) and @@ -672,8 +672,8 @@ useful just for development purposes. See ``_. Using PostgreSQL ================ -As of Synapse 0.9, `PostgreSQL `_ is supported as an -alternative to the `SQLite `_ database that Synapse has +As of Synapse 0.9, `PostgreSQL `_ is supported as an +alternative to the `SQLite `_ database that Synapse has traditionally used for convenience and simplicity. The advantages of Postgres include: @@ -697,7 +697,7 @@ Using a reverse proxy with Synapse It is recommended to put a reverse proxy such as `nginx `_, `Apache `_ or -`HAProxy `_ in front of Synapse. One advantage of +`HAProxy `_ in front of Synapse. One advantage of doing so is that it means that you can expose the default https port (443) to Matrix clients without needing to run Synapse with root privileges. From 5f3d02f6ebbce8fbbb2a71ef1ea7f5537257ff15 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Jul 2018 12:52:56 +1000 Subject: [PATCH 02/16] bump to 0.33.0rc1 --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 3cde33c0d7..6dc16f0808 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -17,4 +17,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.32.2" +__version__ = "0.33.0rc1" From 21d3b879433e040babd43c89b62827f92e3ac861 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 18 Jul 2018 12:53:32 +1000 Subject: [PATCH 03/16] 0.33.0rc1 changelog --- CHANGES.rst | 27 +++++++++++++++++++++++++++ changelog.d/3316.feature | 1 - changelog.d/3351.misc | 0 changelog.d/3463.misc | 0 changelog.d/3464.misc | 0 changelog.d/3496.feature | 1 - changelog.d/3497.feature | 1 - changelog.d/3498.misc | 0 changelog.d/3499.misc | 0 changelog.d/3501.misc | 0 changelog.d/3505.feature | 1 - changelog.d/3521.feature | 1 - changelog.d/3530.misc | 0 changelog.d/3533.bugfix | 1 - changelog.d/3534.misc | 1 - changelog.d/3535.misc | 0 changelog.d/3540.misc | 1 - changelog.d/3541.feature | 1 - changelog.d/3544.misc | 0 changelog.d/3546.bugfix | 1 - 20 files changed, 27 insertions(+), 10 deletions(-) delete mode 100644 changelog.d/3316.feature delete mode 100644 changelog.d/3351.misc delete mode 100644 changelog.d/3463.misc delete mode 100644 changelog.d/3464.misc delete mode 100644 changelog.d/3496.feature delete mode 100644 changelog.d/3497.feature delete mode 100644 changelog.d/3498.misc delete mode 100644 changelog.d/3499.misc delete mode 100644 changelog.d/3501.misc delete mode 100644 changelog.d/3505.feature delete mode 100644 changelog.d/3521.feature delete mode 100644 changelog.d/3530.misc delete mode 100644 changelog.d/3533.bugfix delete mode 100644 changelog.d/3534.misc delete mode 100644 changelog.d/3535.misc delete mode 100644 changelog.d/3540.misc delete mode 100644 changelog.d/3541.feature delete mode 100644 changelog.d/3544.misc delete mode 100644 changelog.d/3546.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index ba6929c435..f81e7e6cf3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,30 @@ +Synapse 0.33.0rc1 (2018-07-18) +============================== + +Features +-------- + +- Enforce the specified API for report_event (`#3316 `_) +- Include CPU time from database threads in request/block metrics. (`#3496 `_) +- Add CPU metrics for _fetch_event_list (`#3497 `_) +- Reduce database consumption when processing large numbers of receipts (`#3505 `_) +- Cache optimisation for /sync requests (`#3521 `_) +- Optimisation to make handling incoming federation requests more efficient. (`#3541 `_) + + +Bugfixes +-------- + +- Fix queued federation requests being processed in the wrong order (`#3533 `_) +- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 `_) + + +Misc +---- + +- `#3351 `_, `#3463 `_, `#3464 `_, `#3498 `_, `#3499 `_, `#3501 `_, `#3530 `_, `#3534 `_, `#3535 `_, `#3540 `_, `#3544 `_ + + Synapse 0.32.2 (2018-07-07) =========================== diff --git a/changelog.d/3316.feature b/changelog.d/3316.feature deleted file mode 100644 index 50068b7222..0000000000 --- a/changelog.d/3316.feature +++ /dev/null @@ -1 +0,0 @@ -Enforce the specified API for report_event diff --git a/changelog.d/3351.misc b/changelog.d/3351.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3463.misc b/changelog.d/3463.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3464.misc b/changelog.d/3464.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3496.feature b/changelog.d/3496.feature deleted file mode 100644 index 6a06a7e755..0000000000 --- a/changelog.d/3496.feature +++ /dev/null @@ -1 +0,0 @@ -Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3497.feature b/changelog.d/3497.feature deleted file mode 100644 index 4e333155b6..0000000000 --- a/changelog.d/3497.feature +++ /dev/null @@ -1 +0,0 @@ -Add CPU metrics for _fetch_event_list diff --git a/changelog.d/3498.misc b/changelog.d/3498.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3499.misc b/changelog.d/3499.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3501.misc b/changelog.d/3501.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3505.feature b/changelog.d/3505.feature deleted file mode 100644 index ca1867f529..0000000000 --- a/changelog.d/3505.feature +++ /dev/null @@ -1 +0,0 @@ -Reduce database consumption when processing large numbers of receipts diff --git a/changelog.d/3521.feature b/changelog.d/3521.feature deleted file mode 100644 index 6dced5f2ae..0000000000 --- a/changelog.d/3521.feature +++ /dev/null @@ -1 +0,0 @@ -Cache optimisation for /sync requests \ No newline at end of file diff --git a/changelog.d/3530.misc b/changelog.d/3530.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3533.bugfix b/changelog.d/3533.bugfix deleted file mode 100644 index 04cbbefd5f..0000000000 --- a/changelog.d/3533.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix queued federation requests being processed in the wrong order diff --git a/changelog.d/3534.misc b/changelog.d/3534.misc deleted file mode 100644 index 949c12dc69..0000000000 --- a/changelog.d/3534.misc +++ /dev/null @@ -1 +0,0 @@ -refactor: use parse_{string,integer} and assert's from http.servlet for deduplication diff --git a/changelog.d/3535.misc b/changelog.d/3535.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3540.misc b/changelog.d/3540.misc deleted file mode 100644 index 99dcad8e46..0000000000 --- a/changelog.d/3540.misc +++ /dev/null @@ -1 +0,0 @@ -check isort for each PR diff --git a/changelog.d/3541.feature b/changelog.d/3541.feature deleted file mode 100644 index 24524136ea..0000000000 --- a/changelog.d/3541.feature +++ /dev/null @@ -1 +0,0 @@ -Optimisation to make handling incoming federation requests more efficient. \ No newline at end of file diff --git a/changelog.d/3544.misc b/changelog.d/3544.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3546.bugfix b/changelog.d/3546.bugfix deleted file mode 100644 index 921dc6e7b0..0000000000 --- a/changelog.d/3546.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. \ No newline at end of file From 667fba68f3ca808f48143a2a739a54665b0162c6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 14:35:24 +0100 Subject: [PATCH 04/16] Run things as background processes This fixes #3518, and ensures that we get useful logs and metrics for lots of things that happen in the background. (There are certainly more things that happen in the background; these are just the common ones I've found running a single-process synapse locally). --- synapse/federation/transaction_queue.py | 15 ++++++--------- synapse/storage/background_updates.py | 10 ++++++++-- synapse/storage/client_ips.py | 15 +++++++++++---- synapse/storage/events.py | 10 ++++------ synapse/storage/events_worker.py | 10 ++++++---- synapse/util/caches/expiringcache.py | 6 +++++- synapse/util/distributor.py | 4 ++++ 7 files changed, 44 insertions(+), 26 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 5c5a73b73c..6996d6b695 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -168,7 +168,7 @@ class TransactionQueue(object): # fire off a processing loop in the background run_as_background_process( - "process_transaction_queue", + "process_event_queue_for_federation", self._process_event_queue_loop, ) @@ -434,14 +434,11 @@ class TransactionQueue(object): logger.debug("TX [%s] Starting transaction loop", destination) - # Drop the logcontext before starting the transaction. It doesn't - # really make sense to log all the outbound transactions against - # whatever path led us to this point: that's pretty arbitrary really. - # - # (this also means we can fire off _perform_transaction without - # yielding) - with logcontext.PreserveLoggingContext(): - self._transaction_transmission_loop(destination) + run_as_background_process( + "federation_transaction_transmission_loop", + self._transaction_transmission_loop, + destination, + ) @defer.inlineCallbacks def _transaction_transmission_loop(self, destination): diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index dc9eca7d15..5fe1ca2de7 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -19,6 +19,8 @@ from canonicaljson import json from twisted.internet import defer +from synapse.metrics.background_process_metrics import run_as_background_process + from . import engines from ._base import SQLBaseStore @@ -87,10 +89,14 @@ class BackgroundUpdateStore(SQLBaseStore): self._background_update_handlers = {} self._all_done = False - @defer.inlineCallbacks def start_doing_background_updates(self): - logger.info("Starting background schema updates") + run_as_background_process( + "background_updates", self._run_background_updates, + ) + @defer.inlineCallbacks + def _run_background_updates(self): + logger.info("Starting background schema updates") while True: yield self.hs.get_clock().sleep( self.BACKGROUND_UPDATE_INTERVAL_MS / 1000.) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index b78eda3413..77ae10da3d 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -19,6 +19,7 @@ from six import iteritems from twisted.internet import defer +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.caches import CACHE_SIZE_FACTOR from . import background_updates @@ -93,10 +94,16 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): self._batch_row_update[key] = (user_agent, device_id, now) def _update_client_ips_batch(self): - to_update = self._batch_row_update - self._batch_row_update = {} - return self.runInteraction( - "_update_client_ips_batch", self._update_client_ips_batch_txn, to_update + def update(): + to_update = self._batch_row_update + self._batch_row_update = {} + return self.runInteraction( + "_update_client_ips_batch", self._update_client_ips_batch_txn, + to_update, + ) + + run_as_background_process( + "update_client_ips", update, ) def _update_client_ips_batch_txn(self, txn, to_update): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2aaab0d02c..4ff0fdc4ab 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -33,12 +33,13 @@ from synapse.api.errors import SynapseError # these are only included to make the type annotations work from synapse.events import EventBase # noqa: F401 from synapse.events.snapshot import EventContext # noqa: F401 +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.events_worker import EventsWorkerStore from synapse.types import RoomStreamToken, get_domain_from_id from synapse.util.async import ObservableDeferred from synapse.util.caches.descriptors import cached, cachedInlineCallbacks from synapse.util.frozenutils import frozendict_json_encoder -from synapse.util.logcontext import PreserveLoggingContext, make_deferred_yieldable +from synapse.util.logcontext import make_deferred_yieldable from synapse.util.logutils import log_function from synapse.util.metrics import Measure @@ -155,11 +156,8 @@ class _EventPeristenceQueue(object): self._event_persist_queues[room_id] = queue self._currently_persisting_rooms.discard(room_id) - # set handle_queue_loop off on the background. We don't want to - # attribute work done in it to the current request, so we drop the - # logcontext altogether. - with PreserveLoggingContext(): - handle_queue_loop() + # set handle_queue_loop off in the background + run_as_background_process("persist_events", handle_queue_loop) def _get_drainining_queue(self, room_id): queue = self._event_persist_queues.setdefault(room_id, deque()) diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index 67433606c6..f28239a808 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -25,6 +25,7 @@ from synapse.events import EventBase # noqa: F401 from synapse.events import FrozenEvent from synapse.events.snapshot import EventContext # noqa: F401 from synapse.events.utils import prune_event +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.logcontext import ( LoggingContext, PreserveLoggingContext, @@ -322,10 +323,11 @@ class EventsWorkerStore(SQLBaseStore): should_start = False if should_start: - with PreserveLoggingContext(): - self.runWithConnection( - self._do_fetch - ) + run_as_background_process( + "fetch_events", + self.runWithConnection, + self._do_fetch, + ) logger.debug("Loading %d events", len(events)) with PreserveLoggingContext(): diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 4abca91f6d..465adc54a8 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -16,6 +16,7 @@ import logging from collections import OrderedDict +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.caches import register_cache logger = logging.getLogger(__name__) @@ -63,7 +64,10 @@ class ExpiringCache(object): return def f(): - self._prune_cache() + run_as_background_process( + "prune_cache_%s" % self._cache_name, + self._prune_cache, + ) self._clock.looping_call(f, self._expiry_ms / 2) diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index 734331caaa..d91ae400eb 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -75,6 +75,10 @@ class Distributor(object): self.pre_registration[name].append(observer) def fire(self, name, *args, **kwargs): + """Dispatches the given signal to the registered observers. + + Runs the observers as a background process. Does not return a deferred. + """ if name not in self.signals: raise KeyError("%r does not have a signal named %s" % (self, name)) From 08436c556a6dd6053ce778d581159f82f8668a63 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 16:14:01 +0100 Subject: [PATCH 05/16] changelog --- changelog.d/3556.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3556.feature diff --git a/changelog.d/3556.feature b/changelog.d/3556.feature new file mode 100644 index 0000000000..77a294cb9f --- /dev/null +++ b/changelog.d/3556.feature @@ -0,0 +1 @@ +Add metrics to track resource usage by background processes From 8c69b735e3186875a8bf676241f990470fb7c592 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Jul 2018 15:33:13 +0100 Subject: [PATCH 06/16] Make Distributor run its processes as a background process This is more involved than it might otherwise be, because the current implementation just drops its logcontexts and runs everything in the sentinel context. It turns out that we aren't actually using a bunch of the functionality here (notably suppress_failures and the fact that Distributor.fire returns a deferred), so the easiest way to fix this is actually by simplifying a bunch of code. --- synapse/util/distributor.py | 44 ++++++++++++----------------- tests/test_distributor.py | 56 +++---------------------------------- 2 files changed, 22 insertions(+), 78 deletions(-) diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index d91ae400eb..194da87639 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -17,20 +17,18 @@ import logging from twisted.internet import defer -from synapse.util import unwrapFirstError -from synapse.util.logcontext import PreserveLoggingContext +from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.util.logcontext import make_deferred_yieldable, run_in_background logger = logging.getLogger(__name__) def user_left_room(distributor, user, room_id): - with PreserveLoggingContext(): - distributor.fire("user_left_room", user=user, room_id=room_id) + distributor.fire("user_left_room", user=user, room_id=room_id) def user_joined_room(distributor, user, room_id): - with PreserveLoggingContext(): - distributor.fire("user_joined_room", user=user, room_id=room_id) + distributor.fire("user_joined_room", user=user, room_id=room_id) class Distributor(object): @@ -44,9 +42,7 @@ class Distributor(object): model will do for today. """ - def __init__(self, suppress_failures=True): - self.suppress_failures = suppress_failures - + def __init__(self): self.signals = {} self.pre_registration = {} @@ -56,7 +52,6 @@ class Distributor(object): self.signals[name] = Signal( name, - suppress_failures=self.suppress_failures, ) if name in self.pre_registration: @@ -82,7 +77,11 @@ class Distributor(object): if name not in self.signals: raise KeyError("%r does not have a signal named %s" % (self, name)) - return self.signals[name].fire(*args, **kwargs) + run_as_background_process( + name, + self.signals[name].fire, + *args, **kwargs + ) class Signal(object): @@ -95,9 +94,8 @@ class Signal(object): method into all of the observers. """ - def __init__(self, name, suppress_failures): + def __init__(self, name): self.name = name - self.suppress_failures = suppress_failures self.observers = [] def observe(self, observer): @@ -107,7 +105,6 @@ class Signal(object): Each observer callable may return a Deferred.""" self.observers.append(observer) - @defer.inlineCallbacks def fire(self, *args, **kwargs): """Invokes every callable in the observer list, passing in the args and kwargs. Exceptions thrown by observers are logged but ignored. It is @@ -125,22 +122,17 @@ class Signal(object): failure.type, failure.value, failure.getTracebackObject())) - if not self.suppress_failures: - return failure return defer.maybeDeferred(observer, *args, **kwargs).addErrback(eb) - with PreserveLoggingContext(): - deferreds = [ - do(observer) - for observer in self.observers - ] + deferreds = [ + run_in_background(do, o) + for o in self.observers + ] - res = yield defer.gatherResults( - deferreds, consumeErrors=True - ).addErrback(unwrapFirstError) - - defer.returnValue(res) + return make_deferred_yieldable(defer.gatherResults( + deferreds, consumeErrors=True, + )) def __repr__(self): return "" % (self.name,) diff --git a/tests/test_distributor.py b/tests/test_distributor.py index 04a88056f1..71d11cda77 100644 --- a/tests/test_distributor.py +++ b/tests/test_distributor.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +16,6 @@ from mock import Mock, patch -from twisted.internet import defer - from synapse.util.distributor import Distributor from . import unittest @@ -27,38 +26,15 @@ class DistributorTestCase(unittest.TestCase): def setUp(self): self.dist = Distributor() - @defer.inlineCallbacks def test_signal_dispatch(self): self.dist.declare("alert") observer = Mock() self.dist.observe("alert", observer) - d = self.dist.fire("alert", 1, 2, 3) - yield d - self.assertTrue(d.called) + self.dist.fire("alert", 1, 2, 3) observer.assert_called_with(1, 2, 3) - @defer.inlineCallbacks - def test_signal_dispatch_deferred(self): - self.dist.declare("whine") - - d_inner = defer.Deferred() - - def observer(): - return d_inner - - self.dist.observe("whine", observer) - - d_outer = self.dist.fire("whine") - - self.assertFalse(d_outer.called) - - d_inner.callback(None) - yield d_outer - self.assertTrue(d_outer.called) - - @defer.inlineCallbacks def test_signal_catch(self): self.dist.declare("alarm") @@ -71,9 +47,7 @@ class DistributorTestCase(unittest.TestCase): with patch( "synapse.util.distributor.logger", spec=["warning"] ) as mock_logger: - d = self.dist.fire("alarm", "Go") - yield d - self.assertTrue(d.called) + self.dist.fire("alarm", "Go") observers[0].assert_called_once_with("Go") observers[1].assert_called_once_with("Go") @@ -83,34 +57,12 @@ class DistributorTestCase(unittest.TestCase): mock_logger.warning.call_args[0][0], str ) - @defer.inlineCallbacks - def test_signal_catch_no_suppress(self): - # Gut-wrenching - self.dist.suppress_failures = False - - self.dist.declare("whail") - - class MyException(Exception): - pass - - @defer.inlineCallbacks - def observer(): - raise MyException("Oopsie") - - self.dist.observe("whail", observer) - - d = self.dist.fire("whail") - - yield self.assertFailure(d, MyException) - self.dist.suppress_failures = True - - @defer.inlineCallbacks def test_signal_prereg(self): observer = Mock() self.dist.observe("flare", observer) self.dist.declare("flare") - yield self.dist.fire("flare", 4, 5) + self.dist.fire("flare", 4, 5) observer.assert_called_with(4, 5) From 00bc9791379109690fb5e465adf0e0e8e76167aa Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 19 Jul 2018 10:51:15 +0100 Subject: [PATCH 07/16] Disable logcontext warning Temporary workaround to #3518 while we release 0.33.0. --- synapse/storage/_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 98dde77431..a6a0e6ec9f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -343,9 +343,10 @@ class SQLBaseStore(object): """ parent_context = LoggingContext.current_context() if parent_context == LoggingContext.sentinel: - logger.warn( - "Running db txn from sentinel context: metrics will be lost", - ) + # warning disabled for 0.33.0 release; proper fixes will land imminently. + # logger.warn( + # "Running db txn from sentinel context: metrics will be lost", + # ) parent_context = None start_time = time.time() From 18a2b2c0b402af89f818a50ca2f4f0c733f864e4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 19 Jul 2018 10:54:39 +0100 Subject: [PATCH 08/16] changelog --- changelog.d/3561.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3561.bugfix diff --git a/changelog.d/3561.bugfix b/changelog.d/3561.bugfix new file mode 100644 index 0000000000..9a197cd67f --- /dev/null +++ b/changelog.d/3561.bugfix @@ -0,0 +1 @@ +Disable a noisy warning about logcontexts From a97c845271f9a89ebdb7186d4c9d04c099bd1beb Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 20:03:33 +1000 Subject: [PATCH 09/16] Move v1-only APIs into their own module & isolate deprecated ones (#3460) --- changelog.d/3460.misc | 0 setup.cfg | 1 + synapse/http/client.py | 6 +- synapse/rest/__init__.py | 43 +++++++-- synapse/rest/client/v1/room.py | 5 +- synapse/rest/client/v1_only/__init__.py | 3 + synapse/rest/client/v1_only/base.py | 39 ++++++++ .../rest/client/{v1 => v1_only}/register.py | 7 +- tests/rest/client/v1/test_events.py | 90 +++---------------- tests/rest/client/v1/test_register.py | 5 +- tests/rest/client/v1/test_rooms.py | 2 +- tests/rest/client/v2_alpha/test_filter.py | 8 +- tests/rest/client/v2_alpha/test_sync.py | 8 +- 13 files changed, 116 insertions(+), 101 deletions(-) create mode 100644 changelog.d/3460.misc create mode 100644 synapse/rest/client/v1_only/__init__.py create mode 100644 synapse/rest/client/v1_only/base.py rename synapse/rest/client/{v1 => v1_only}/register.py (98%) diff --git a/changelog.d/3460.misc b/changelog.d/3460.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/setup.cfg b/setup.cfg index 3f84283a38..c2620be6c5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,3 +36,4 @@ known_compat = mock,six known_twisted=twisted,OpenSSL multi_line_output=3 include_trailing_comma=true +combine_as_imports=true diff --git a/synapse/http/client.py b/synapse/http/client.py index d6a0d75b2b..25b6307884 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -26,9 +26,11 @@ from OpenSSL.SSL import VERIFY_NONE from twisted.internet import defer, protocol, reactor, ssl, task from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.web._newclient import ResponseDone -from twisted.web.client import Agent, BrowserLikeRedirectAgent, ContentDecoderAgent -from twisted.web.client import FileBodyProducer as TwistedFileBodyProducer from twisted.web.client import ( + Agent, + BrowserLikeRedirectAgent, + ContentDecoderAgent, + FileBodyProducer as TwistedFileBodyProducer, GzipDecoder, HTTPConnectionPool, PartialDownloadError, diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 75c2a4ec8e..3418f06fd6 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,13 +14,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +from six import PY3 + from synapse.http.server import JsonResource from synapse.rest.client import versions -from synapse.rest.client.v1 import admin, directory, events, initial_sync -from synapse.rest.client.v1 import login as v1_login -from synapse.rest.client.v1 import logout, presence, profile, push_rule, pusher -from synapse.rest.client.v1 import register as v1_register -from synapse.rest.client.v1 import room, voip +from synapse.rest.client.v1 import ( + admin, + directory, + events, + initial_sync, + login as v1_login, + logout, + presence, + profile, + push_rule, + pusher, + room, + voip, +) from synapse.rest.client.v2_alpha import ( account, account_data, @@ -42,6 +54,11 @@ from synapse.rest.client.v2_alpha import ( user_directory, ) +if not PY3: + from synapse.rest.client.v1_only import ( + register as v1_register, + ) + class ClientRestResource(JsonResource): """A resource for version 1 of the matrix client API.""" @@ -54,14 +71,22 @@ class ClientRestResource(JsonResource): def register_servlets(client_resource, hs): versions.register_servlets(client_resource) - # "v1" - room.register_servlets(hs, client_resource) + if not PY3: + # "v1" (Python 2 only) + v1_register.register_servlets(hs, client_resource) + + # Deprecated in r0 + initial_sync.register_servlets(hs, client_resource) + room.register_deprecated_servlets(hs, client_resource) + + # Partially deprecated in r0 events.register_servlets(hs, client_resource) - v1_register.register_servlets(hs, client_resource) + + # "v1" + "r0" + room.register_servlets(hs, client_resource) v1_login.register_servlets(hs, client_resource) profile.register_servlets(hs, client_resource) presence.register_servlets(hs, client_resource) - initial_sync.register_servlets(hs, client_resource) directory.register_servlets(hs, client_resource) voip.register_servlets(hs, client_resource) admin.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 3d62447854..b9512a2b61 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -832,10 +832,13 @@ def register_servlets(hs, http_server): RoomSendEventRestServlet(hs).register(http_server) PublicRoomListRestServlet(hs).register(http_server) RoomStateRestServlet(hs).register(http_server) - RoomInitialSyncRestServlet(hs).register(http_server) RoomRedactEventRestServlet(hs).register(http_server) RoomTypingRestServlet(hs).register(http_server) SearchRestServlet(hs).register(http_server) JoinedRoomsRestServlet(hs).register(http_server) RoomEventServlet(hs).register(http_server) RoomEventContextServlet(hs).register(http_server) + + +def register_deprecated_servlets(hs, http_server): + RoomInitialSyncRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v1_only/__init__.py b/synapse/rest/client/v1_only/__init__.py new file mode 100644 index 0000000000..936f902ace --- /dev/null +++ b/synapse/rest/client/v1_only/__init__.py @@ -0,0 +1,3 @@ +""" +REST APIs that are only used in v1 (the legacy API). +""" diff --git a/synapse/rest/client/v1_only/base.py b/synapse/rest/client/v1_only/base.py new file mode 100644 index 0000000000..9d4db7437c --- /dev/null +++ b/synapse/rest/client/v1_only/base.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains base REST classes for constructing client v1 servlets. +""" + +import re + +from synapse.api.urls import CLIENT_PREFIX + + +def v1_only_client_path_patterns(path_regex, include_in_unstable=True): + """Creates a regex compiled client path with the correct client path + prefix. + + Args: + path_regex (str): The regex string to match. This should NOT have a ^ + as this will be prefixed. + Returns: + list of SRE_Pattern + """ + patterns = [re.compile("^" + CLIENT_PREFIX + path_regex)] + if include_in_unstable: + unstable_prefix = CLIENT_PREFIX.replace("/api/v1", "/unstable") + patterns.append(re.compile("^" + unstable_prefix + path_regex)) + return patterns diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1_only/register.py similarity index 98% rename from synapse/rest/client/v1/register.py rename to synapse/rest/client/v1_only/register.py index 25a143af8d..3439c3c6d4 100644 --- a/synapse/rest/client/v1/register.py +++ b/synapse/rest/client/v1_only/register.py @@ -24,9 +24,10 @@ import synapse.util.stringutils as stringutils from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError from synapse.http.servlet import assert_params_in_dict, parse_json_object_from_request +from synapse.rest.client.v1.base import ClientV1RestServlet from synapse.types import create_requester -from .base import ClientV1RestServlet, client_path_patterns +from .base import v1_only_client_path_patterns logger = logging.getLogger(__name__) @@ -49,7 +50,7 @@ class RegisterRestServlet(ClientV1RestServlet): handler doesn't have a concept of multi-stages or sessions. """ - PATTERNS = client_path_patterns("/register$", releases=(), include_in_unstable=False) + PATTERNS = v1_only_client_path_patterns("/register$", include_in_unstable=False) def __init__(self, hs): """ @@ -379,7 +380,7 @@ class CreateUserRestServlet(ClientV1RestServlet): """Handles user creation via a server-to-server interface """ - PATTERNS = client_path_patterns("/createUser$", releases=()) + PATTERNS = v1_only_client_path_patterns("/createUser$") def __init__(self, hs): super(CreateUserRestServlet, self).__init__(hs) diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index a5af36a99c..50418153fa 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -14,100 +14,30 @@ # limitations under the License. """ Tests REST events for /events paths.""" + from mock import Mock, NonCallableMock +from six import PY3 -# twisted imports from twisted.internet import defer -import synapse.rest.client.v1.events -import synapse.rest.client.v1.register -import synapse.rest.client.v1.room - -from tests import unittest - from ....utils import MockHttpResource, setup_test_homeserver from .utils import RestTestCase PATH_PREFIX = "/_matrix/client/api/v1" -class EventStreamPaginationApiTestCase(unittest.TestCase): - """ Tests event streaming query parameters and start/end keys used in the - Pagination stream API. """ - user_id = "sid1" - - def setUp(self): - # configure stream and inject items - pass - - def tearDown(self): - pass - - def TODO_test_long_poll(self): - # stream from 'end' key, send (self+other) message, expect message. - - # stream from 'END', send (self+other) message, expect message. - - # stream from 'end' key, send (self+other) topic, expect topic. - - # stream from 'END', send (self+other) topic, expect topic. - - # stream from 'end' key, send (self+other) invite, expect invite. - - # stream from 'END', send (self+other) invite, expect invite. - - pass - - def TODO_test_stream_forward(self): - # stream from START, expect injected items - - # stream from 'start' key, expect same content - - # stream from 'end' key, expect nothing - - # stream from 'END', expect nothing - - # The following is needed for cases where content is removed e.g. you - # left a room, so the token you're streaming from is > the one that - # would be returned naturally from START>END. - # stream from very new token (higher than end key), expect same token - # returned as end key - pass - - def TODO_test_limits(self): - # stream from a key, expect limit_num items - - # stream from START, expect limit_num items - - pass - - def TODO_test_range(self): - # stream from key to key, expect X items - - # stream from key to END, expect X items - - # stream from START to key, expect X items - - # stream from START to END, expect all items - pass - - def TODO_test_direction(self): - # stream from END to START and fwds, expect newest first - - # stream from END to START and bwds, expect oldest first - - # stream from START to END and fwds, expect oldest first - - # stream from START to END and bwds, expect newest first - - pass - - class EventStreamPermissionsTestCase(RestTestCase): """ Tests event streaming (GET /events). """ + if PY3: + skip = "Skip on Py3 until ported to use not V1 only register." + @defer.inlineCallbacks def setUp(self): + import synapse.rest.client.v1.events + import synapse.rest.client.v1_only.register + import synapse.rest.client.v1.room + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) hs = yield setup_test_homeserver( @@ -125,7 +55,7 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource) + synapse.rest.client.v1_only.register.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py index f15fb36213..83a23cd8fe 100644 --- a/tests/rest/client/v1/test_register.py +++ b/tests/rest/client/v1/test_register.py @@ -16,11 +16,12 @@ import json from mock import Mock +from six import PY3 from twisted.test.proto_helpers import MemoryReactorClock from synapse.http.server import JsonResource -from synapse.rest.client.v1.register import register_servlets +from synapse.rest.client.v1_only.register import register_servlets from synapse.util import Clock from tests import unittest @@ -31,6 +32,8 @@ class CreateUserServletTestCase(unittest.TestCase): """ Tests for CreateUserRestServlet. """ + if PY3: + skip = "Not ported to Python 3." def setUp(self): self.registration_handler = Mock() diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 6b5764095e..00fc796787 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -20,7 +20,6 @@ import json from mock import Mock, NonCallableMock from six.moves.urllib import parse as urlparse -# twisted imports from twisted.internet import defer import synapse.rest.client.v1.room @@ -86,6 +85,7 @@ class RoomBase(unittest.TestCase): self.resource = JsonResource(self.hs) synapse.rest.client.v1.room.register_servlets(self.hs, self.resource) + synapse.rest.client.v1.room.register_deprecated_servlets(self.hs, self.resource) self.helper = RestHelper(self.hs, self.resource, self.user_id) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 5ea9cc825f..e890f0feac 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -21,8 +21,12 @@ from synapse.types import UserID from synapse.util import Clock from tests import unittest -from tests.server import ThreadedMemoryReactorClock as MemoryReactorClock -from tests.server import make_request, setup_test_homeserver, wait_until_result +from tests.server import ( + ThreadedMemoryReactorClock as MemoryReactorClock, + make_request, + setup_test_homeserver, + wait_until_result, +) PATH_PREFIX = "/_matrix/client/v2_alpha" diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 704cf97a40..03ec3993b2 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -20,8 +20,12 @@ from synapse.types import UserID from synapse.util import Clock from tests import unittest -from tests.server import ThreadedMemoryReactorClock as MemoryReactorClock -from tests.server import make_request, setup_test_homeserver, wait_until_result +from tests.server import ( + ThreadedMemoryReactorClock as MemoryReactorClock, + make_request, + setup_test_homeserver, + wait_until_result, +) PATH_PREFIX = "/_matrix/client/v2_alpha" From f1a15ea20692e9f4b347cdddf6d714912356ba64 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 19 Jul 2018 11:14:20 +0100 Subject: [PATCH 10/16] revert 00bc979 ... we've fixed the things that caused the warnings, so we should reinstate the warning. --- synapse/storage/_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a6a0e6ec9f..1d41d8d445 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -343,10 +343,9 @@ class SQLBaseStore(object): """ parent_context = LoggingContext.current_context() if parent_context == LoggingContext.sentinel: - # warning disabled for 0.33.0 release; proper fixes will land imminently. - # logger.warn( - # "Running db txn from sentinel context: metrics will be lost", - # ) + logger.warn( + "Starting db connection from sentinel context: metrics will be lost", + ) parent_context = None start_time = time.time() From 95ccb6e2ec57f2150a697ea9cde030e8f78d6db9 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 20:58:18 +1000 Subject: [PATCH 11/16] Don't spew errors because we can't save metrics (#3563) --- changelog.d/3548.bugfix | 1 + synapse/util/logcontext.py | 11 +++++++++++ synapse/util/metrics.py | 19 +++++++++++++------ 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 changelog.d/3548.bugfix diff --git a/changelog.d/3548.bugfix b/changelog.d/3548.bugfix new file mode 100644 index 0000000000..38dc3b1232 --- /dev/null +++ b/changelog.d/3548.bugfix @@ -0,0 +1 @@ +Catch failures saving metrics captured by Measure, and instead log the faulty metrics information for further analysis. diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index f6c7175f74..8dcae50b39 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -99,6 +99,17 @@ class ContextResourceUsage(object): self.db_sched_duration_sec = 0 self.evt_db_fetch_count = 0 + def __repr__(self): + return ("") % ( + self.ru_stime, + self.ru_utime, + self.db_txn_count, + self.db_txn_duration_sec, + self.db_sched_duration_sec, + self.evt_db_fetch_count,) + def __iadd__(self, other): """Add another ContextResourceUsage's stats to this one's. diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 6ba7107896..97f1267380 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -104,12 +104,19 @@ class Measure(object): logger.warn("Expected context. (%r)", self.name) return - usage = context.get_resource_usage() - self.start_usage - block_ru_utime.labels(self.name).inc(usage.ru_utime) - block_ru_stime.labels(self.name).inc(usage.ru_stime) - block_db_txn_count.labels(self.name).inc(usage.db_txn_count) - block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec) - block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec) + current = context.get_resource_usage() + usage = current - self.start_usage + try: + block_ru_utime.labels(self.name).inc(usage.ru_utime) + block_ru_stime.labels(self.name).inc(usage.ru_stime) + block_db_txn_count.labels(self.name).inc(usage.db_txn_count) + block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec) + block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec) + except ValueError: + logger.warn( + "Failed to save metrics! OLD: %r, NEW: %r", + self.start_usage, current + ) if self.created_context: self.start_context.__exit__(exc_type, exc_val, exc_tb) From ce0545eca1daa9f0f50335b644d53ba2ec965a08 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:03:15 +1000 Subject: [PATCH 12/16] Revert "0.33.0rc1 changelog" This reverts commit 21d3b879433e040babd43c89b62827f92e3ac861. --- CHANGES.rst | 27 --------------------------- changelog.d/3316.feature | 1 + changelog.d/3351.misc | 0 changelog.d/3463.misc | 0 changelog.d/3464.misc | 0 changelog.d/3496.feature | 1 + changelog.d/3497.feature | 1 + changelog.d/3498.misc | 0 changelog.d/3499.misc | 0 changelog.d/3501.misc | 0 changelog.d/3505.feature | 1 + changelog.d/3521.feature | 1 + changelog.d/3530.misc | 0 changelog.d/3533.bugfix | 1 + changelog.d/3534.misc | 1 + changelog.d/3535.misc | 0 changelog.d/3540.misc | 1 + changelog.d/3541.feature | 1 + changelog.d/3544.misc | 0 changelog.d/3546.bugfix | 1 + 20 files changed, 10 insertions(+), 27 deletions(-) create mode 100644 changelog.d/3316.feature create mode 100644 changelog.d/3351.misc create mode 100644 changelog.d/3463.misc create mode 100644 changelog.d/3464.misc create mode 100644 changelog.d/3496.feature create mode 100644 changelog.d/3497.feature create mode 100644 changelog.d/3498.misc create mode 100644 changelog.d/3499.misc create mode 100644 changelog.d/3501.misc create mode 100644 changelog.d/3505.feature create mode 100644 changelog.d/3521.feature create mode 100644 changelog.d/3530.misc create mode 100644 changelog.d/3533.bugfix create mode 100644 changelog.d/3534.misc create mode 100644 changelog.d/3535.misc create mode 100644 changelog.d/3540.misc create mode 100644 changelog.d/3541.feature create mode 100644 changelog.d/3544.misc create mode 100644 changelog.d/3546.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index f81e7e6cf3..ba6929c435 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,30 +1,3 @@ -Synapse 0.33.0rc1 (2018-07-18) -============================== - -Features --------- - -- Enforce the specified API for report_event (`#3316 `_) -- Include CPU time from database threads in request/block metrics. (`#3496 `_) -- Add CPU metrics for _fetch_event_list (`#3497 `_) -- Reduce database consumption when processing large numbers of receipts (`#3505 `_) -- Cache optimisation for /sync requests (`#3521 `_) -- Optimisation to make handling incoming federation requests more efficient. (`#3541 `_) - - -Bugfixes --------- - -- Fix queued federation requests being processed in the wrong order (`#3533 `_) -- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 `_) - - -Misc ----- - -- `#3351 `_, `#3463 `_, `#3464 `_, `#3498 `_, `#3499 `_, `#3501 `_, `#3530 `_, `#3534 `_, `#3535 `_, `#3540 `_, `#3544 `_ - - Synapse 0.32.2 (2018-07-07) =========================== diff --git a/changelog.d/3316.feature b/changelog.d/3316.feature new file mode 100644 index 0000000000..50068b7222 --- /dev/null +++ b/changelog.d/3316.feature @@ -0,0 +1 @@ +Enforce the specified API for report_event diff --git a/changelog.d/3351.misc b/changelog.d/3351.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3463.misc b/changelog.d/3463.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3464.misc b/changelog.d/3464.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3496.feature b/changelog.d/3496.feature new file mode 100644 index 0000000000..6a06a7e755 --- /dev/null +++ b/changelog.d/3496.feature @@ -0,0 +1 @@ +Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3497.feature b/changelog.d/3497.feature new file mode 100644 index 0000000000..4e333155b6 --- /dev/null +++ b/changelog.d/3497.feature @@ -0,0 +1 @@ +Add CPU metrics for _fetch_event_list diff --git a/changelog.d/3498.misc b/changelog.d/3498.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3499.misc b/changelog.d/3499.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3501.misc b/changelog.d/3501.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3505.feature b/changelog.d/3505.feature new file mode 100644 index 0000000000..ca1867f529 --- /dev/null +++ b/changelog.d/3505.feature @@ -0,0 +1 @@ +Reduce database consumption when processing large numbers of receipts diff --git a/changelog.d/3521.feature b/changelog.d/3521.feature new file mode 100644 index 0000000000..6dced5f2ae --- /dev/null +++ b/changelog.d/3521.feature @@ -0,0 +1 @@ +Cache optimisation for /sync requests \ No newline at end of file diff --git a/changelog.d/3530.misc b/changelog.d/3530.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3533.bugfix b/changelog.d/3533.bugfix new file mode 100644 index 0000000000..04cbbefd5f --- /dev/null +++ b/changelog.d/3533.bugfix @@ -0,0 +1 @@ +Fix queued federation requests being processed in the wrong order diff --git a/changelog.d/3534.misc b/changelog.d/3534.misc new file mode 100644 index 0000000000..949c12dc69 --- /dev/null +++ b/changelog.d/3534.misc @@ -0,0 +1 @@ +refactor: use parse_{string,integer} and assert's from http.servlet for deduplication diff --git a/changelog.d/3535.misc b/changelog.d/3535.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3540.misc b/changelog.d/3540.misc new file mode 100644 index 0000000000..99dcad8e46 --- /dev/null +++ b/changelog.d/3540.misc @@ -0,0 +1 @@ +check isort for each PR diff --git a/changelog.d/3541.feature b/changelog.d/3541.feature new file mode 100644 index 0000000000..24524136ea --- /dev/null +++ b/changelog.d/3541.feature @@ -0,0 +1 @@ +Optimisation to make handling incoming federation requests more efficient. \ No newline at end of file diff --git a/changelog.d/3544.misc b/changelog.d/3544.misc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/changelog.d/3546.bugfix b/changelog.d/3546.bugfix new file mode 100644 index 0000000000..921dc6e7b0 --- /dev/null +++ b/changelog.d/3546.bugfix @@ -0,0 +1 @@ +Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. \ No newline at end of file From 13d501c773e4c7a5fcf409db7360293a4f1347e0 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:11:24 +1000 Subject: [PATCH 13/16] update changelogs --- changelog.d/3498.bugfix | 1 + changelog.d/3498.misc | 0 changelog.d/3501.feature | 1 + changelog.d/3501.misc | 0 changelog.d/3505.feature | 2 +- changelog.d/3521.feature | 2 +- 6 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog.d/3498.bugfix delete mode 100644 changelog.d/3498.misc create mode 100644 changelog.d/3501.feature delete mode 100644 changelog.d/3501.misc diff --git a/changelog.d/3498.bugfix b/changelog.d/3498.bugfix new file mode 100644 index 0000000000..d0d01101f3 --- /dev/null +++ b/changelog.d/3498.bugfix @@ -0,0 +1 @@ +Use more portable syntax in our use of the attrs package, widening the supported versions diff --git a/changelog.d/3498.misc b/changelog.d/3498.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3501.feature b/changelog.d/3501.feature new file mode 100644 index 0000000000..6a06a7e755 --- /dev/null +++ b/changelog.d/3501.feature @@ -0,0 +1 @@ +Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3501.misc b/changelog.d/3501.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3505.feature b/changelog.d/3505.feature index ca1867f529..a6b686f68d 100644 --- a/changelog.d/3505.feature +++ b/changelog.d/3505.feature @@ -1 +1 @@ -Reduce database consumption when processing large numbers of receipts +optimisation for /sync diff --git a/changelog.d/3521.feature b/changelog.d/3521.feature index 6dced5f2ae..a6b686f68d 100644 --- a/changelog.d/3521.feature +++ b/changelog.d/3521.feature @@ -1 +1 @@ -Cache optimisation for /sync requests \ No newline at end of file +optimisation for /sync From 38f53399a28a75016255509c8e8939d2815cd807 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:11:40 +1000 Subject: [PATCH 14/16] 0.33 final --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 6dc16f0808..5c0f2f83aa 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -17,4 +17,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.33.0rc1" +__version__ = "0.33.0" From d69decd5c78c72abef50b597a689e2bc55a39702 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 19 Jul 2018 21:12:15 +1000 Subject: [PATCH 15/16] 0.33.0 final changelog --- CHANGES.rst | 28 ++++++++++++++++++++++++++++ changelog.d/3316.feature | 1 - changelog.d/3351.misc | 0 changelog.d/3463.misc | 0 changelog.d/3464.misc | 0 changelog.d/3496.feature | 1 - changelog.d/3497.feature | 1 - changelog.d/3498.bugfix | 1 - changelog.d/3499.misc | 0 changelog.d/3501.feature | 1 - changelog.d/3505.feature | 1 - changelog.d/3521.feature | 1 - changelog.d/3530.misc | 0 changelog.d/3533.bugfix | 1 - changelog.d/3534.misc | 1 - changelog.d/3535.misc | 0 changelog.d/3540.misc | 1 - changelog.d/3541.feature | 1 - changelog.d/3544.misc | 0 changelog.d/3546.bugfix | 1 - changelog.d/3561.bugfix | 1 - 21 files changed, 28 insertions(+), 13 deletions(-) delete mode 100644 changelog.d/3316.feature delete mode 100644 changelog.d/3351.misc delete mode 100644 changelog.d/3463.misc delete mode 100644 changelog.d/3464.misc delete mode 100644 changelog.d/3496.feature delete mode 100644 changelog.d/3497.feature delete mode 100644 changelog.d/3498.bugfix delete mode 100644 changelog.d/3499.misc delete mode 100644 changelog.d/3501.feature delete mode 100644 changelog.d/3505.feature delete mode 100644 changelog.d/3521.feature delete mode 100644 changelog.d/3530.misc delete mode 100644 changelog.d/3533.bugfix delete mode 100644 changelog.d/3534.misc delete mode 100644 changelog.d/3535.misc delete mode 100644 changelog.d/3540.misc delete mode 100644 changelog.d/3541.feature delete mode 100644 changelog.d/3544.misc delete mode 100644 changelog.d/3546.bugfix delete mode 100644 changelog.d/3561.bugfix diff --git a/CHANGES.rst b/CHANGES.rst index ba6929c435..afc4fee882 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,31 @@ +Synapse 0.33.0 (2018-07-19) +=========================== + +Features +-------- + +- Enforce the specified API for report_event (`#3316 `_) +- Include CPU time from database threads in request/block metrics. (`#3496 `_, `#3501 `_) +- Add CPU metrics for _fetch_event_list (`#3497 `_) +- optimisation for /sync (`#3505 `_, `#3521 `_) +- Optimisation to make handling incoming federation requests more efficient. (`#3541 `_) + + +Bugfixes +-------- + +- Use more portable syntax in our use of the attrs package, widening the supported versions (`#3498 `_) +- Fix queued federation requests being processed in the wrong order (`#3533 `_) +- Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 `_) +- Disable a noisy warning about logcontexts (`#3561 `_) + + +Misc +---- + +- `#3351 `_, `#3463 `_, `#3464 `_, `#3499 `_, `#3530 `_, `#3534 `_, `#3535 `_, `#3540 `_, `#3544 `_ + + Synapse 0.32.2 (2018-07-07) =========================== diff --git a/changelog.d/3316.feature b/changelog.d/3316.feature deleted file mode 100644 index 50068b7222..0000000000 --- a/changelog.d/3316.feature +++ /dev/null @@ -1 +0,0 @@ -Enforce the specified API for report_event diff --git a/changelog.d/3351.misc b/changelog.d/3351.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3463.misc b/changelog.d/3463.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3464.misc b/changelog.d/3464.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3496.feature b/changelog.d/3496.feature deleted file mode 100644 index 6a06a7e755..0000000000 --- a/changelog.d/3496.feature +++ /dev/null @@ -1 +0,0 @@ -Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3497.feature b/changelog.d/3497.feature deleted file mode 100644 index 4e333155b6..0000000000 --- a/changelog.d/3497.feature +++ /dev/null @@ -1 +0,0 @@ -Add CPU metrics for _fetch_event_list diff --git a/changelog.d/3498.bugfix b/changelog.d/3498.bugfix deleted file mode 100644 index d0d01101f3..0000000000 --- a/changelog.d/3498.bugfix +++ /dev/null @@ -1 +0,0 @@ -Use more portable syntax in our use of the attrs package, widening the supported versions diff --git a/changelog.d/3499.misc b/changelog.d/3499.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3501.feature b/changelog.d/3501.feature deleted file mode 100644 index 6a06a7e755..0000000000 --- a/changelog.d/3501.feature +++ /dev/null @@ -1 +0,0 @@ -Include CPU time from database threads in request/block metrics. diff --git a/changelog.d/3505.feature b/changelog.d/3505.feature deleted file mode 100644 index a6b686f68d..0000000000 --- a/changelog.d/3505.feature +++ /dev/null @@ -1 +0,0 @@ -optimisation for /sync diff --git a/changelog.d/3521.feature b/changelog.d/3521.feature deleted file mode 100644 index a6b686f68d..0000000000 --- a/changelog.d/3521.feature +++ /dev/null @@ -1 +0,0 @@ -optimisation for /sync diff --git a/changelog.d/3530.misc b/changelog.d/3530.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3533.bugfix b/changelog.d/3533.bugfix deleted file mode 100644 index 04cbbefd5f..0000000000 --- a/changelog.d/3533.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix queued federation requests being processed in the wrong order diff --git a/changelog.d/3534.misc b/changelog.d/3534.misc deleted file mode 100644 index 949c12dc69..0000000000 --- a/changelog.d/3534.misc +++ /dev/null @@ -1 +0,0 @@ -refactor: use parse_{string,integer} and assert's from http.servlet for deduplication diff --git a/changelog.d/3535.misc b/changelog.d/3535.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3540.misc b/changelog.d/3540.misc deleted file mode 100644 index 99dcad8e46..0000000000 --- a/changelog.d/3540.misc +++ /dev/null @@ -1 +0,0 @@ -check isort for each PR diff --git a/changelog.d/3541.feature b/changelog.d/3541.feature deleted file mode 100644 index 24524136ea..0000000000 --- a/changelog.d/3541.feature +++ /dev/null @@ -1 +0,0 @@ -Optimisation to make handling incoming federation requests more efficient. \ No newline at end of file diff --git a/changelog.d/3544.misc b/changelog.d/3544.misc deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/changelog.d/3546.bugfix b/changelog.d/3546.bugfix deleted file mode 100644 index 921dc6e7b0..0000000000 --- a/changelog.d/3546.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. \ No newline at end of file diff --git a/changelog.d/3561.bugfix b/changelog.d/3561.bugfix deleted file mode 100644 index 9a197cd67f..0000000000 --- a/changelog.d/3561.bugfix +++ /dev/null @@ -1 +0,0 @@ -Disable a noisy warning about logcontexts From 6284f579bf1bea620c578c3c82ea54d59a76b708 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 19 Jul 2018 12:33:40 +0100 Subject: [PATCH 16/16] Update r0.33.0 release notes (mostly just clarifications) --- CHANGES.rst | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index afc4fee882..da6e0eb195 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,29 +1,39 @@ Synapse 0.33.0 (2018-07-19) =========================== +Bugfixes +-------- + +- Disable a noisy warning about logcontexts. (`#3561 `_) + + +Synapse 0.33.0rc1 (2018-07-18) +============================== + Features -------- -- Enforce the specified API for report_event (`#3316 `_) +- Enforce the specified API for report_event. (`#3316 `_) - Include CPU time from database threads in request/block metrics. (`#3496 `_, `#3501 `_) -- Add CPU metrics for _fetch_event_list (`#3497 `_) -- optimisation for /sync (`#3505 `_, `#3521 `_) +- Add CPU metrics for _fetch_event_list. (`#3497 `_) - Optimisation to make handling incoming federation requests more efficient. (`#3541 `_) Bugfixes -------- -- Use more portable syntax in our use of the attrs package, widening the supported versions (`#3498 `_) -- Fix queued federation requests being processed in the wrong order (`#3533 `_) +- Fix a significant performance regression in /sync. (`#3505 `_, `#3521 `_, `#3530 `_, `#3544 `_) +- Use more portable syntax in our use of the attrs package, widening the supported versions. (`#3498 `_) +- Fix queued federation requests being processed in the wrong order. (`#3533 `_) - Ensure that erasure requests are correctly honoured for publicly accessible rooms when accessed over federation. (`#3546 `_) -- Disable a noisy warning about logcontexts (`#3561 `_) Misc ---- -- `#3351 `_, `#3463 `_, `#3464 `_, `#3499 `_, `#3530 `_, `#3534 `_, `#3535 `_, `#3540 `_, `#3544 `_ +- Refactoring to improve testability. (`#3351 `_, `#3499 `_) +- Use ``isort`` to sort imports. (`#3463 `_, `#3464 `_, `#3540 `_) +- Use parse and asserts from http.servlet. (`#3534 `_, `#3535 `_). Synapse 0.32.2 (2018-07-07)