From 2ade05dca3d6da67e35c3a8ccdd278221f2566ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Sep 2019 14:16:10 +0100 Subject: [PATCH 01/21] Add last seen info to devices table. This allows us to purge old user_ips entries without having to preserve the latest last seen info for active devices. --- synapse/storage/client_ips.py | 15 +++++++++++++ .../schema/delta/56/devices_last_seen.sql | 21 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 synapse/storage/schema/delta/56/devices_last_seen.sql diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 6db8c54077..4db2e7f481 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -354,6 +354,21 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): }, lock=False, ) + + # Technically an access token might not be associated with + # a device so we need to check. + if device_id: + self._simple_upsert_txn( + txn, + table="devices", + keyvalues={"user_id": user_id, "device_id": device_id}, + values={ + "user_agent": user_agent, + "last_seen": last_seen, + "ip": ip, + }, + lock=False, + ) except Exception as e: # Failed to upsert, log and continue logger.error("Failed to insert client IP %r: %r", entry, e) diff --git a/synapse/storage/schema/delta/56/devices_last_seen.sql b/synapse/storage/schema/delta/56/devices_last_seen.sql new file mode 100644 index 0000000000..8818eeeb7e --- /dev/null +++ b/synapse/storage/schema/delta/56/devices_last_seen.sql @@ -0,0 +1,21 @@ +/* Copyright 2019 Matrix.org Foundation CIC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Track last seen information for a device in the devices table, rather +-- than relying on it being in the user_ips table (which we want to be able +-- to purge old entries from) +ALTER TABLE devices ADD COLUMN last_seen BIGINT; +ALTER TABLE devices ADD COLUMN ip TEXT; +ALTER TABLE devices ADD COLUMN user_agent TEXT; From ed80231ade20ce7881bb2026692fe3a6252f1c02 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Sep 2019 15:59:43 +0100 Subject: [PATCH 02/21] Add BG update to populate devices last seen info --- synapse/storage/client_ips.py | 52 +++++++++++++++++++ .../schema/delta/56/devices_last_seen.sql | 3 ++ 2 files changed, 55 insertions(+) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 4db2e7f481..8839562269 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -85,6 +85,11 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): "user_ips_drop_nonunique_index", self._remove_user_ip_nonunique ) + # Update the last seen info in devices. + self.register_background_update_handler( + "devices_last_seen", self._devices_last_seen_update + ) + # (user_id, access_token, ip,) -> (user_agent, device_id, last_seen) self._batch_row_update = {} @@ -485,3 +490,50 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): } for (access_token, ip), (user_agent, last_seen) in iteritems(results) ) + + @defer.inlineCallbacks + def _devices_last_seen_update(self, progress, batch_size): + """Background update to insert last seen info into devices table + """ + + last_user_id = progress.get("last_user_id", "") + last_device_id = progress.get("last_device_id", "") + + def _devices_last_seen_update_txn(txn): + sql = """ + SELECT u.last_seen, u.ip, u.user_agent, user_id, device_id FROM devices + INNER JOIN user_ips AS u USING (user_id, device_id) + WHERE user_id > ? OR (user_id = ? AND device_id > ?) + ORDER BY user_id ASC, device_id ASC + LIMIT ? + """ + txn.execute(sql, (last_user_id, last_user_id, last_device_id, batch_size)) + + rows = txn.fetchall() + if not rows: + return 0 + + sql = """ + UPDATE devices + SET last_seen = ?, ip = ?, user_agent = ? + WHERE user_id = ? AND device_id = ? + """ + txn.execute_batch(sql, rows) + + _, _, _, user_id, device_id = rows[-1] + self._background_update_progress_txn( + txn, + "devices_last_seen", + {"last_user_id": user_id, "last_device_id": device_id}, + ) + + return len(rows) + + updated = yield self.runInteraction( + "_devices_last_seen_update", _devices_last_seen_update_txn + ) + + if not updated: + yield self._end_background_update("devices_last_seen") + + return updated diff --git a/synapse/storage/schema/delta/56/devices_last_seen.sql b/synapse/storage/schema/delta/56/devices_last_seen.sql index 8818eeeb7e..dfa902d0ba 100644 --- a/synapse/storage/schema/delta/56/devices_last_seen.sql +++ b/synapse/storage/schema/delta/56/devices_last_seen.sql @@ -19,3 +19,6 @@ ALTER TABLE devices ADD COLUMN last_seen BIGINT; ALTER TABLE devices ADD COLUMN ip TEXT; ALTER TABLE devices ADD COLUMN user_agent TEXT; + +INSERT INTO background_updates (update_name, progress_json) VALUES + ('devices_last_seen', '{}'); From 51d28272e20d799b2e35a8a14b3c1d9d5f555d10 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Sep 2019 16:00:18 +0100 Subject: [PATCH 03/21] Query devices table for last seen info. This is a) simpler than querying user_ips directly and b) means we can purge older entries from user_ips without losing the required info. The storage functions now no longer return the access_token, since it was unused. --- synapse/storage/client_ips.py | 57 +++++--------------------------- tests/storage/test_client_ips.py | 1 - 2 files changed, 8 insertions(+), 50 deletions(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 8839562269..a4e6d9dbe7 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -392,19 +392,14 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): keys giving the column names """ - res = yield self.runInteraction( - "get_last_client_ip_by_device", - self._get_last_client_ip_by_device_txn, - user_id, - device_id, - retcols=( - "user_id", - "access_token", - "ip", - "user_agent", - "device_id", - "last_seen", - ), + keyvalues = {"user_id": user_id} + if device_id: + keyvalues["device_id"] = device_id + + res = yield self._simple_select_list( + table="devices", + keyvalues=keyvalues, + retcols=("user_id", "ip", "user_agent", "device_id", "last_seen"), ) ret = {(d["user_id"], d["device_id"]): d for d in res} @@ -423,42 +418,6 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): } return ret - @classmethod - def _get_last_client_ip_by_device_txn(cls, txn, user_id, device_id, retcols): - where_clauses = [] - bindings = [] - if device_id is None: - where_clauses.append("user_id = ?") - bindings.extend((user_id,)) - else: - where_clauses.append("(user_id = ? AND device_id = ?)") - bindings.extend((user_id, device_id)) - - if not where_clauses: - return [] - - inner_select = ( - "SELECT MAX(last_seen) mls, user_id, device_id FROM user_ips " - "WHERE %(where)s " - "GROUP BY user_id, device_id" - ) % {"where": " OR ".join(where_clauses)} - - sql = ( - "SELECT %(retcols)s FROM user_ips " - "JOIN (%(inner_select)s) ips ON" - " user_ips.last_seen = ips.mls AND" - " user_ips.user_id = ips.user_id AND" - " (user_ips.device_id = ips.device_id OR" - " (user_ips.device_id IS NULL AND ips.device_id IS NULL)" - " )" - ) % { - "retcols": ",".join("user_ips." + c for c in retcols), - "inner_select": inner_select, - } - - txn.execute(sql, bindings) - return cls.cursor_to_dict(txn) - @defer.inlineCallbacks def get_user_ip_and_agents(self, user): user_id = user.to_string() diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index 09305c3bf1..6ac4654085 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -55,7 +55,6 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase): { "user_id": user_id, "device_id": "device_id", - "access_token": "access_token", "ip": "ip", "user_agent": "user_agent", "last_seen": 12345678000, From 691a70190b76aa29481f6299580b71160068ef8e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Sep 2019 16:04:41 +0100 Subject: [PATCH 04/21] Newsfile --- changelog.d/6089.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6089.misc diff --git a/changelog.d/6089.misc b/changelog.d/6089.misc new file mode 100644 index 0000000000..fa3c197c54 --- /dev/null +++ b/changelog.d/6089.misc @@ -0,0 +1 @@ +Move last seen info into devices table. From acb62a7cc6973618397a868289b5881f1c3c1ec3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Sep 2019 16:50:31 +0100 Subject: [PATCH 05/21] Test background update --- tests/storage/test_client_ips.py | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index 6ac4654085..76fe65b59e 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -200,6 +200,85 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase): active = self.get_success(self.store.user_last_seen_monthly_active(user_id)) self.assertTrue(active) + def test_devices_last_seen_bg_update(self): + # First make sure we have completed all updates. + while not self.get_success(self.store.has_completed_background_updates()): + self.get_success(self.store.do_next_background_update(100), by=0.1) + + # Insert a user IP + user_id = "@user:id" + self.get_success( + self.store.insert_client_ip( + user_id, "access_token", "ip", "user_agent", "device_id" + ) + ) + + # Force persisting to disk + self.reactor.advance(200) + + # But clear the associated entry in devices table + self.get_success( + self.store._simple_update( + table="devices", + keyvalues={"user_id": user_id, "device_id": "device_id"}, + updatevalues={"last_seen": None, "ip": None, "user_agent": None}, + desc="test_devices_last_seen_bg_update", + ) + ) + + # We should now get nulls when querying + result = self.get_success( + self.store.get_last_client_ip_by_device(user_id, "device_id") + ) + + r = result[(user_id, "device_id")] + self.assertDictContainsSubset( + { + "user_id": user_id, + "device_id": "device_id", + "ip": None, + "user_agent": None, + "last_seen": None, + }, + r, + ) + + # Register the background update to run again. + self.get_success( + self.store._simple_insert( + table="background_updates", + values={ + "update_name": "devices_last_seen", + "progress_json": "{}", + "depends_on": None, + }, + ) + ) + + # ... and tell the DataStore that it hasn't finished all updates yet + self.store._all_done = False + + # Now let's actually drive the updates to completion + while not self.get_success(self.store.has_completed_background_updates()): + self.get_success(self.store.do_next_background_update(100), by=0.1) + + # We should now get the correct result again + result = self.get_success( + self.store.get_last_client_ip_by_device(user_id, "device_id") + ) + + r = result[(user_id, "device_id")] + self.assertDictContainsSubset( + { + "user_id": user_id, + "device_id": "device_id", + "ip": "ip", + "user_agent": "user_agent", + "last_seen": 0, + }, + r, + ) + class ClientIpAuthTestCase(unittest.HomeserverTestCase): From 367158a609d18b6dbd143f8bee0529e743d5b5a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Sep 2019 14:16:16 +0100 Subject: [PATCH 06/21] Add wrap_as_background_process decorator. This does the same thing as `run_as_background_process` but means we don't need to create superfluous functions. --- synapse/metrics/background_process_metrics.py | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index edd6b42db3..b24e2fab4a 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -15,6 +15,8 @@ import logging import threading +from asyncio import iscoroutine +from functools import wraps import six @@ -197,7 +199,15 @@ def run_as_background_process(desc, func, *args, **kwargs): _background_processes.setdefault(desc, set()).add(proc) try: - yield func(*args, **kwargs) + # We ensureDeferred here to handle coroutines + result = func(*args, **kwargs) + + # We need this check because ensureDeferred doesn't like when + # func doesn't return a Deferred or coroutine. + if iscoroutine(result): + result = defer.ensureDeferred(result) + + return (yield result) except Exception: logger.exception("Background process '%s' threw an exception", desc) finally: @@ -208,3 +218,20 @@ def run_as_background_process(desc, func, *args, **kwargs): with PreserveLoggingContext(): return run() + + +def wrap_as_background_process(desc): + """Decorator that wraps a function that gets called as a background + process. + + Equivalent of calling the function with `run_as_background_process` + """ + + def wrap_as_background_process_inner(func): + @wraps(func) + def wrap_as_background_process_inner_2(*args, **kwargs): + return run_as_background_process(desc, func, *args, **kwargs) + + return wrap_as_background_process_inner_2 + + return wrap_as_background_process_inner From 2135c198d17b41297511a2fc3b39551d160069b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Sep 2019 14:18:31 +0100 Subject: [PATCH 07/21] Add has_completed_background_update This allows checking if a specific background update has completed. --- synapse/storage/background_updates.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index e5f0668f09..3fc25cd828 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -140,7 +140,7 @@ class BackgroundUpdateStore(SQLBaseStore): "background_updates", keyvalues=None, retcol="1", - desc="check_background_updates", + desc="has_completed_background_updates", ) if not updates: self._all_done = True @@ -148,6 +148,29 @@ class BackgroundUpdateStore(SQLBaseStore): return False + async def has_completed_background_update(self, update_name): + """Check if the given background update has finished running. + + Returns: + Deferred[bool] + """ + + if self._all_done: + return True + + if update_name in self._background_update_queue: + return False + + update_exists = await self._simple_select_one_onecol( + "background_updates", + keyvalues={"update_name": update_name}, + retcol="1", + desc="has_completed_background_update", + allow_none=True, + ) + + return not update_exists + @defer.inlineCallbacks def do_next_background_update(self, desired_duration_ms): """Does some amount of work on the next queued background update From 242017db8b7b57be28a019ecbba1619d75d54889 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Sep 2019 15:20:40 +0100 Subject: [PATCH 08/21] Prune rows in user_ips older than configured period Defaults to pruning everything older than 28d. --- docs/sample_config.yaml | 6 ++++ synapse/config/server.py | 13 ++++++++ synapse/storage/client_ips.py | 62 ++++++++++++++++++++++++++++++----- 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 61d9f09a99..cc6035c838 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -313,6 +313,12 @@ listeners: # redaction_retention_period: 7d +# How long to track users' last seen time and IPs in the database. +# +# Defaults to `28d`. Set to `null` to disable. +# +#user_ips_max_age: 14d + ## TLS ## diff --git a/synapse/config/server.py b/synapse/config/server.py index 7f8d315954..655e7487a4 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -172,6 +172,13 @@ class ServerConfig(Config): else: self.redaction_retention_period = None + # How long to keep entries in the `users_ips` table. + user_ips_max_age = config.get("user_ips_max_age", "28d") + if user_ips_max_age is not None: + self.user_ips_max_age = self.parse_duration(user_ips_max_age) + else: + self.user_ips_max_age = None + # Options to disable HS self.hs_disabled = config.get("hs_disabled", False) self.hs_disabled_message = config.get("hs_disabled_message", "") @@ -735,6 +742,12 @@ class ServerConfig(Config): # Defaults to `7d`. Set to `null` to disable. # redaction_retention_period: 7d + + # How long to track users' last seen time and IPs in the database. + # + # Defaults to `28d`. Set to `null` to disable. + # + #user_ips_max_age: 14d """ % locals() ) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index a4e6d9dbe7..176c812b1f 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -19,7 +19,7 @@ from six import iteritems from twisted.internet import defer -from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.util.caches import CACHE_SIZE_FACTOR from . import background_updates @@ -42,6 +42,8 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): super(ClientIpStore, self).__init__(db_conn, hs) + self.user_ips_max_age = hs.config.user_ips_max_age + self.register_background_index_update( "user_ips_device_index", index_name="user_ips_device_id", @@ -100,6 +102,9 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): "before", "shutdown", self._update_client_ips_batch ) + if self.user_ips_max_age: + self._clock.looping_call(self._prune_old_user_ips, 5 * 1000) + @defer.inlineCallbacks def _remove_user_ip_nonunique(self, progress, batch_size): def f(conn): @@ -319,20 +324,19 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): self._batch_row_update[key] = (user_agent, device_id, now) + @wrap_as_background_process("update_client_ips") def _update_client_ips_batch(self): # If the DB pool has already terminated, don't try updating if not self.hs.get_db_pool().running: return - def update(): - to_update = self._batch_row_update - self._batch_row_update = {} - return self.runInteraction( - "_update_client_ips_batch", self._update_client_ips_batch_txn, to_update - ) + to_update = self._batch_row_update + self._batch_row_update = {} - return run_as_background_process("update_client_ips", update) + return self.runInteraction( + "_update_client_ips_batch", self._update_client_ips_batch_txn, to_update + ) def _update_client_ips_batch_txn(self, txn, to_update): if "user_ips" in self._unsafe_to_upsert_tables or ( @@ -496,3 +500,45 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): yield self._end_background_update("devices_last_seen") return updated + + @wrap_as_background_process("prune_old_user_ips") + async def _prune_old_user_ips(self): + """Removes entries in user IPs older than the configured period. + """ + + if not self.user_ips_max_age: + # Nothing to do + return + + if not await self.has_completed_background_update("devices_last_seen"): + # Only start pruning if we have finished populating the devices + # last seen info. + return + + # We do a slightly funky SQL delete to ensure we don't try and delete + # too much at once (as the table may be very large from before we + # started pruning). + # + # This works by finding the max last_seen that is less than the given + # time, but has no more than N rows before it, deleting all rows with + # a lesser last_seen time. (We COALESCE so that the sub-SELECT always + # returns exactly one row). + sql = """ + DELETE FROM user_ips + WHERE last_seen <= ( + SELECT COALESCE(MAX(last_seen), -1) + FROM ( + SELECT last_seen FROM user_ips + WHERE last_seen <= ? + ORDER BY last_seen ASC + LIMIT 5000 + ) AS u + ) + """ + + timestamp = self.clock.time_msec() - self.user_ips_max_age + + def _prune_old_user_ips_txn(txn): + txn.execute(sql, (timestamp,)) + + await self.runInteraction("_prune_old_user_ips", _prune_old_user_ips_txn) From faac453f08046ddf00b39b90ba255f774b75c253 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Sep 2019 15:51:42 +0100 Subject: [PATCH 09/21] Test that pruning of old user IPs works --- tests/storage/test_client_ips.py | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index 76fe65b59e..afac5dec7f 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -279,6 +279,77 @@ class ClientIpStoreTestCase(unittest.HomeserverTestCase): r, ) + def test_old_user_ips_pruned(self): + # First make sure we have completed all updates. + while not self.get_success(self.store.has_completed_background_updates()): + self.get_success(self.store.do_next_background_update(100), by=0.1) + + # Insert a user IP + user_id = "@user:id" + self.get_success( + self.store.insert_client_ip( + user_id, "access_token", "ip", "user_agent", "device_id" + ) + ) + + # Force persisting to disk + self.reactor.advance(200) + + # We should see that in the DB + result = self.get_success( + self.store._simple_select_list( + table="user_ips", + keyvalues={"user_id": user_id}, + retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"], + desc="get_user_ip_and_agents", + ) + ) + + self.assertEqual( + result, + [ + { + "access_token": "access_token", + "ip": "ip", + "user_agent": "user_agent", + "device_id": "device_id", + "last_seen": 0, + } + ], + ) + + # Now advance by a couple of months + self.reactor.advance(60 * 24 * 60 * 60) + + # We should get no results. + result = self.get_success( + self.store._simple_select_list( + table="user_ips", + keyvalues={"user_id": user_id}, + retcols=["access_token", "ip", "user_agent", "device_id", "last_seen"], + desc="get_user_ip_and_agents", + ) + ) + + self.assertEqual(result, []) + + # But we should still get the correct values for the device + result = self.get_success( + self.store.get_last_client_ip_by_device(user_id, "device_id") + ) + + r = result[(user_id, "device_id")] + self.assertDictContainsSubset( + { + "user_id": user_id, + "device_id": "device_id", + "ip": "ip", + "user_agent": "user_agent", + "last_seen": 0, + }, + r, + ) + class ClientIpAuthTestCase(unittest.HomeserverTestCase): From f8b02c54207e5e99fcd95cb3e19a11423768e696 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Sep 2019 15:59:43 +0100 Subject: [PATCH 10/21] Newsfile --- changelog.d/6098.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6098.feature diff --git a/changelog.d/6098.feature b/changelog.d/6098.feature new file mode 100644 index 0000000000..f3c693c06b --- /dev/null +++ b/changelog.d/6098.feature @@ -0,0 +1 @@ +Add support for pruning old rows in `user_ips` table. From 50572db837f3e6a0869e9ec573e02d4af72548ea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 25 Sep 2019 17:00:23 +0100 Subject: [PATCH 11/21] Use if `is not None` Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- synapse/storage/client_ips.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index a4e6d9dbe7..8996689744 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -393,7 +393,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): """ keyvalues = {"user_id": user_id} - if device_id: + if device_id is not None: keyvalues["device_id"] = device_id res = yield self._simple_select_list( From 39b50ad42a8cf784e627959e9652589338121ccd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 25 Sep 2019 17:22:33 +0100 Subject: [PATCH 12/21] Review comments --- docs/sample_config.yaml | 2 +- synapse/config/server.py | 2 +- synapse/storage/background_updates.py | 5 +---- synapse/storage/client_ips.py | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index cc6035c838..7902d9ed6f 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -315,7 +315,7 @@ redaction_retention_period: 7d # How long to track users' last seen time and IPs in the database. # -# Defaults to `28d`. Set to `null` to disable. +# Defaults to `28d`. Set to `null` to disable clearing out of old rows. # #user_ips_max_age: 14d diff --git a/synapse/config/server.py b/synapse/config/server.py index 655e7487a4..f8b7b4bef9 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -745,7 +745,7 @@ class ServerConfig(Config): # How long to track users' last seen time and IPs in the database. # - # Defaults to `28d`. Set to `null` to disable. + # Defaults to `28d`. Set to `null` to disable clearing out of old rows. # #user_ips_max_age: 14d """ diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 3fc25cd828..30788137a8 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -148,11 +148,8 @@ class BackgroundUpdateStore(SQLBaseStore): return False - async def has_completed_background_update(self, update_name): + async def has_completed_background_update(self, update_name) -> bool: """Check if the given background update has finished running. - - Returns: - Deferred[bool] """ if self._all_done: diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 176c812b1f..a4d40dfa1e 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -506,7 +506,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): """Removes entries in user IPs older than the configured period. """ - if not self.user_ips_max_age: + if self.user_ips_max_age is None: # Nothing to do return From a4f3ca48b5250a1c2c4de8a363f69bbeb0adeefd Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 25 Sep 2019 17:27:35 +0100 Subject: [PATCH 13/21] Enable cleaning up extremities with dummy events by default to prevent undue build up of forward extremities. (#5884) --- changelog.d/5884.feature | 1 + synapse/config/server.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 changelog.d/5884.feature diff --git a/changelog.d/5884.feature b/changelog.d/5884.feature new file mode 100644 index 0000000000..bfd0489392 --- /dev/null +++ b/changelog.d/5884.feature @@ -0,0 +1 @@ +Enable cleaning up extremities with dummy events by default to prevent undue build up of forward extremities. diff --git a/synapse/config/server.py b/synapse/config/server.py index 419787a89c..3a7a49bc91 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -355,10 +355,8 @@ class ServerConfig(Config): _check_resource_config(self.listeners) - # An experimental option to try and periodically clean up extremities - # by sending dummy events. self.cleanup_extremities_with_dummy_events = config.get( - "cleanup_extremities_with_dummy_events", False + "cleanup_extremities_with_dummy_events", True ) def has_tls_listener(self): From a96318127dc17ee102bcf90821d90b7e6079a85d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 25 Sep 2019 18:17:39 +0100 Subject: [PATCH 14/21] Update comments and docstring --- synapse/metrics/background_process_metrics.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index b24e2fab4a..c53d2a0d40 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -175,7 +175,7 @@ def run_as_background_process(desc, func, *args, **kwargs): Args: desc (str): a description for this background process type - func: a function, which may return a Deferred + func: a function, which may return a Deferred or a coroutine args: positional args for func kwargs: keyword args for func @@ -199,11 +199,13 @@ def run_as_background_process(desc, func, *args, **kwargs): _background_processes.setdefault(desc, set()).add(proc) try: - # We ensureDeferred here to handle coroutines result = func(*args, **kwargs) - # We need this check because ensureDeferred doesn't like when - # func doesn't return a Deferred or coroutine. + # We probably don't have an ensureDeferred in our call stack to handle + # coroutine results, so we need to ensureDeferred here. + # + # But we need this check because ensureDeferred doesn't like being + # called on immediate values (as opposed to Deferreds or coroutines). if iscoroutine(result): result = defer.ensureDeferred(result) From 034db2ba2115d935ce62b641b4051e477a454eac Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 26 Sep 2019 11:47:53 +0100 Subject: [PATCH 15/21] Fix dummy event insertion consent bug (#6053) Fixes #5905 --- changelog.d/6053.bugfix | 1 + synapse/handlers/message.py | 97 +++++++++++---- synapse/storage/event_federation.py | 18 ++- tests/storage/test_cleanup_extrems.py | 157 ++++++++++++++++++++++--- tests/storage/test_event_federation.py | 40 +++++++ 5 files changed, 270 insertions(+), 43 deletions(-) create mode 100644 changelog.d/6053.bugfix diff --git a/changelog.d/6053.bugfix b/changelog.d/6053.bugfix new file mode 100644 index 0000000000..6311157bf6 --- /dev/null +++ b/changelog.d/6053.bugfix @@ -0,0 +1 @@ +Prevent exceptions being logged when extremity-cleanup events fail due to lack of user consent to the terms of service. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 1f8272784e..0f8cce8ffe 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -222,6 +222,13 @@ class MessageHandler(object): } +# The duration (in ms) after which rooms should be removed +# `_rooms_to_exclude_from_dummy_event_insertion` (with the effect that we will try +# to generate a dummy event for them once more) +# +_DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY = 7 * 24 * 60 * 60 * 1000 + + class EventCreationHandler(object): def __init__(self, hs): self.hs = hs @@ -258,6 +265,13 @@ class EventCreationHandler(object): self.config.block_events_without_consent_error ) + # Rooms which should be excluded from dummy insertion. (For instance, + # those without local users who can send events into the room). + # + # map from room id to time-of-last-attempt. + # + self._rooms_to_exclude_from_dummy_event_insertion = {} # type: dict[str, int] + # we need to construct a ConsentURIBuilder here, as it checks that the necessary # config options, but *only* if we have a configuration for which we are # going to need it. @@ -888,9 +902,11 @@ class EventCreationHandler(object): """Background task to send dummy events into rooms that have a large number of extremities """ - + self._expire_rooms_to_exclude_from_dummy_event_insertion() room_ids = yield self.store.get_rooms_with_many_extremities( - min_count=10, limit=5 + min_count=10, + limit=5, + room_id_filter=self._rooms_to_exclude_from_dummy_event_insertion.keys(), ) for room_id in room_ids: @@ -904,32 +920,61 @@ class EventCreationHandler(object): members = yield self.state.get_current_users_in_room( room_id, latest_event_ids=latest_event_ids ) + dummy_event_sent = False + for user_id in members: + if not self.hs.is_mine_id(user_id): + continue + requester = create_requester(user_id) + try: + event, context = yield self.create_event( + requester, + { + "type": "org.matrix.dummy_event", + "content": {}, + "room_id": room_id, + "sender": user_id, + }, + prev_events_and_hashes=prev_events_and_hashes, + ) - user_id = None - for member in members: - if self.hs.is_mine_id(member): - user_id = member + event.internal_metadata.proactively_send = False + + yield self.send_nonmember_event( + requester, event, context, ratelimit=False + ) + dummy_event_sent = True break + except ConsentNotGivenError: + logger.info( + "Failed to send dummy event into room %s for user %s due to " + "lack of consent. Will try another user" % (room_id, user_id) + ) + except AuthError: + logger.info( + "Failed to send dummy event into room %s for user %s due to " + "lack of power. Will try another user" % (room_id, user_id) + ) - if not user_id: - # We don't have a joined user. - # TODO: We should do something here to stop the room from - # appearing next time. - continue + if not dummy_event_sent: + # Did not find a valid user in the room, so remove from future attempts + # Exclusion is time limited, so the room will be rechecked in the future + # dependent on _DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY + logger.info( + "Failed to send dummy event into room %s. Will exclude it from " + "future attempts until cache expires" % (room_id,) + ) + now = self.clock.time_msec() + self._rooms_to_exclude_from_dummy_event_insertion[room_id] = now - requester = create_requester(user_id) - - event, context = yield self.create_event( - requester, - { - "type": "org.matrix.dummy_event", - "content": {}, - "room_id": room_id, - "sender": user_id, - }, - prev_events_and_hashes=prev_events_and_hashes, + def _expire_rooms_to_exclude_from_dummy_event_insertion(self): + expire_before = self.clock.time_msec() - _DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY + to_expire = set() + for room_id, time in self._rooms_to_exclude_from_dummy_event_insertion.items(): + if time < expire_before: + to_expire.add(room_id) + for room_id in to_expire: + logger.debug( + "Expiring room id %s from dummy event insertion exclusion cache", + room_id, ) - - event.internal_metadata.proactively_send = False - - yield self.send_nonmember_event(requester, event, context, ratelimit=False) + del self._rooms_to_exclude_from_dummy_event_insertion[room_id] diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 4f500d893e..f5e8c39262 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import itertools import logging import random @@ -190,12 +191,13 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas room_id, ) - def get_rooms_with_many_extremities(self, min_count, limit): + def get_rooms_with_many_extremities(self, min_count, limit, room_id_filter): """Get the top rooms with at least N extremities. Args: min_count (int): The minimum number of extremities limit (int): The maximum number of rooms to return. + room_id_filter (iterable[str]): room_ids to exclude from the results Returns: Deferred[list]: At most `limit` room IDs that have at least @@ -203,15 +205,25 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas """ def _get_rooms_with_many_extremities_txn(txn): + where_clause = "1=1" + if room_id_filter: + where_clause = "room_id NOT IN (%s)" % ( + ",".join("?" for _ in room_id_filter), + ) + sql = """ SELECT room_id FROM event_forward_extremities + WHERE %s GROUP BY room_id HAVING count(*) > ? ORDER BY count(*) DESC LIMIT ? - """ + """ % ( + where_clause, + ) - txn.execute(sql, (min_count, limit)) + query_args = list(itertools.chain(room_id_filter, [min_count, limit])) + txn.execute(sql, query_args) return [room_id for room_id, in txn] return self.runInteraction( diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py index e9e2d5337c..34f9c72709 100644 --- a/tests/storage/test_cleanup_extrems.py +++ b/tests/storage/test_cleanup_extrems.py @@ -14,7 +14,13 @@ # limitations under the License. import os.path +from unittest.mock import patch +from mock import Mock + +import synapse.rest.admin +from synapse.api.constants import EventTypes +from synapse.rest.client.v1 import login, room from synapse.storage import prepare_database from synapse.types import Requester, UserID @@ -225,6 +231,14 @@ class CleanupExtremBackgroundUpdateStoreTestCase(HomeserverTestCase): class CleanupExtremDummyEventsTestCase(HomeserverTestCase): + CONSENT_VERSION = "1" + EXTREMITIES_COUNT = 50 + servlets = [ + synapse.rest.admin.register_servlets_for_client_rest_resource, + login.register_servlets, + room.register_servlets, + ] + def make_homeserver(self, reactor, clock): config = self.default_config() config["cleanup_extremities_with_dummy_events"] = True @@ -233,27 +247,19 @@ class CleanupExtremDummyEventsTestCase(HomeserverTestCase): def prepare(self, reactor, clock, homeserver): self.store = homeserver.get_datastore() self.room_creator = homeserver.get_room_creation_handler() + self.event_creator_handler = homeserver.get_event_creation_handler() # Create a test user and room - self.user = UserID("alice", "test") + self.user = UserID.from_string(self.register_user("user1", "password")) + self.token1 = self.login("user1", "password") self.requester = Requester(self.user, None, False, None, None) info = self.get_success(self.room_creator.create_room(self.requester, {})) self.room_id = info["room_id"] + self.event_creator = homeserver.get_event_creation_handler() + homeserver.config.user_consent_version = self.CONSENT_VERSION def test_send_dummy_event(self): - # Create a bushy graph with 50 extremities. - - event_id_start = self.create_and_send_event(self.room_id, self.user) - - for _ in range(50): - self.create_and_send_event( - self.room_id, self.user, prev_event_ids=[event_id_start] - ) - - latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) - ) - self.assertEqual(len(latest_event_ids), 50) + self._create_extremity_rich_graph() # Pump the reactor repeatedly so that the background updates have a # chance to run. @@ -263,3 +269,126 @@ class CleanupExtremDummyEventsTestCase(HomeserverTestCase): self.store.get_latest_event_ids_in_room(self.room_id) ) self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) + + @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=0) + def test_send_dummy_events_when_insufficient_power(self): + self._create_extremity_rich_graph() + # Criple power levels + self.helper.send_state( + self.room_id, + EventTypes.PowerLevels, + body={"users": {str(self.user): -1}}, + tok=self.token1, + ) + # Pump the reactor repeatedly so that the background updates have a + # chance to run. + self.pump(10 * 60) + + latest_event_ids = self.get_success( + self.store.get_latest_event_ids_in_room(self.room_id) + ) + # Check that the room has not been pruned + self.assertTrue(len(latest_event_ids) > 10) + + # New user with regular levels + user2 = self.register_user("user2", "password") + token2 = self.login("user2", "password") + self.helper.join(self.room_id, user2, tok=token2) + self.pump(10 * 60) + + latest_event_ids = self.get_success( + self.store.get_latest_event_ids_in_room(self.room_id) + ) + self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) + + @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=0) + def test_send_dummy_event_without_consent(self): + self._create_extremity_rich_graph() + self._enable_consent_checking() + + # Pump the reactor repeatedly so that the background updates have a + # chance to run. Attempt to add dummy event with user that has not consented + # Check that dummy event send fails. + self.pump(10 * 60) + latest_event_ids = self.get_success( + self.store.get_latest_event_ids_in_room(self.room_id) + ) + self.assertTrue(len(latest_event_ids) == self.EXTREMITIES_COUNT) + + # Create new user, and add consent + user2 = self.register_user("user2", "password") + token2 = self.login("user2", "password") + self.get_success( + self.store.user_set_consent_version(user2, self.CONSENT_VERSION) + ) + self.helper.join(self.room_id, user2, tok=token2) + + # Background updates should now cause a dummy event to be added to the graph + self.pump(10 * 60) + + latest_event_ids = self.get_success( + self.store.get_latest_event_ids_in_room(self.room_id) + ) + self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) + + @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=250) + def test_expiry_logic(self): + """Simple test to ensure that _expire_rooms_to_exclude_from_dummy_event_insertion() + expires old entries correctly. + """ + self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[ + "1" + ] = 100000 + self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[ + "2" + ] = 200000 + self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion[ + "3" + ] = 300000 + self.event_creator_handler._expire_rooms_to_exclude_from_dummy_event_insertion() + # All entries within time frame + self.assertEqual( + len( + self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion + ), + 3, + ) + # Oldest room to expire + self.pump(1) + self.event_creator_handler._expire_rooms_to_exclude_from_dummy_event_insertion() + self.assertEqual( + len( + self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion + ), + 2, + ) + # All rooms to expire + self.pump(2) + self.assertEqual( + len( + self.event_creator_handler._rooms_to_exclude_from_dummy_event_insertion + ), + 0, + ) + + def _create_extremity_rich_graph(self): + """Helper method to create bushy graph on demand""" + + event_id_start = self.create_and_send_event(self.room_id, self.user) + + for _ in range(self.EXTREMITIES_COUNT): + self.create_and_send_event( + self.room_id, self.user, prev_event_ids=[event_id_start] + ) + + latest_event_ids = self.get_success( + self.store.get_latest_event_ids_in_room(self.room_id) + ) + self.assertEqual(len(latest_event_ids), 50) + + def _enable_consent_checking(self): + """Helper method to enable consent checking""" + self.event_creator._block_events_without_consent_error = "No consent from user" + consent_uri_builder = Mock() + consent_uri_builder.build_user_consent_uri.return_value = "http://example.com" + self.event_creator._consent_uri_builder = consent_uri_builder diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index 86c7ac350d..b58386994e 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -75,3 +75,43 @@ class EventFederationWorkerStoreTestCase(tests.unittest.TestCase): el = r[i] depth = el[2] self.assertLessEqual(5, depth) + + @defer.inlineCallbacks + def test_get_rooms_with_many_extremities(self): + room1 = "#room1" + room2 = "#room2" + room3 = "#room3" + + def insert_event(txn, i, room_id): + event_id = "$event_%i:local" % i + txn.execute( + ( + "INSERT INTO event_forward_extremities (room_id, event_id) " + "VALUES (?, ?)" + ), + (room_id, event_id), + ) + + for i in range(0, 20): + yield self.store.runInteraction("insert", insert_event, i, room1) + yield self.store.runInteraction("insert", insert_event, i, room2) + yield self.store.runInteraction("insert", insert_event, i, room3) + + # Test simple case + r = yield self.store.get_rooms_with_many_extremities(5, 5, []) + self.assertEqual(len(r), 3) + + # Does filter work? + + r = yield self.store.get_rooms_with_many_extremities(5, 5, [room1]) + self.assertTrue(room2 in r) + self.assertTrue(room3 in r) + self.assertEqual(len(r), 2) + + r = yield self.store.get_rooms_with_many_extremities(5, 5, [room1, room2]) + self.assertEqual(r, [room3]) + + # Does filter and limit work? + + r = yield self.store.get_rooms_with_many_extremities(5, 1, [room1]) + self.assertTrue(r == [room2] or r == [room3]) From 2927c6bc4c4e0c975a875d7eb5aa736b6abd66cd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 26 Sep 2019 12:29:59 +0100 Subject: [PATCH 16/21] bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 6766ef445c..ddfe9ec542 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -35,4 +35,4 @@ try: except ImportError: pass -__version__ = "1.3.1" +__version__ = "1.4.0rc1" From 1b23f991abb99c50908aca7c4ccfdea0c789c900 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 26 Sep 2019 12:30:10 +0100 Subject: [PATCH 17/21] Clarify upgrade notes ahead of 1.4.0 release --- UPGRADE.rst | 195 +++++++++++++++++++++++++++++++++---------- changelog.d/6027.doc | 1 + 2 files changed, 153 insertions(+), 43 deletions(-) create mode 100644 changelog.d/6027.doc diff --git a/UPGRADE.rst b/UPGRADE.rst index 4ede973a08..9562114d59 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -78,53 +78,161 @@ for example: Upgrading to v1.4.0 =================== -Config options --------------- +New custom templates +-------------------- -**Note: Registration by email address or phone number will not work in this release unless -some config options are changed from their defaults.** +If you have configured a custom template directory with the +``email.template_dir`` option, be aware that there are new templates regarding +registration and threepid management (see below) that must be included. -This is due to Synapse v1.4.0 now defaulting to sending registration and password reset tokens -itself. This is for security reasons as well as putting less reliance on identity servers. -However, currently Synapse only supports sending emails, and does not have support for -phone-based password reset or account registration. If Synapse is configured to handle these on -its own, phone-based password resets and registration will be disabled. For Synapse to send -emails, the ``email`` block of the config must be filled out. If not, then password resets and -registration via email will be disabled entirely. +* ``registration.html`` and ``registration.txt`` +* ``registration_success.html`` and ``registration_failure.html`` +* ``add_threepid.html`` and ``add_threepid.txt`` +* ``add_threepid_failure.html`` and ``add_threepid_success.html`` -This release also deprecates the ``email.trust_identity_server_for_password_resets`` option and -replaces it with the ``account_threepid_delegates`` dictionary. This option defines whether the -homeserver should delegate an external server (typically an `identity server -`_) to handle sending password reset or -registration messages via email and SMS. - -If ``email.trust_identity_server_for_password_resets`` is set to ``true``, and -``account_threepid_delegates.email`` is not set, then the first entry in -``trusted_third_party_id_servers`` will be used as the account threepid delegate for email. -This is to ensure compatibility with existing Synapse installs that set up external server -handling for these tasks before v1.4.0. If ``email.trust_identity_server_for_password_resets`` -is ``true`` and no trusted identity server domains are configured, Synapse will throw an error. - -If ``email.trust_identity_server_for_password_resets`` is ``false`` or absent and a threepid -type in ``account_threepid_delegates`` is not set to a domain, then Synapse will attempt to -send password reset and registration messages for that type. - -Email templates ---------------- - -If you have configured a custom template directory with the ``email.template_dir`` option, be -aware that there are new templates regarding registration. ``registration.html`` and -``registration.txt`` have been added and contain the content that is sent to a client upon -registering via an email address. - -``registration_success.html`` and ``registration_failure.html`` are also new HTML templates -that will be shown to the user when they click the link in their registration emai , either -showing them a success or failure page (assuming a redirect URL is not configured). - -Synapse will expect these files to exist inside the configured template directory. To view the -default templates, see `synapse/res/templates +Synapse will expect these files to exist inside the configured template +directory, and **will fail to start** if they are absent. +To view the default templates, see `synapse/res/templates `_. +3pid verification changes +------------------------- + +**Note: As of this release, users will be unable to add phone numbers or email +addresses to their accounts, without changes to the Synapse configuration. This +includes adding an email address during registration.** + +It is possible for a user to associate an email address or phone number +with their account, for a number of reasons: + +* for use when logging in, as an alternative to the user id. +* in the case of email, as an alternative contact to help with account recovery. +* in the case of email, to receive notifications of missed messages. + +Before an email address or phone number can be added to a user's account, +or before such an address is used to carry out a password-reset, Synapse must +confirm the operation with the owner of the email address or phone number. +It does this by sending an email or text giving the user a link or token to confirm +receipt. This process is known as '3pid verification'. ('3pid', or 'threepid', +stands for third-party identifier, and we use it to refer to external +identifiers such as email addresses and phone numbers.) + +Previous versions of Synapse delegated the task of 3pid verification to an +identity server by default. In most cases this server is ``vector.im`` or +``matrix.org``. + +In Synapse 1.4.0, for security and privacy reasons, the homeserver will no +longer delegate this task to an identity server by default. Instead, +the server administrator will need to explicitly decide how they would like the +verification messages to be sent. + +In the medium term, the ``vector.im`` and ``matrix.org`` identity servers will +disable support for delegated 3pid verification entirely. However, in order to +ease the transition, they will retain the capability for a limited +period. Delegated email verification will be disabled on Monday 2nd December +2019 (giving roughly 2 months notice). Disabling delegated SMS verification +will follow some time after that once SMS verification support lands in +Synapse. + +Once delegated 3pid verification support has been disabled in the ``vector.im`` and +``matrix.org`` identity servers, all Synapse versions that depend on those +instances will be unable to verify email and phone numbers through them. There +are no imminent plans to remove delegated 3pid verification from Sydent +generally. (Sydent is the identity server project that backs the ``vector.im`` and +``matrix.org`` instances). + +Email +~~~~~ +Following upgrade, to continue verifying email (e.g. as part of the +registration process), admins can either:- + +* Configure Synapse to use an email server. +* Run or choose an identity server which allows delegated email verification + and delegate to it. + +Configure SMTP in Synapse ++++++++++++++++++++++++++ + +To configure an SMTP server for Synapse, modify the configuration section +headed ``email``, and be sure to have at least the ``smtp_host, smtp_port`` +and ``notif_from`` fields filled out. + +You may also need to set ``smtp_user``, ``smtp_pass``, and +``require_transport_security``. + +See the `sample configuration file `_ for more details +on these settings. + +Delegate email to an identity server +++++++++++++++++++++++++++++++++++++ + +Some admins will wish to continue using email verification as part of the +registration process, but will not immediately have an appropriate SMTP server +at hand. + +To this end, we will continue to support email verification delegation via the +``vector.im`` and ``matrix.org`` identity servers for two months. Support for +delegated email verification will be disabled on Monday 2nd December. + +The ``account_threepid_delegates`` dictionary defines whether the homeserver +should delegate an external server (typically an `identity server +`_) to handle sending +confirmation messages via email and SMS. + +So to delegate email verification, in ``homeserver.yaml``, set +``account_threepid_delegates.email`` to the base URL of an identity server. For +example: + +.. code:: yaml + + account_threepid_delegates: + email: https://example.com # Delegate email sending to example.com + +Note that ``account_threepid_delegates.email`` replaces the deprecated +``email.trust_identity_server_for_password_resets``: if +``email.trust_identity_server_for_password_resets`` is set to ``true``, and +``account_threepid_delegates.email`` is not set, then the first entry in +``trusted_third_party_id_servers`` will be used as the +``account_threepid_delegate`` for email. This is to ensure compatibility with +existing Synapse installs that set up external server handling for these tasks +before v1.4.0. If ``email.trust_identity_server_for_password_resets`` is +``true`` and no trusted identity server domains are configured, Synapse will +report an error and refuse to start. + +If ``email.trust_identity_server_for_password_resets`` is ``false`` or absent +and no ``email`` delegate is configured in ``account_threepid_delegates``, +then Synapse will send email verification messages itself, using the configured +SMTP server (see above). +that type. + +Phone numbers +~~~~~~~~~~~~~ + +Synapse does not support phone-number verification itself, so the only way to +maintain the ability for users to add phone numbers to their accounts will be +by continuing to delegate phone number verification to the ``matrix.org`` and +``vector.im`` identity servers (or another identity server that supports SMS +sending). + +The ``account_threepid_delegates`` dictionary defines whether the homeserver +should delegate an external server (typically an `identity server +`_) to handle sending +confirmation messages via email and SMS. + +So to delegate phone number verification, in ``homeserver.yaml``, set +``account_threepid_delegates.msisdn`` to the base URL of an identity +server. For example: + +.. code:: yaml + + account_threepid_delegates: + msisdn: https://example.com # Delegate sms sending to example.com + +The ``matrix.org`` and ``vector.im`` identity servers will continue to support +delegated phone number verification via SMS until such time as it is possible +for admins to configure their servers to perform phone number verification +directly. More details will follow in a future release. + Rolling back to v1.3.1 ---------------------- @@ -140,7 +248,8 @@ v1.3.1, subject to the following: The room statistics are essentially unused in v1.3.1 (in future versions of Synapse, they will be used to populate the room directory), so there should be no loss of functionality. However, the statistics engine will write errors - to the logs, which can be avoided by setting the following in `homeserver.yaml`: + to the logs, which can be avoided by setting the following in + `homeserver.yaml`: .. code:: yaml diff --git a/changelog.d/6027.doc b/changelog.d/6027.doc new file mode 100644 index 0000000000..f0af68f3b1 --- /dev/null +++ b/changelog.d/6027.doc @@ -0,0 +1 @@ +Clarify Synapse 1.4.0 upgrade notes. From 8b8f8c7b3c6136ea777265fff8052afed2b7031e Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 26 Sep 2019 12:57:01 +0100 Subject: [PATCH 18/21] Explicitly log when a homeserver does not have a trusted key server configured (#6090) --- changelog.d/6090.feature | 1 + docs/sample_config.yaml | 14 ++++++++---- synapse/config/key.py | 48 ++++++++++++++++++++++++++++++++++++---- synapse/config/server.py | 16 +++++++------- 4 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 changelog.d/6090.feature diff --git a/changelog.d/6090.feature b/changelog.d/6090.feature new file mode 100644 index 0000000000..a6da448a1a --- /dev/null +++ b/changelog.d/6090.feature @@ -0,0 +1 @@ +Explicitly log when a homeserver does not have the 'trusted_key_servers' config field configured. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 8f801daf35..254e1b17b4 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1072,6 +1072,10 @@ signing_key_path: "CONFDIR/SERVERNAME.signing.key" # This setting supercedes an older setting named `perspectives`. The old format # is still supported for backwards-compatibility, but it is deprecated. # +# 'trusted_key_servers' defaults to matrix.org, but using it will generate a +# warning on start-up. To suppress this warning, set +# 'suppress_key_server_warning' to true. +# # Options for each entry in the list include: # # server_name: the name of the server. required. @@ -1096,11 +1100,13 @@ signing_key_path: "CONFDIR/SERVERNAME.signing.key" # "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" # - server_name: "my_other_trusted_server.example.com" # -# The default configuration is: -# -#trusted_key_servers: -# - server_name: "matrix.org" +trusted_key_servers: + - server_name: "matrix.org" + +# Uncomment the following to disable the warning that is emitted when the +# trusted_key_servers include 'matrix.org'. See above. # +#suppress_key_server_warning: true # The signing keys to use when acting as a trusted key server. If not specified # defaults to the server signing key. diff --git a/synapse/config/key.py b/synapse/config/key.py index ba2199bceb..f039f96e9c 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -50,6 +50,33 @@ and you should enable 'federation_verify_certificates' in your configuration. If you are *sure* you want to do this, set 'accept_keys_insecurely' on the trusted_key_server configuration.""" +TRUSTED_KEY_SERVER_NOT_CONFIGURED_WARN = """\ +Synapse requires that a list of trusted key servers are specified in order to +provide signing keys for other servers in the federation. + +This homeserver does not have a trusted key server configured in +homeserver.yaml and will fall back to the default of 'matrix.org'. + +Trusted key servers should be long-lived and stable which makes matrix.org a +good choice for many admins, but some admins may wish to choose another. To +suppress this warning, the admin should set 'trusted_key_servers' in +homeserver.yaml to their desired key server and 'suppress_key_server_warning' +to 'true'. + +In a future release the software-defined default will be removed entirely and +the trusted key server will be defined exclusively by the value of +'trusted_key_servers'. +--------------------------------------------------------------------------------""" + +TRUSTED_KEY_SERVER_CONFIGURED_AS_M_ORG_WARN = """\ +This server is configured to use 'matrix.org' as its trusted key server via the +'trusted_key_servers' config option. 'matrix.org' is a good choice for a key +server since it is long-lived, stable and trusted. However, some admins may +wish to use another server for this purpose. + +To suppress this warning and continue using 'matrix.org', admins should set +'suppress_key_server_warning' to 'true' in homeserver.yaml. +--------------------------------------------------------------------------------""" logger = logging.getLogger(__name__) @@ -85,6 +112,7 @@ class KeyConfig(Config): config.get("key_refresh_interval", "1d") ) + suppress_key_server_warning = config.get("suppress_key_server_warning", False) key_server_signing_keys_path = config.get("key_server_signing_keys_path") if key_server_signing_keys_path: self.key_server_signing_keys = self.read_signing_keys( @@ -95,6 +123,7 @@ class KeyConfig(Config): # if neither trusted_key_servers nor perspectives are given, use the default. if "perspectives" not in config and "trusted_key_servers" not in config: + logger.warn(TRUSTED_KEY_SERVER_NOT_CONFIGURED_WARN) key_servers = [{"server_name": "matrix.org"}] else: key_servers = config.get("trusted_key_servers", []) @@ -108,6 +137,11 @@ class KeyConfig(Config): # merge the 'perspectives' config into the 'trusted_key_servers' config. key_servers.extend(_perspectives_to_key_servers(config)) + if not suppress_key_server_warning and "matrix.org" in ( + s["server_name"] for s in key_servers + ): + logger.warning(TRUSTED_KEY_SERVER_CONFIGURED_AS_M_ORG_WARN) + # list of TrustedKeyServer objects self.key_servers = list( _parse_key_servers(key_servers, self.federation_verify_certificates) @@ -190,6 +224,10 @@ class KeyConfig(Config): # This setting supercedes an older setting named `perspectives`. The old format # is still supported for backwards-compatibility, but it is deprecated. # + # 'trusted_key_servers' defaults to matrix.org, but using it will generate a + # warning on start-up. To suppress this warning, set + # 'suppress_key_server_warning' to true. + # # Options for each entry in the list include: # # server_name: the name of the server. required. @@ -214,11 +252,13 @@ class KeyConfig(Config): # "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" # - server_name: "my_other_trusted_server.example.com" # - # The default configuration is: - # - #trusted_key_servers: - # - server_name: "matrix.org" + trusted_key_servers: + - server_name: "matrix.org" + + # Uncomment the following to disable the warning that is emitted when the + # trusted_key_servers include 'matrix.org'. See above. # + #suppress_key_server_warning: true # The signing keys to use when acting as a trusted key server. If not specified # defaults to the server signing key. diff --git a/synapse/config/server.py b/synapse/config/server.py index 9d3f1b5bfc..5ad7ee911d 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -48,6 +48,13 @@ ROOM_COMPLEXITY_TOO_GREAT = ( "to join this room." ) +METRICS_PORT_WARNING = """\ +The metrics_port configuration option is deprecated in Synapse 0.31 in favour of +a listener. Please see +https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md +on how to configure the new listener. +--------------------------------------------------------------------------------""" + class ServerConfig(Config): def read_config(self, config, **kwargs): @@ -341,14 +348,7 @@ class ServerConfig(Config): metrics_port = config.get("metrics_port") if metrics_port: - logger.warn( - ( - "The metrics_port configuration option is deprecated in Synapse 0.31 " - "in favour of a listener. Please see " - "http://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md" - " on how to configure the new listener." - ) - ) + logger.warning(METRICS_PORT_WARNING) self.listeners.append( { From 3fbca80a8da753e07dcf6c9539978c45c06cd1e1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 26 Sep 2019 12:34:35 +0100 Subject: [PATCH 19/21] changelog --- CHANGES.md | 156 +++++++++++++++++++++++++++++++++++++++ changelog.d/5633.bugfix | 1 - changelog.d/5680.misc | 1 - changelog.d/5771.feature | 1 - changelog.d/5776.misc | 1 - changelog.d/5835.feature | 1 - changelog.d/5844.misc | 1 - changelog.d/5845.feature | 1 - changelog.d/5849.doc | 1 - changelog.d/5850.feature | 1 - changelog.d/5852.feature | 1 - changelog.d/5853.feature | 1 - changelog.d/5855.misc | 1 - changelog.d/5856.feature | 1 - changelog.d/5857.bugfix | 1 - changelog.d/5859.feature | 1 - changelog.d/5860.misc | 1 - changelog.d/5863.bugfix | 1 - changelog.d/5864.feature | 1 - changelog.d/5868.feature | 1 - changelog.d/5875.misc | 1 - changelog.d/5876.feature | 1 - changelog.d/5877.removal | 1 - changelog.d/5878.feature | 1 - changelog.d/5884.feature | 1 - changelog.d/5885.bugfix | 1 - changelog.d/5886.misc | 1 - changelog.d/5892.misc | 1 - changelog.d/5893.misc | 1 - changelog.d/5894.misc | 1 - changelog.d/5895.feature | 1 - changelog.d/5896.misc | 1 - changelog.d/5897.feature | 1 - changelog.d/5900.feature | 1 - changelog.d/5902.feature | 1 - changelog.d/5904.feature | 1 - changelog.d/5906.feature | 1 - changelog.d/5909.misc | 1 - changelog.d/5911.misc | 1 - changelog.d/5914.feature | 1 - changelog.d/5915.bugfix | 1 - changelog.d/5920.bugfix | 1 - changelog.d/5922.misc | 1 - changelog.d/5926.misc | 1 - changelog.d/5931.misc | 1 - changelog.d/5934.feature | 1 - changelog.d/5938.misc | 1 - changelog.d/5940.feature | 1 - changelog.d/5943.misc | 1 - changelog.d/5953.misc | 1 - changelog.d/5962.misc | 1 - changelog.d/5963.misc | 1 - changelog.d/5964.feature | 1 - changelog.d/5966.bugfix | 1 - changelog.d/5967.bugfix | 1 - changelog.d/5969.feature | 1 - changelog.d/5970.docker | 1 - changelog.d/5971.bugfix | 1 - changelog.d/5972.misc | 1 - changelog.d/5974.feature | 1 - changelog.d/5975.misc | 1 - changelog.d/5979.feature | 1 - changelog.d/5980.feature | 1 - changelog.d/5981.feature | 1 - changelog.d/5982.bugfix | 1 - changelog.d/5983.feature | 1 - changelog.d/5984.bugfix | 1 - changelog.d/5985.feature | 1 - changelog.d/5986.feature | 1 - changelog.d/5988.bugfix | 1 - changelog.d/5989.misc | 1 - changelog.d/5991.bugfix | 1 - changelog.d/5992.feature | 1 - changelog.d/5993.feature | 1 - changelog.d/5994.feature | 1 - changelog.d/5995.bugfix | 1 - changelog.d/5996.bugfix | 1 - changelog.d/5998.bugfix | 1 - changelog.d/6000.feature | 1 - changelog.d/6003.misc | 1 - changelog.d/6004.bugfix | 1 - changelog.d/6005.feature | 1 - changelog.d/6009.misc | 1 - changelog.d/6010.misc | 1 - changelog.d/6011.feature | 1 - changelog.d/6012.feature | 1 - changelog.d/6013.misc | 1 - changelog.d/6015.feature | 1 - changelog.d/6016.misc | 1 - changelog.d/6017.misc | 1 - changelog.d/6020.bugfix | 1 - changelog.d/6023.misc | 1 - changelog.d/6024.bugfix | 1 - changelog.d/6025.bugfix | 1 - changelog.d/6026.feature | 1 - changelog.d/6027.doc | 1 - changelog.d/6028.feature | 1 - changelog.d/6029.bugfix | 1 - changelog.d/6032.misc | 1 - changelog.d/6037.feature | 1 - changelog.d/6042.feature | 1 - changelog.d/6043.feature | 1 - changelog.d/6044.feature | 1 - changelog.d/6047.misc | 2 - changelog.d/6049.doc | 1 - changelog.d/6050.doc | 1 - changelog.d/6053.bugfix | 1 - changelog.d/6056.bugfix | 1 - changelog.d/6058.docker | 1 - changelog.d/6059.bugfix | 1 - changelog.d/6062.bugfix | 1 - changelog.d/6063.bugfix | 1 - changelog.d/6064.misc | 1 - changelog.d/6067.feature | 1 - changelog.d/6069.bugfix | 1 - changelog.d/6072.misc | 1 - changelog.d/6073.feature | 1 - changelog.d/6074.feature | 1 - changelog.d/6075.misc | 1 - changelog.d/6078.feature | 1 - changelog.d/6079.feature | 1 - changelog.d/6082.feature | 1 - changelog.d/6089.misc | 1 - changelog.d/6090.feature | 1 - changelog.d/6092.bugfix | 1 - changelog.d/6097.bugfix | 1 - changelog.d/6098.feature | 1 - changelog.d/6099.misc | 1 - changelog.d/6104.bugfix | 1 - changelog.d/6105.misc | 1 - changelog.d/6106.misc | 1 - changelog.d/6107.bugfix | 1 - 132 files changed, 156 insertions(+), 132 deletions(-) delete mode 100644 changelog.d/5633.bugfix delete mode 100644 changelog.d/5680.misc delete mode 100644 changelog.d/5771.feature delete mode 100644 changelog.d/5776.misc delete mode 100644 changelog.d/5835.feature delete mode 100644 changelog.d/5844.misc delete mode 100644 changelog.d/5845.feature delete mode 100644 changelog.d/5849.doc delete mode 100644 changelog.d/5850.feature delete mode 100644 changelog.d/5852.feature delete mode 100644 changelog.d/5853.feature delete mode 100644 changelog.d/5855.misc delete mode 100644 changelog.d/5856.feature delete mode 100644 changelog.d/5857.bugfix delete mode 100644 changelog.d/5859.feature delete mode 100644 changelog.d/5860.misc delete mode 100644 changelog.d/5863.bugfix delete mode 100644 changelog.d/5864.feature delete mode 100644 changelog.d/5868.feature delete mode 100644 changelog.d/5875.misc delete mode 100644 changelog.d/5876.feature delete mode 100644 changelog.d/5877.removal delete mode 100644 changelog.d/5878.feature delete mode 100644 changelog.d/5884.feature delete mode 100644 changelog.d/5885.bugfix delete mode 100644 changelog.d/5886.misc delete mode 100644 changelog.d/5892.misc delete mode 100644 changelog.d/5893.misc delete mode 100644 changelog.d/5894.misc delete mode 100644 changelog.d/5895.feature delete mode 100644 changelog.d/5896.misc delete mode 100644 changelog.d/5897.feature delete mode 100644 changelog.d/5900.feature delete mode 100644 changelog.d/5902.feature delete mode 100644 changelog.d/5904.feature delete mode 100644 changelog.d/5906.feature delete mode 100644 changelog.d/5909.misc delete mode 100644 changelog.d/5911.misc delete mode 100644 changelog.d/5914.feature delete mode 100644 changelog.d/5915.bugfix delete mode 100644 changelog.d/5920.bugfix delete mode 100644 changelog.d/5922.misc delete mode 100644 changelog.d/5926.misc delete mode 100644 changelog.d/5931.misc delete mode 100644 changelog.d/5934.feature delete mode 100644 changelog.d/5938.misc delete mode 100644 changelog.d/5940.feature delete mode 100644 changelog.d/5943.misc delete mode 100644 changelog.d/5953.misc delete mode 100644 changelog.d/5962.misc delete mode 100644 changelog.d/5963.misc delete mode 100644 changelog.d/5964.feature delete mode 100644 changelog.d/5966.bugfix delete mode 100644 changelog.d/5967.bugfix delete mode 100644 changelog.d/5969.feature delete mode 100644 changelog.d/5970.docker delete mode 100644 changelog.d/5971.bugfix delete mode 100644 changelog.d/5972.misc delete mode 100644 changelog.d/5974.feature delete mode 100644 changelog.d/5975.misc delete mode 100644 changelog.d/5979.feature delete mode 100644 changelog.d/5980.feature delete mode 100644 changelog.d/5981.feature delete mode 100644 changelog.d/5982.bugfix delete mode 100644 changelog.d/5983.feature delete mode 100644 changelog.d/5984.bugfix delete mode 100644 changelog.d/5985.feature delete mode 100644 changelog.d/5986.feature delete mode 100644 changelog.d/5988.bugfix delete mode 100644 changelog.d/5989.misc delete mode 100644 changelog.d/5991.bugfix delete mode 100644 changelog.d/5992.feature delete mode 100644 changelog.d/5993.feature delete mode 100644 changelog.d/5994.feature delete mode 100644 changelog.d/5995.bugfix delete mode 100644 changelog.d/5996.bugfix delete mode 100644 changelog.d/5998.bugfix delete mode 100644 changelog.d/6000.feature delete mode 100644 changelog.d/6003.misc delete mode 100644 changelog.d/6004.bugfix delete mode 100644 changelog.d/6005.feature delete mode 100644 changelog.d/6009.misc delete mode 100644 changelog.d/6010.misc delete mode 100644 changelog.d/6011.feature delete mode 100644 changelog.d/6012.feature delete mode 100644 changelog.d/6013.misc delete mode 100644 changelog.d/6015.feature delete mode 100644 changelog.d/6016.misc delete mode 100644 changelog.d/6017.misc delete mode 100644 changelog.d/6020.bugfix delete mode 100644 changelog.d/6023.misc delete mode 100644 changelog.d/6024.bugfix delete mode 100644 changelog.d/6025.bugfix delete mode 100644 changelog.d/6026.feature delete mode 100644 changelog.d/6027.doc delete mode 100644 changelog.d/6028.feature delete mode 100644 changelog.d/6029.bugfix delete mode 100644 changelog.d/6032.misc delete mode 100644 changelog.d/6037.feature delete mode 100644 changelog.d/6042.feature delete mode 100644 changelog.d/6043.feature delete mode 100644 changelog.d/6044.feature delete mode 100644 changelog.d/6047.misc delete mode 100644 changelog.d/6049.doc delete mode 100644 changelog.d/6050.doc delete mode 100644 changelog.d/6053.bugfix delete mode 100644 changelog.d/6056.bugfix delete mode 100644 changelog.d/6058.docker delete mode 100644 changelog.d/6059.bugfix delete mode 100644 changelog.d/6062.bugfix delete mode 100644 changelog.d/6063.bugfix delete mode 100644 changelog.d/6064.misc delete mode 100644 changelog.d/6067.feature delete mode 100644 changelog.d/6069.bugfix delete mode 100644 changelog.d/6072.misc delete mode 100644 changelog.d/6073.feature delete mode 100644 changelog.d/6074.feature delete mode 100644 changelog.d/6075.misc delete mode 100644 changelog.d/6078.feature delete mode 100644 changelog.d/6079.feature delete mode 100644 changelog.d/6082.feature delete mode 100644 changelog.d/6089.misc delete mode 100644 changelog.d/6090.feature delete mode 100644 changelog.d/6092.bugfix delete mode 100644 changelog.d/6097.bugfix delete mode 100644 changelog.d/6098.feature delete mode 100644 changelog.d/6099.misc delete mode 100644 changelog.d/6104.bugfix delete mode 100644 changelog.d/6105.misc delete mode 100644 changelog.d/6106.misc delete mode 100644 changelog.d/6107.bugfix diff --git a/CHANGES.md b/CHANGES.md index f25c7d0c1a..9f610e4c12 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,159 @@ +Synapse 1.4.0rc1 (2019-09-26) +============================= + +Note that this release includes significant changes around 3pid +verification. Administrators are reminded to review the [upgrade notes](UPGRADE.rst##upgrading-to-v140). + +Features +-------- + +- Changes to 3pid verification: + - Add the ability to send registration emails from the homeserver rather than delegating to an identity server. ([\#5835](https://github.com/matrix-org/synapse/issues/5835), [\#5940](https://github.com/matrix-org/synapse/issues/5940), [\#5993](https://github.com/matrix-org/synapse/issues/5993), [\#5994](https://github.com/matrix-org/synapse/issues/5994), [\#5868](https://github.com/matrix-org/synapse/issues/5868)) + - Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`. ([\#5876](https://github.com/matrix-org/synapse/issues/5876), [\#5969](https://github.com/matrix-org/synapse/issues/5969), [\#6028](https://github.com/matrix-org/synapse/issues/6028)) + - Switch to using the v2 Identity Service `/lookup` API where available, with fallback to v1. (Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) plus id_access_token authentication for v2 Identity Service APIs from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140)). ([\#5897](https://github.com/matrix-org/synapse/issues/5897)) + - Remove `bind_email` and `bind_msisdn` parameters from `/register` ala [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140). ([\#5964](https://github.com/matrix-org/synapse/issues/5964)) + - Add `m.id_access_token` to `unstable_features` in `/versions` as per [MSC2264](https://github.com/matrix-org/matrix-doc/pull/2264). ([\#5974](https://github.com/matrix-org/synapse/issues/5974)) + - Use the v2 Identity Service API for 3PID invites. ([\#5979](https://github.com/matrix-org/synapse/issues/5979)) + - Add `POST /_matrix/client/unstable/account/3pid/unbind` endpoint from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140) for unbinding a 3PID from an identity server without removing it from the homeserver user account. ([\#5980](https://github.com/matrix-org/synapse/issues/5980), [\#6062](https://github.com/matrix-org/synapse/issues/6062)) +) + - Use `account_threepid_delegate.email` and `account_threepid_delegate.msisdn` for validating threepid sessions. ([\#6011](https://github.com/matrix-org/synapse/issues/6011)) + - Allow homeserver to handle or delegate email validation when adding an email to a user's account. ([\#6042](https://github.com/matrix-org/synapse/issues/6042)) + - Implement new Client Server API endpoints `/account/3pid/add` and `/account/3pid/bind` as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290). ([\#6043](https://github.com/matrix-org/synapse/issues/6043)) + - Add an unstable feature flag for separate add/bind 3pid APIs. ([\#6044](https://github.com/matrix-org/synapse/issues/6044)) + - Remove `bind` parameter from Client Server POST `/account` endpoint as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290/). ([\#6067](https://github.com/matrix-org/synapse/issues/6067)) + - Add `POST /add_threepid/msisdn/submit_token` endpoint for proxying submitToken on an account_threepid_handler. ([\#6078](https://github.com/matrix-org/synapse/issues/6078)) + - Add `submit_url` response parameter to `*/msisdn/requestToken` endpoints. ([\#6079](https://github.com/matrix-org/synapse/issues/6079)) + - Add `m.require_identity_server` flag to /version's unstable_features. ([\#5972](https://github.com/matrix-org/synapse/issues/5972)) +- Enhancements to OpenTracing support: + - Make OpenTracing work in worker mode. ([\#5771](https://github.com/matrix-org/synapse/issues/5771)) + - Pass OpenTracing contexts between servers when transmitting EDUs. ([\#5852](https://github.com/matrix-org/synapse/issues/5852)) + - OpenTracing for device list updates. ([\#5853](https://github.com/matrix-org/synapse/issues/5853)) + - Add a tag recording a request's authenticated entity and corresponding servlet in OpenTracing. ([\#5856](https://github.com/matrix-org/synapse/issues/5856)) + - Add minimum OpenTracing for client servlets. ([\#5983](https://github.com/matrix-org/synapse/issues/5983)) + - Check at setup that OpenTracing is installed if it's enabled in the config. ([\#5985](https://github.com/matrix-org/synapse/issues/5985)) + - Trace replication send times. ([\#5986](https://github.com/matrix-org/synapse/issues/5986)) + - Include missing OpenTracing contexts in outbout replication requests. ([\#5982](https://github.com/matrix-org/synapse/issues/5982)) + - Fix sending of EDUs when OpenTracing is enabled with an empty whitelist. ([\#5984](https://github.com/matrix-org/synapse/issues/5984)) + - Fix invalid references to None while OpenTracing if the log context slips. ([\#5988](https://github.com/matrix-org/synapse/issues/5988), [\#5991](https://github.com/matrix-org/synapse/issues/5991)) + - OpenTracing for room and e2e keys. ([\#5855](https://github.com/matrix-org/synapse/issues/5855)) + - Add OpenTracing span over HTTP push processing. ([\#6003](https://github.com/matrix-org/synapse/issues/6003)) +- Add an admin API to purge old rooms from the database. ([\#5845](https://github.com/matrix-org/synapse/issues/5845)) +- Retry well-known lookups if we have recently seen a valid well-known record for the server. ([\#5850](https://github.com/matrix-org/synapse/issues/5850)) +- Add support for filtered room-directory search requests over federation ([MSC2197](https://github.com/matrix-org/matrix-doc/pull/2197), in order to allow upcoming room directory query performance improvements. ([\#5859](https://github.com/matrix-org/synapse/issues/5859)) +- Correctly retry all hosts returned from SRV when we fail to connect. ([\#5864](https://github.com/matrix-org/synapse/issues/5864)) +- Add admin API endpoint for setting whether or not a user is a server administrator. ([\#5878](https://github.com/matrix-org/synapse/issues/5878)) +- Enable cleaning up extremities with dummy events by default to prevent undue build up of forward extremities. ([\#5884](https://github.com/matrix-org/synapse/issues/5884)) +- Add config option to sign remote key query responses with a separate key. ([\#5895](https://github.com/matrix-org/synapse/issues/5895)) +- Add support for config templating. ([\#5900](https://github.com/matrix-org/synapse/issues/5900)) +- Users with the type of "support" or "bot" are no longer required to consent. ([\#5902](https://github.com/matrix-org/synapse/issues/5902)) +- Let synctl accept a directory of config files. ([\#5904](https://github.com/matrix-org/synapse/issues/5904)) +- Increase max display name size to 256. ([\#5906](https://github.com/matrix-org/synapse/issues/5906)) +- Add admin API endpoint for getting whether or not a user is a server administrator. ([\#5914](https://github.com/matrix-org/synapse/issues/5914)) +- Redact events in the database that have been redacted for a month. ([\#5934](https://github.com/matrix-org/synapse/issues/5934)) +- New prometheus metrics: + - `synapse_federation_known_servers`: represents the total number of servers your server knows about (i.e. is in rooms with), including itself. Enable by setting `metrics_flags.known_servers` to True in the configuration.([\#5981](https://github.com/matrix-org/synapse/issues/5981)) + - `synapse_build_info`: exposes the Python version, OS version, and Synapse version of the running server. ([\#6005](https://github.com/matrix-org/synapse/issues/6005)) +- Give appropriate exit codes when synctl fails. ([\#5992](https://github.com/matrix-org/synapse/issues/5992)) +- Apply the federation blacklist to requests to identity servers. ([\#6000](https://github.com/matrix-org/synapse/issues/6000)) +- Add `report_stats_endpoint` option to configure where stats are reported to, if enabled. Contributed by @Sorunome. ([\#6012](https://github.com/matrix-org/synapse/issues/6012)) +- Add config option to increase ratelimits for room admins redacting messages. ([\#6015](https://github.com/matrix-org/synapse/issues/6015)) +- Stop sending federation transactions to servers which have been down for a long time. ([\#6026](https://github.com/matrix-org/synapse/issues/6026)) +- Make the process for mapping SAML2 users to matrix IDs more flexible. ([\#6037](https://github.com/matrix-org/synapse/issues/6037)) +- Return a clearer error message when a timeout occurs when attempting to contact an identity server. ([\#6073](https://github.com/matrix-org/synapse/issues/6073)) +- Prevent password reset's submit_token endpoint from accepting trailing slashes. ([\#6074](https://github.com/matrix-org/synapse/issues/6074)) +- Return 403 on `/register/available` if registration has been disabled. ([\#6082](https://github.com/matrix-org/synapse/issues/6082)) +- Explicitly log when a homeserver does not have the `trusted_key_servers` config field configured. ([\#6090](https://github.com/matrix-org/synapse/issues/6090)) +- Add support for pruning old rows in `user_ips` table. ([\#6098](https://github.com/matrix-org/synapse/issues/6098)) + +Bugfixes +-------- + +- Don't create broken room when `power_level_content_override.users` does not contain `creator_id`. ([\#5633](https://github.com/matrix-org/synapse/issues/5633)) +- Fix database index so that different backup versions can have the same sessions. ([\#5857](https://github.com/matrix-org/synapse/issues/5857)) +- Fix Synapse looking for config options `password_reset_failure_template` and `password_reset_success_template`, when they are actually `password_reset_template_failure_html`, `password_reset_template_success_html`. ([\#5863](https://github.com/matrix-org/synapse/issues/5863)) +- Fix stack overflow when recovering an appservice which had an outage. ([\#5885](https://github.com/matrix-org/synapse/issues/5885)) +- Fix error message which referred to `public_base_url` instead of `public_baseurl`. Thanks to @aaronraimist for the fix! ([\#5909](https://github.com/matrix-org/synapse/issues/5909)) +- Fix 404 for thumbnail download when `dynamic_thumbnails` is `false` and the thumbnail was dynamically generated. Fix reported by rkfg. ([\#5915](https://github.com/matrix-org/synapse/issues/5915)) +- Fix a cache-invalidation bug for worker-based deployments. ([\#5920](https://github.com/matrix-org/synapse/issues/5920)) +- Fix admin API for listing media in a room not being available with an external media repo. ([\#5966](https://github.com/matrix-org/synapse/issues/5966)) +- Fix list media admin API always returning an error. ([\#5967](https://github.com/matrix-org/synapse/issues/5967)) +- Fix room and user stats tracking. ([\#5971](https://github.com/matrix-org/synapse/issues/5971), [\#5998](https://github.com/matrix-org/synapse/issues/5998), [\#6029](https://github.com/matrix-org/synapse/issues/6029)) +- Return a `M_MISSING_PARAM` if `sid` is not provided to `/account/3pid`. ([\#5995](https://github.com/matrix-org/synapse/issues/5995)) +- `federation_certificate_verification_whitelist` now will not cause `TypeErrors` to be raised (a regression in 1.3). Additionally, it now supports internationalised domain names in their non-canonical representation. ([\#5996](https://github.com/matrix-org/synapse/issues/5996)) +- Only count real users when checking for auto-creation of auto-join room. ([\#6004](https://github.com/matrix-org/synapse/issues/6004)) +- Ensure support users can be registered even if MAU limit is reached. ([\#6020](https://github.com/matrix-org/synapse/issues/6020)) +- Fix bug where login error was shown incorrectly on SSO fallback login. ([\#6024](https://github.com/matrix-org/synapse/issues/6024)) +- Fix bug in calculating the federation retry backoff period. ([\#6025](https://github.com/matrix-org/synapse/issues/6025)) +- Prevent exceptions being logged when extremity-cleanup events fail due to lack of user consent to the terms of service. ([\#6053](https://github.com/matrix-org/synapse/issues/6053)) +- Remove POST method from password-reset `submit_token` endpoint until we implement `submit_url` functionality. ([\#6056](https://github.com/matrix-org/synapse/issues/6056)) +- Fix logcontext spam on non-Linux platforms. ([\#6059](https://github.com/matrix-org/synapse/issues/6059)) +- Ensure query parameters in email validation links are URL-encoded. ([\#6063](https://github.com/matrix-org/synapse/issues/6063)) +- Fix a bug which caused SAML attribute maps to be overridden by defaults. ([\#6069](https://github.com/matrix-org/synapse/issues/6069)) +- Fix the logged number of updated items for the users_set_deactivated_flag background update. ([\#6092](https://github.com/matrix-org/synapse/issues/6092)) +- Add sid to `next_link` for email validation. ([\#6097](https://github.com/matrix-org/synapse/issues/6097)) +- Threepid validity checks on msisdns should not be dependent on `threepid_behaviour_email`. ([\#6104](https://github.com/matrix-org/synapse/issues/6104)) +- Ensure that servers which are not configured to support email address verification do not offer it in the registration flows. ([\#6107](https://github.com/matrix-org/synapse/issues/6107)) + + +Updates to the Docker image +--------------------------- + +- Avoid changing UID/GID if they are already correct. ([\#5970](https://github.com/matrix-org/synapse/issues/5970)) +- Provide SYNAPSE_WORKER envvar to specify python module. ([\#6058](https://github.com/matrix-org/synapse/issues/6058)) + + +Improved Documentation +---------------------- + +- Convert documentation to markdown (from rst) ([\#5849](https://github.com/matrix-org/synapse/issues/5849)) +- Update `INSTALL.md` to say that Python 2 is no longer supported. ([\#5953](https://github.com/matrix-org/synapse/issues/5953)) +- Add developer documentation for using SAML2. ([\#6032](https://github.com/matrix-org/synapse/issues/6032)) +- Add some notes on rolling back to v1.3.1. ([\#6049](https://github.com/matrix-org/synapse/issues/6049)) +- Update the upgrade notes. ([\#6050](https://github.com/matrix-org/synapse/issues/6050)) + + +Deprecations and Removals +------------------------- + +- Remove shared-secret registration from `/_matrix/client/r0/register` endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5877](https://github.com/matrix-org/synapse/issues/5877)) +- Deprecate the `trusted_third_party_id_servers` option. ([\#5875](https://github.com/matrix-org/synapse/issues/5875)) + + +Internal Changes +---------------- + +- Lay the groundwork for structured logging output. ([\#5680](https://github.com/matrix-org/synapse/issues/5680)) +- Retry well-known lookup before the cache expires, giving a grace period where the remote well-known can be down but we still use the old result. ([\#5844](https://github.com/matrix-org/synapse/issues/5844)) +- Remove log line for debugging issue #5407. ([\#5860](https://github.com/matrix-org/synapse/issues/5860)) +- Refactor the Appservice scheduler code. ([\#5886](https://github.com/matrix-org/synapse/issues/5886)) +- Compatibility with v2 Identity Service APIs other than /lookup. ([\#5892](https://github.com/matrix-org/synapse/issues/5892), [\#6013](https://github.com/matrix-org/synapse/issues/6013)) +- Stop populating some unused tables. ([\#5893](https://github.com/matrix-org/synapse/issues/5893), [\#6047](https://github.com/matrix-org/synapse/issues/6047)) +- Add missing index on users_in_public_rooms to improve the performance of directory queries. ([\#5894](https://github.com/matrix-org/synapse/issues/5894)) +- Improve the logging when we have an error when fetching signing keys. ([\#5896](https://github.com/matrix-org/synapse/issues/5896)) +- Add support for database engine-specific schema deltas, based on file extension. ([\#5911](https://github.com/matrix-org/synapse/issues/5911)) +- Update Buildkite pipeline to use plugins instead of buildkite-agent commands. ([\#5922](https://github.com/matrix-org/synapse/issues/5922)) +- Add link in sample config to the logging config schema. ([\#5926](https://github.com/matrix-org/synapse/issues/5926)) +- Remove unnecessary parentheses in return statements. ([\#5931](https://github.com/matrix-org/synapse/issues/5931)) +- Remove unused `jenkins/prepare_sytest.sh` file. ([\#5938](https://github.com/matrix-org/synapse/issues/5938)) +- Move Buildkite pipeline config to the pipelines repo. ([\#5943](https://github.com/matrix-org/synapse/issues/5943)) +- Remove unnecessary return statements in the codebase which were the result of a regex run. ([\#5962](https://github.com/matrix-org/synapse/issues/5962)) +- Remove left-over methods from v1 registration API. ([\#5963](https://github.com/matrix-org/synapse/issues/5963)) +- Cleanup event auth type initialisation. ([\#5975](https://github.com/matrix-org/synapse/issues/5975)) +- Clean up dependency checking at setup. ([\#5989](https://github.com/matrix-org/synapse/issues/5989)) +- Update OpenTracing docs to use the unified `trace` method. ([\#5776](https://github.com/matrix-org/synapse/issues/5776)) +- Small refactor of function arguments and docstrings in` RoomMemberHandler`. ([\#6009](https://github.com/matrix-org/synapse/issues/6009)) +- Remove unused `origin` argument on `FederationHandler.add_display_name_to_third_party_invite`. ([\#6010](https://github.com/matrix-org/synapse/issues/6010)) +- Add a `failure_ts` column to the `destinations` database table. ([\#6016](https://github.com/matrix-org/synapse/issues/6016), [\#6072](https://github.com/matrix-org/synapse/issues/6072)) +- Clean up some code in the retry logic. ([\#6017](https://github.com/matrix-org/synapse/issues/6017)) +- Fix the structured logging tests stomping on the global log configuration for subsequent tests. ([\#6023](https://github.com/matrix-org/synapse/issues/6023)) +- Clean up the sample config for SAML authentication. ([\#6064](https://github.com/matrix-org/synapse/issues/6064)) +- Change mailer logging to reflect Synapse doesn't just do chat notifications by email now. ([\#6075](https://github.com/matrix-org/synapse/issues/6075)) +- Move last-seen info into devices table. ([\#6089](https://github.com/matrix-org/synapse/issues/6089)) +- Remove unused parameter to `get_user_id_by_threepid`. ([\#6099](https://github.com/matrix-org/synapse/issues/6099)) +- Refactor the user-interactive auth handling. ([\#6105](https://github.com/matrix-org/synapse/issues/6105)) +- Refactor code for calculating registration flows. ([\#6106](https://github.com/matrix-org/synapse/issues/6106)) + + Synapse 1.3.1 (2019-08-17) ========================== diff --git a/changelog.d/5633.bugfix b/changelog.d/5633.bugfix deleted file mode 100644 index b2ff803b9d..0000000000 --- a/changelog.d/5633.bugfix +++ /dev/null @@ -1 +0,0 @@ -Don't create broken room when power_level_content_override.users does not contain creator_id. \ No newline at end of file diff --git a/changelog.d/5680.misc b/changelog.d/5680.misc deleted file mode 100644 index 46a403a188..0000000000 --- a/changelog.d/5680.misc +++ /dev/null @@ -1 +0,0 @@ -Lay the groundwork for structured logging output. diff --git a/changelog.d/5771.feature b/changelog.d/5771.feature deleted file mode 100644 index f2f4de1fdd..0000000000 --- a/changelog.d/5771.feature +++ /dev/null @@ -1 +0,0 @@ -Make Opentracing work in worker mode. diff --git a/changelog.d/5776.misc b/changelog.d/5776.misc deleted file mode 100644 index 1fb1b9c152..0000000000 --- a/changelog.d/5776.misc +++ /dev/null @@ -1 +0,0 @@ -Update opentracing docs to use the unified `trace` method. diff --git a/changelog.d/5835.feature b/changelog.d/5835.feature deleted file mode 100644 index 3e8bf5068d..0000000000 --- a/changelog.d/5835.feature +++ /dev/null @@ -1 +0,0 @@ -Add the ability to send registration emails from the homeserver rather than delegating to an identity server. diff --git a/changelog.d/5844.misc b/changelog.d/5844.misc deleted file mode 100644 index a0826af0d2..0000000000 --- a/changelog.d/5844.misc +++ /dev/null @@ -1 +0,0 @@ -Retry well-known lookup before the cache expires, giving a grace period where the remote well-known can be down but we still use the old result. diff --git a/changelog.d/5845.feature b/changelog.d/5845.feature deleted file mode 100644 index 7b0dc9a95e..0000000000 --- a/changelog.d/5845.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin API to purge old rooms from the database. diff --git a/changelog.d/5849.doc b/changelog.d/5849.doc deleted file mode 100644 index fbe62e8633..0000000000 --- a/changelog.d/5849.doc +++ /dev/null @@ -1 +0,0 @@ -Convert documentation to markdown (from rst) diff --git a/changelog.d/5850.feature b/changelog.d/5850.feature deleted file mode 100644 index b565929a54..0000000000 --- a/changelog.d/5850.feature +++ /dev/null @@ -1 +0,0 @@ -Add retry to well-known lookups if we have recently seen a valid well-known record for the server. diff --git a/changelog.d/5852.feature b/changelog.d/5852.feature deleted file mode 100644 index 4a0fc6c542..0000000000 --- a/changelog.d/5852.feature +++ /dev/null @@ -1 +0,0 @@ -Pass opentracing contexts between servers when transmitting EDUs. diff --git a/changelog.d/5853.feature b/changelog.d/5853.feature deleted file mode 100644 index 80a04ae2ee..0000000000 --- a/changelog.d/5853.feature +++ /dev/null @@ -1 +0,0 @@ -Opentracing for device list updates. diff --git a/changelog.d/5855.misc b/changelog.d/5855.misc deleted file mode 100644 index 32db7fbe37..0000000000 --- a/changelog.d/5855.misc +++ /dev/null @@ -1 +0,0 @@ -Opentracing for room and e2e keys. diff --git a/changelog.d/5856.feature b/changelog.d/5856.feature deleted file mode 100644 index f4310b9244..0000000000 --- a/changelog.d/5856.feature +++ /dev/null @@ -1 +0,0 @@ -Add a tag recording a request's authenticated entity and corresponding servlet in opentracing. diff --git a/changelog.d/5857.bugfix b/changelog.d/5857.bugfix deleted file mode 100644 index 008799ccbb..0000000000 --- a/changelog.d/5857.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix database index so that different backup versions can have the same sessions. diff --git a/changelog.d/5859.feature b/changelog.d/5859.feature deleted file mode 100644 index 52df7fc81b..0000000000 --- a/changelog.d/5859.feature +++ /dev/null @@ -1 +0,0 @@ -Add unstable support for MSC2197 (filtered search requests over federation), in order to allow upcoming room directory query performance improvements. diff --git a/changelog.d/5860.misc b/changelog.d/5860.misc deleted file mode 100644 index f9960b17b4..0000000000 --- a/changelog.d/5860.misc +++ /dev/null @@ -1 +0,0 @@ -Remove log line for debugging issue #5407. diff --git a/changelog.d/5863.bugfix b/changelog.d/5863.bugfix deleted file mode 100644 index bceae5be67..0000000000 --- a/changelog.d/5863.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix Synapse looking for config options `password_reset_failure_template` and `password_reset_success_template`, when they are actually `password_reset_template_failure_html`, `password_reset_template_success_html`. diff --git a/changelog.d/5864.feature b/changelog.d/5864.feature deleted file mode 100644 index 40ac11db64..0000000000 --- a/changelog.d/5864.feature +++ /dev/null @@ -1 +0,0 @@ -Correctly retry all hosts returned from SRV when we fail to connect. diff --git a/changelog.d/5868.feature b/changelog.d/5868.feature deleted file mode 100644 index 69605c1ae1..0000000000 --- a/changelog.d/5868.feature +++ /dev/null @@ -1 +0,0 @@ -Add `m.require_identity_server` key to `/versions`'s `unstable_features` section. \ No newline at end of file diff --git a/changelog.d/5875.misc b/changelog.d/5875.misc deleted file mode 100644 index e188c28d2f..0000000000 --- a/changelog.d/5875.misc +++ /dev/null @@ -1 +0,0 @@ -Deprecate the `trusted_third_party_id_servers` option. \ No newline at end of file diff --git a/changelog.d/5876.feature b/changelog.d/5876.feature deleted file mode 100644 index df88193fbd..0000000000 --- a/changelog.d/5876.feature +++ /dev/null @@ -1 +0,0 @@ -Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`. \ No newline at end of file diff --git a/changelog.d/5877.removal b/changelog.d/5877.removal deleted file mode 100644 index b6d84fb401..0000000000 --- a/changelog.d/5877.removal +++ /dev/null @@ -1 +0,0 @@ -Remove shared secret registration from client/r0/register endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. diff --git a/changelog.d/5878.feature b/changelog.d/5878.feature deleted file mode 100644 index d9d6df880e..0000000000 --- a/changelog.d/5878.feature +++ /dev/null @@ -1 +0,0 @@ -Add admin API endpoint for setting whether or not a user is a server administrator. diff --git a/changelog.d/5884.feature b/changelog.d/5884.feature deleted file mode 100644 index bfd0489392..0000000000 --- a/changelog.d/5884.feature +++ /dev/null @@ -1 +0,0 @@ -Enable cleaning up extremities with dummy events by default to prevent undue build up of forward extremities. diff --git a/changelog.d/5885.bugfix b/changelog.d/5885.bugfix deleted file mode 100644 index 411d925fd4..0000000000 --- a/changelog.d/5885.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix stack overflow when recovering an appservice which had an outage. diff --git a/changelog.d/5886.misc b/changelog.d/5886.misc deleted file mode 100644 index 22adba3d85..0000000000 --- a/changelog.d/5886.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor the Appservice scheduler code. diff --git a/changelog.d/5892.misc b/changelog.d/5892.misc deleted file mode 100644 index 939fe8c655..0000000000 --- a/changelog.d/5892.misc +++ /dev/null @@ -1 +0,0 @@ -Compatibility with v2 Identity Service APIs other than /lookup. \ No newline at end of file diff --git a/changelog.d/5893.misc b/changelog.d/5893.misc deleted file mode 100644 index 5ef171cb3e..0000000000 --- a/changelog.d/5893.misc +++ /dev/null @@ -1 +0,0 @@ -Stop populating some unused tables. diff --git a/changelog.d/5894.misc b/changelog.d/5894.misc deleted file mode 100644 index fca4485ff7..0000000000 --- a/changelog.d/5894.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing index on users_in_public_rooms to improve the performance of directory queries. diff --git a/changelog.d/5895.feature b/changelog.d/5895.feature deleted file mode 100644 index c394a3772c..0000000000 --- a/changelog.d/5895.feature +++ /dev/null @@ -1 +0,0 @@ -Add config option to sign remote key query responses with a separate key. diff --git a/changelog.d/5896.misc b/changelog.d/5896.misc deleted file mode 100644 index ed47c747bd..0000000000 --- a/changelog.d/5896.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the logging when we have an error when fetching signing keys. diff --git a/changelog.d/5897.feature b/changelog.d/5897.feature deleted file mode 100644 index 1557e559e8..0000000000 --- a/changelog.d/5897.feature +++ /dev/null @@ -1 +0,0 @@ -Switch to using the v2 Identity Service `/lookup` API where available, with fallback to v1. (Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) plus id_access_token authentication for v2 Identity Service APIs from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140)). diff --git a/changelog.d/5900.feature b/changelog.d/5900.feature deleted file mode 100644 index b62d88a76b..0000000000 --- a/changelog.d/5900.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for config templating. diff --git a/changelog.d/5902.feature b/changelog.d/5902.feature deleted file mode 100644 index 0660f65cfa..0000000000 --- a/changelog.d/5902.feature +++ /dev/null @@ -1 +0,0 @@ -Users with the type of "support" or "bot" are no longer required to consent. \ No newline at end of file diff --git a/changelog.d/5904.feature b/changelog.d/5904.feature deleted file mode 100644 index 43b5304f39..0000000000 --- a/changelog.d/5904.feature +++ /dev/null @@ -1 +0,0 @@ -Let synctl accept a directory of config files. diff --git a/changelog.d/5906.feature b/changelog.d/5906.feature deleted file mode 100644 index 7c789510a6..0000000000 --- a/changelog.d/5906.feature +++ /dev/null @@ -1 +0,0 @@ -Increase max display name size to 256. diff --git a/changelog.d/5909.misc b/changelog.d/5909.misc deleted file mode 100644 index 03d0c4367b..0000000000 --- a/changelog.d/5909.misc +++ /dev/null @@ -1 +0,0 @@ -Fix error message which referred to public_base_url instead of public_baseurl. Thanks to @aaronraimist for the fix! diff --git a/changelog.d/5911.misc b/changelog.d/5911.misc deleted file mode 100644 index fe5a8fd59c..0000000000 --- a/changelog.d/5911.misc +++ /dev/null @@ -1 +0,0 @@ -Add support for database engine-specific schema deltas, based on file extension. \ No newline at end of file diff --git a/changelog.d/5914.feature b/changelog.d/5914.feature deleted file mode 100644 index 85c7bf5963..0000000000 --- a/changelog.d/5914.feature +++ /dev/null @@ -1 +0,0 @@ -Add admin API endpoint for getting whether or not a user is a server administrator. diff --git a/changelog.d/5915.bugfix b/changelog.d/5915.bugfix deleted file mode 100644 index bf5b99fedc..0000000000 --- a/changelog.d/5915.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 404 for thumbnail download when `dynamic_thumbnails` is `false` and the thumbnail was dynamically generated. Fix reported by rkfg. diff --git a/changelog.d/5920.bugfix b/changelog.d/5920.bugfix deleted file mode 100644 index e45eb0ffee..0000000000 --- a/changelog.d/5920.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a cache-invalidation bug for worker-based deployments. diff --git a/changelog.d/5922.misc b/changelog.d/5922.misc deleted file mode 100644 index 2cc864897e..0000000000 --- a/changelog.d/5922.misc +++ /dev/null @@ -1 +0,0 @@ -Update Buildkite pipeline to use plugins instead of buildkite-agent commands. diff --git a/changelog.d/5926.misc b/changelog.d/5926.misc deleted file mode 100644 index 4383c302ec..0000000000 --- a/changelog.d/5926.misc +++ /dev/null @@ -1 +0,0 @@ -Add link in sample config to the logging config schema. diff --git a/changelog.d/5931.misc b/changelog.d/5931.misc deleted file mode 100644 index ac8e74f5b9..0000000000 --- a/changelog.d/5931.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unnecessary parentheses in return statements. \ No newline at end of file diff --git a/changelog.d/5934.feature b/changelog.d/5934.feature deleted file mode 100644 index eae969a52a..0000000000 --- a/changelog.d/5934.feature +++ /dev/null @@ -1 +0,0 @@ -Redact events in the database that have been redacted for a month. diff --git a/changelog.d/5938.misc b/changelog.d/5938.misc deleted file mode 100644 index b5a3b6ee3b..0000000000 --- a/changelog.d/5938.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused jenkins/prepare_sytest.sh file. diff --git a/changelog.d/5940.feature b/changelog.d/5940.feature deleted file mode 100644 index 5b69b97fe7..0000000000 --- a/changelog.d/5940.feature +++ /dev/null @@ -1 +0,0 @@ -Add the ability to send registration emails from the homeserver rather than delegating to an identity server. \ No newline at end of file diff --git a/changelog.d/5943.misc b/changelog.d/5943.misc deleted file mode 100644 index 6545e1244a..0000000000 --- a/changelog.d/5943.misc +++ /dev/null @@ -1 +0,0 @@ -Move Buildkite pipeline config to the pipelines repo. diff --git a/changelog.d/5953.misc b/changelog.d/5953.misc deleted file mode 100644 index 38e885f42a..0000000000 --- a/changelog.d/5953.misc +++ /dev/null @@ -1 +0,0 @@ -Update INSTALL.md to say that Python 2 is no longer supported. diff --git a/changelog.d/5962.misc b/changelog.d/5962.misc deleted file mode 100644 index d97d376c36..0000000000 --- a/changelog.d/5962.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unnecessary return statements in the codebase which were the result of a regex run. \ No newline at end of file diff --git a/changelog.d/5963.misc b/changelog.d/5963.misc deleted file mode 100644 index 0d6c3c3d65..0000000000 --- a/changelog.d/5963.misc +++ /dev/null @@ -1 +0,0 @@ -Remove left-over methods from C/S registration API. \ No newline at end of file diff --git a/changelog.d/5964.feature b/changelog.d/5964.feature deleted file mode 100644 index 273c9df026..0000000000 --- a/changelog.d/5964.feature +++ /dev/null @@ -1 +0,0 @@ -Remove `bind_email` and `bind_msisdn` parameters from /register ala MSC2140. \ No newline at end of file diff --git a/changelog.d/5966.bugfix b/changelog.d/5966.bugfix deleted file mode 100644 index b8ef5a7819..0000000000 --- a/changelog.d/5966.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix admin API for listing media in a room not being available with an external media repo. diff --git a/changelog.d/5967.bugfix b/changelog.d/5967.bugfix deleted file mode 100644 index 8d7bf5c2e9..0000000000 --- a/changelog.d/5967.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix list media admin API always returning an error. diff --git a/changelog.d/5969.feature b/changelog.d/5969.feature deleted file mode 100644 index cf603fa0c6..0000000000 --- a/changelog.d/5969.feature +++ /dev/null @@ -1 +0,0 @@ -Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`. diff --git a/changelog.d/5970.docker b/changelog.d/5970.docker deleted file mode 100644 index c9d04da9cd..0000000000 --- a/changelog.d/5970.docker +++ /dev/null @@ -1 +0,0 @@ -Avoid changing UID/GID if they are already correct. diff --git a/changelog.d/5971.bugfix b/changelog.d/5971.bugfix deleted file mode 100644 index 9ea095103b..0000000000 --- a/changelog.d/5971.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix room and user stats tracking. diff --git a/changelog.d/5972.misc b/changelog.d/5972.misc deleted file mode 100644 index 1dc217e899..0000000000 --- a/changelog.d/5972.misc +++ /dev/null @@ -1 +0,0 @@ -Add m.require_identity_server flag to /version's unstable_features. \ No newline at end of file diff --git a/changelog.d/5974.feature b/changelog.d/5974.feature deleted file mode 100644 index 387a444fc4..0000000000 --- a/changelog.d/5974.feature +++ /dev/null @@ -1 +0,0 @@ -Add m.id_access_token to unstable_features in /versions as per MSC2264. \ No newline at end of file diff --git a/changelog.d/5975.misc b/changelog.d/5975.misc deleted file mode 100644 index 5fcd229b89..0000000000 --- a/changelog.d/5975.misc +++ /dev/null @@ -1 +0,0 @@ -Cleanup event auth type initialisation. \ No newline at end of file diff --git a/changelog.d/5979.feature b/changelog.d/5979.feature deleted file mode 100644 index 94888aa2d3..0000000000 --- a/changelog.d/5979.feature +++ /dev/null @@ -1 +0,0 @@ -Use the v2 Identity Service API for 3PID invites. \ No newline at end of file diff --git a/changelog.d/5980.feature b/changelog.d/5980.feature deleted file mode 100644 index e20117cf1c..0000000000 --- a/changelog.d/5980.feature +++ /dev/null @@ -1 +0,0 @@ -Add POST /_matrix/client/unstable/account/3pid/unbind endpoint from MSC2140 for unbinding a 3PID from an identity server without removing it from the homeserver user account. diff --git a/changelog.d/5981.feature b/changelog.d/5981.feature deleted file mode 100644 index e39514273d..0000000000 --- a/changelog.d/5981.feature +++ /dev/null @@ -1 +0,0 @@ -Setting metrics_flags.known_servers to True in the configuration will publish the synapse_federation_known_servers metric over Prometheus. This represents the total number of servers your server knows about (i.e. is in rooms with), including itself. diff --git a/changelog.d/5982.bugfix b/changelog.d/5982.bugfix deleted file mode 100644 index 3ea281a3a0..0000000000 --- a/changelog.d/5982.bugfix +++ /dev/null @@ -1 +0,0 @@ -Include missing opentracing contexts in outbout replication requests. diff --git a/changelog.d/5983.feature b/changelog.d/5983.feature deleted file mode 100644 index aa23ee6dcd..0000000000 --- a/changelog.d/5983.feature +++ /dev/null @@ -1 +0,0 @@ -Add minimum opentracing for client servlets. diff --git a/changelog.d/5984.bugfix b/changelog.d/5984.bugfix deleted file mode 100644 index 3387bf82bb..0000000000 --- a/changelog.d/5984.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix sending of EDUs when opentracing is enabled with an empty whitelist. diff --git a/changelog.d/5985.feature b/changelog.d/5985.feature deleted file mode 100644 index e5e29504af..0000000000 --- a/changelog.d/5985.feature +++ /dev/null @@ -1 +0,0 @@ -Check at setup that opentracing is installed if it's enabled in the config. diff --git a/changelog.d/5986.feature b/changelog.d/5986.feature deleted file mode 100644 index f56aec1b32..0000000000 --- a/changelog.d/5986.feature +++ /dev/null @@ -1 +0,0 @@ -Trace replication send times. diff --git a/changelog.d/5988.bugfix b/changelog.d/5988.bugfix deleted file mode 100644 index 5c3597cb53..0000000000 --- a/changelog.d/5988.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix invalid references to None while opentracing if the log context slips. diff --git a/changelog.d/5989.misc b/changelog.d/5989.misc deleted file mode 100644 index 9f2525fd3e..0000000000 --- a/changelog.d/5989.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up dependency checking at setup. diff --git a/changelog.d/5991.bugfix b/changelog.d/5991.bugfix deleted file mode 100644 index 5c3597cb53..0000000000 --- a/changelog.d/5991.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix invalid references to None while opentracing if the log context slips. diff --git a/changelog.d/5992.feature b/changelog.d/5992.feature deleted file mode 100644 index 31866c2925..0000000000 --- a/changelog.d/5992.feature +++ /dev/null @@ -1 +0,0 @@ -Give appropriate exit codes when synctl fails. diff --git a/changelog.d/5993.feature b/changelog.d/5993.feature deleted file mode 100644 index 3e8bf5068d..0000000000 --- a/changelog.d/5993.feature +++ /dev/null @@ -1 +0,0 @@ -Add the ability to send registration emails from the homeserver rather than delegating to an identity server. diff --git a/changelog.d/5994.feature b/changelog.d/5994.feature deleted file mode 100644 index 5b69b97fe7..0000000000 --- a/changelog.d/5994.feature +++ /dev/null @@ -1 +0,0 @@ -Add the ability to send registration emails from the homeserver rather than delegating to an identity server. \ No newline at end of file diff --git a/changelog.d/5995.bugfix b/changelog.d/5995.bugfix deleted file mode 100644 index e03ab98bc6..0000000000 --- a/changelog.d/5995.bugfix +++ /dev/null @@ -1 +0,0 @@ -Return a M_MISSING_PARAM if `sid` is not provided to `/account/3pid`. \ No newline at end of file diff --git a/changelog.d/5996.bugfix b/changelog.d/5996.bugfix deleted file mode 100644 index 05e31faaa2..0000000000 --- a/changelog.d/5996.bugfix +++ /dev/null @@ -1 +0,0 @@ -federation_certificate_verification_whitelist now will not cause TypeErrors to be raised (a regression in 1.3). Additionally, it now supports internationalised domain names in their non-canonical representation. diff --git a/changelog.d/5998.bugfix b/changelog.d/5998.bugfix deleted file mode 100644 index 9ea095103b..0000000000 --- a/changelog.d/5998.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix room and user stats tracking. diff --git a/changelog.d/6000.feature b/changelog.d/6000.feature deleted file mode 100644 index 0a159bd10d..0000000000 --- a/changelog.d/6000.feature +++ /dev/null @@ -1 +0,0 @@ -Apply the federation blacklist to requests to identity servers. \ No newline at end of file diff --git a/changelog.d/6003.misc b/changelog.d/6003.misc deleted file mode 100644 index 4152d05f87..0000000000 --- a/changelog.d/6003.misc +++ /dev/null @@ -1 +0,0 @@ -Add opentracing span over HTTP push processing. diff --git a/changelog.d/6004.bugfix b/changelog.d/6004.bugfix deleted file mode 100644 index 45c179c8fd..0000000000 --- a/changelog.d/6004.bugfix +++ /dev/null @@ -1 +0,0 @@ -Only count real users when checking for auto-creation of auto-join room. diff --git a/changelog.d/6005.feature b/changelog.d/6005.feature deleted file mode 100644 index ed6491d3e4..0000000000 --- a/changelog.d/6005.feature +++ /dev/null @@ -1 +0,0 @@ -The new Prometheus metric `synapse_build_info` exposes the Python version, OS version, and Synapse version of the running server. diff --git a/changelog.d/6009.misc b/changelog.d/6009.misc deleted file mode 100644 index fea479e1dd..0000000000 --- a/changelog.d/6009.misc +++ /dev/null @@ -1 +0,0 @@ -Small refactor of function arguments and docstrings in RoomMemberHandler. \ No newline at end of file diff --git a/changelog.d/6010.misc b/changelog.d/6010.misc deleted file mode 100644 index 0659f12ebd..0000000000 --- a/changelog.d/6010.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused `origin` argument on FederationHandler.add_display_name_to_third_party_invite. \ No newline at end of file diff --git a/changelog.d/6011.feature b/changelog.d/6011.feature deleted file mode 100644 index ad16acb12b..0000000000 --- a/changelog.d/6011.feature +++ /dev/null @@ -1 +0,0 @@ -Use account_threepid_delegate.email and account_threepid_delegate.msisdn for validating threepid sessions. \ No newline at end of file diff --git a/changelog.d/6012.feature b/changelog.d/6012.feature deleted file mode 100644 index 25425510c6..0000000000 --- a/changelog.d/6012.feature +++ /dev/null @@ -1 +0,0 @@ -Add report_stats_endpoint option to configure where stats are reported to, if enabled. Contributed by @Sorunome. diff --git a/changelog.d/6013.misc b/changelog.d/6013.misc deleted file mode 100644 index 939fe8c655..0000000000 --- a/changelog.d/6013.misc +++ /dev/null @@ -1 +0,0 @@ -Compatibility with v2 Identity Service APIs other than /lookup. \ No newline at end of file diff --git a/changelog.d/6015.feature b/changelog.d/6015.feature deleted file mode 100644 index 42aaffced9..0000000000 --- a/changelog.d/6015.feature +++ /dev/null @@ -1 +0,0 @@ -Add config option to increase ratelimits for room admins redacting messages. diff --git a/changelog.d/6016.misc b/changelog.d/6016.misc deleted file mode 100644 index 91cf164714..0000000000 --- a/changelog.d/6016.misc +++ /dev/null @@ -1 +0,0 @@ -Add a 'failure_ts' column to the 'destinations' database table. diff --git a/changelog.d/6017.misc b/changelog.d/6017.misc deleted file mode 100644 index 5ccab9c6ca..0000000000 --- a/changelog.d/6017.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up some code in the retry logic. diff --git a/changelog.d/6020.bugfix b/changelog.d/6020.bugfix deleted file mode 100644 index 58a7deba9d..0000000000 --- a/changelog.d/6020.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure support users can be registered even if MAU limit is reached. diff --git a/changelog.d/6023.misc b/changelog.d/6023.misc deleted file mode 100644 index d80410c22c..0000000000 --- a/changelog.d/6023.misc +++ /dev/null @@ -1 +0,0 @@ -Fix the structured logging tests stomping on the global log configuration for subsequent tests. diff --git a/changelog.d/6024.bugfix b/changelog.d/6024.bugfix deleted file mode 100644 index ddad34595b..0000000000 --- a/changelog.d/6024.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where login error was shown incorrectly on SSO fallback login. diff --git a/changelog.d/6025.bugfix b/changelog.d/6025.bugfix deleted file mode 100644 index 50d7f9aab5..0000000000 --- a/changelog.d/6025.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug in calculating the federation retry backoff period. \ No newline at end of file diff --git a/changelog.d/6026.feature b/changelog.d/6026.feature deleted file mode 100644 index 2489ff09b5..0000000000 --- a/changelog.d/6026.feature +++ /dev/null @@ -1 +0,0 @@ -Stop sending federation transactions to servers which have been down for a long time. diff --git a/changelog.d/6027.doc b/changelog.d/6027.doc deleted file mode 100644 index f0af68f3b1..0000000000 --- a/changelog.d/6027.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify Synapse 1.4.0 upgrade notes. diff --git a/changelog.d/6028.feature b/changelog.d/6028.feature deleted file mode 100644 index cf603fa0c6..0000000000 --- a/changelog.d/6028.feature +++ /dev/null @@ -1 +0,0 @@ -Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`. diff --git a/changelog.d/6029.bugfix b/changelog.d/6029.bugfix deleted file mode 100644 index 9ea095103b..0000000000 --- a/changelog.d/6029.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix room and user stats tracking. diff --git a/changelog.d/6032.misc b/changelog.d/6032.misc deleted file mode 100644 index ec5b5eb881..0000000000 --- a/changelog.d/6032.misc +++ /dev/null @@ -1 +0,0 @@ -Add developer documentation for using SAML2. diff --git a/changelog.d/6037.feature b/changelog.d/6037.feature deleted file mode 100644 index 85553d2da0..0000000000 --- a/changelog.d/6037.feature +++ /dev/null @@ -1 +0,0 @@ -Make the process for mapping SAML2 users to matrix IDs more flexible. diff --git a/changelog.d/6042.feature b/changelog.d/6042.feature deleted file mode 100644 index a737760363..0000000000 --- a/changelog.d/6042.feature +++ /dev/null @@ -1 +0,0 @@ -Allow homeserver to handle or delegate email validation when adding an email to a user's account. diff --git a/changelog.d/6043.feature b/changelog.d/6043.feature deleted file mode 100644 index cd27b0400b..0000000000 --- a/changelog.d/6043.feature +++ /dev/null @@ -1 +0,0 @@ -Implement new Client Server API endpoints `/account/3pid/add` and `/account/3pid/bind` as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290). \ No newline at end of file diff --git a/changelog.d/6044.feature b/changelog.d/6044.feature deleted file mode 100644 index 7dc05d4845..0000000000 --- a/changelog.d/6044.feature +++ /dev/null @@ -1 +0,0 @@ -Add an unstable feature flag for separate add/bind 3pid APIs. \ No newline at end of file diff --git a/changelog.d/6047.misc b/changelog.d/6047.misc deleted file mode 100644 index a4cdb8abb3..0000000000 --- a/changelog.d/6047.misc +++ /dev/null @@ -1,2 +0,0 @@ -Stop populating some unused tables. - diff --git a/changelog.d/6049.doc b/changelog.d/6049.doc deleted file mode 100644 index e0307bf5c1..0000000000 --- a/changelog.d/6049.doc +++ /dev/null @@ -1 +0,0 @@ -Add some notes on rolling back to v1.3.1. diff --git a/changelog.d/6050.doc b/changelog.d/6050.doc deleted file mode 100644 index 3d19c69bc4..0000000000 --- a/changelog.d/6050.doc +++ /dev/null @@ -1 +0,0 @@ -Update the upgrade notes. diff --git a/changelog.d/6053.bugfix b/changelog.d/6053.bugfix deleted file mode 100644 index 6311157bf6..0000000000 --- a/changelog.d/6053.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent exceptions being logged when extremity-cleanup events fail due to lack of user consent to the terms of service. diff --git a/changelog.d/6056.bugfix b/changelog.d/6056.bugfix deleted file mode 100644 index 4d9573a58d..0000000000 --- a/changelog.d/6056.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove POST method from password reset submit_token endpoint until we implement submit_url functionality. \ No newline at end of file diff --git a/changelog.d/6058.docker b/changelog.d/6058.docker deleted file mode 100644 index 30be6933c9..0000000000 --- a/changelog.d/6058.docker +++ /dev/null @@ -1 +0,0 @@ -Provide SYNAPSE_WORKER envvar to specify python module. diff --git a/changelog.d/6059.bugfix b/changelog.d/6059.bugfix deleted file mode 100644 index 49d5bd3fa0..0000000000 --- a/changelog.d/6059.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix logcontext spam on non-Linux platforms. diff --git a/changelog.d/6062.bugfix b/changelog.d/6062.bugfix deleted file mode 100644 index e20117cf1c..0000000000 --- a/changelog.d/6062.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add POST /_matrix/client/unstable/account/3pid/unbind endpoint from MSC2140 for unbinding a 3PID from an identity server without removing it from the homeserver user account. diff --git a/changelog.d/6063.bugfix b/changelog.d/6063.bugfix deleted file mode 100644 index 7485e32a2c..0000000000 --- a/changelog.d/6063.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure query parameters in email validation links are URL-encoded. \ No newline at end of file diff --git a/changelog.d/6064.misc b/changelog.d/6064.misc deleted file mode 100644 index 28dc89111b..0000000000 --- a/changelog.d/6064.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up the sample config for SAML authentication. diff --git a/changelog.d/6067.feature b/changelog.d/6067.feature deleted file mode 100644 index 72685961c9..0000000000 --- a/changelog.d/6067.feature +++ /dev/null @@ -1 +0,0 @@ -Remove `bind` parameter from Client Server POST `/account` endpoint as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290/). \ No newline at end of file diff --git a/changelog.d/6069.bugfix b/changelog.d/6069.bugfix deleted file mode 100644 index a437ac41a9..0000000000 --- a/changelog.d/6069.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug which caused SAML attribute maps to be overridden by defaults. diff --git a/changelog.d/6072.misc b/changelog.d/6072.misc deleted file mode 100644 index 91cf164714..0000000000 --- a/changelog.d/6072.misc +++ /dev/null @@ -1 +0,0 @@ -Add a 'failure_ts' column to the 'destinations' database table. diff --git a/changelog.d/6073.feature b/changelog.d/6073.feature deleted file mode 100644 index 15d9933891..0000000000 --- a/changelog.d/6073.feature +++ /dev/null @@ -1 +0,0 @@ -Return a clearer error message when a timeout occurs when attempting to contact an identity server. \ No newline at end of file diff --git a/changelog.d/6074.feature b/changelog.d/6074.feature deleted file mode 100644 index b7aa9c99d8..0000000000 --- a/changelog.d/6074.feature +++ /dev/null @@ -1 +0,0 @@ -Prevent password reset's submit_token endpoint from accepting trailing slashes. \ No newline at end of file diff --git a/changelog.d/6075.misc b/changelog.d/6075.misc deleted file mode 100644 index 914e56bcfe..0000000000 --- a/changelog.d/6075.misc +++ /dev/null @@ -1 +0,0 @@ -Change mailer logging to reflect Synapse doesn't just do chat notifications by email now. \ No newline at end of file diff --git a/changelog.d/6078.feature b/changelog.d/6078.feature deleted file mode 100644 index fae1e52322..0000000000 --- a/changelog.d/6078.feature +++ /dev/null @@ -1 +0,0 @@ -Add `POST /add_threepid/msisdn/submit_token` endpoint for proxying submitToken on an account_threepid_handler. \ No newline at end of file diff --git a/changelog.d/6079.feature b/changelog.d/6079.feature deleted file mode 100644 index bcbb49ac58..0000000000 --- a/changelog.d/6079.feature +++ /dev/null @@ -1 +0,0 @@ -Add `submit_url` response parameter to `*/msisdn/requestToken` endpoints. diff --git a/changelog.d/6082.feature b/changelog.d/6082.feature deleted file mode 100644 index c30662b608..0000000000 --- a/changelog.d/6082.feature +++ /dev/null @@ -1 +0,0 @@ -Return 403 on `/register/available` if registration has been disabled. \ No newline at end of file diff --git a/changelog.d/6089.misc b/changelog.d/6089.misc deleted file mode 100644 index fa3c197c54..0000000000 --- a/changelog.d/6089.misc +++ /dev/null @@ -1 +0,0 @@ -Move last seen info into devices table. diff --git a/changelog.d/6090.feature b/changelog.d/6090.feature deleted file mode 100644 index a6da448a1a..0000000000 --- a/changelog.d/6090.feature +++ /dev/null @@ -1 +0,0 @@ -Explicitly log when a homeserver does not have the 'trusted_key_servers' config field configured. diff --git a/changelog.d/6092.bugfix b/changelog.d/6092.bugfix deleted file mode 100644 index 01a7498ec6..0000000000 --- a/changelog.d/6092.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the logged number of updated items for the users_set_deactivated_flag background update. diff --git a/changelog.d/6097.bugfix b/changelog.d/6097.bugfix deleted file mode 100644 index 750a8ecf0a..0000000000 --- a/changelog.d/6097.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add sid to next_link for email validation. diff --git a/changelog.d/6098.feature b/changelog.d/6098.feature deleted file mode 100644 index f3c693c06b..0000000000 --- a/changelog.d/6098.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for pruning old rows in `user_ips` table. diff --git a/changelog.d/6099.misc b/changelog.d/6099.misc deleted file mode 100644 index 8415c6759b..0000000000 --- a/changelog.d/6099.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused parameter to get_user_id_by_threepid. diff --git a/changelog.d/6104.bugfix b/changelog.d/6104.bugfix deleted file mode 100644 index 41114a66ef..0000000000 --- a/changelog.d/6104.bugfix +++ /dev/null @@ -1 +0,0 @@ -Threepid validity checks on msisdns should not be dependent on 'threepid_behaviour_email'. diff --git a/changelog.d/6105.misc b/changelog.d/6105.misc deleted file mode 100644 index 2e838a35c6..0000000000 --- a/changelog.d/6105.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor the user-interactive auth handling. diff --git a/changelog.d/6106.misc b/changelog.d/6106.misc deleted file mode 100644 index d732091779..0000000000 --- a/changelog.d/6106.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor code for calculating registration flows. diff --git a/changelog.d/6107.bugfix b/changelog.d/6107.bugfix deleted file mode 100644 index d4b9516ac7..0000000000 --- a/changelog.d/6107.bugfix +++ /dev/null @@ -1 +0,0 @@ -Ensure that servers which are not configured to support email address verification do not offer it in the registration flows. \ No newline at end of file From 5384c43626a2a54196b3bf08d49a0101141d39fb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 26 Sep 2019 13:38:56 +0100 Subject: [PATCH 20/21] Changelog formatting --- CHANGES.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9f610e4c12..86261b27d6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,13 +15,12 @@ Features - Add `m.id_access_token` to `unstable_features` in `/versions` as per [MSC2264](https://github.com/matrix-org/matrix-doc/pull/2264). ([\#5974](https://github.com/matrix-org/synapse/issues/5974)) - Use the v2 Identity Service API for 3PID invites. ([\#5979](https://github.com/matrix-org/synapse/issues/5979)) - Add `POST /_matrix/client/unstable/account/3pid/unbind` endpoint from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140) for unbinding a 3PID from an identity server without removing it from the homeserver user account. ([\#5980](https://github.com/matrix-org/synapse/issues/5980), [\#6062](https://github.com/matrix-org/synapse/issues/6062)) -) - Use `account_threepid_delegate.email` and `account_threepid_delegate.msisdn` for validating threepid sessions. ([\#6011](https://github.com/matrix-org/synapse/issues/6011)) - Allow homeserver to handle or delegate email validation when adding an email to a user's account. ([\#6042](https://github.com/matrix-org/synapse/issues/6042)) - Implement new Client Server API endpoints `/account/3pid/add` and `/account/3pid/bind` as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290). ([\#6043](https://github.com/matrix-org/synapse/issues/6043)) - Add an unstable feature flag for separate add/bind 3pid APIs. ([\#6044](https://github.com/matrix-org/synapse/issues/6044)) - Remove `bind` parameter from Client Server POST `/account` endpoint as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290/). ([\#6067](https://github.com/matrix-org/synapse/issues/6067)) - - Add `POST /add_threepid/msisdn/submit_token` endpoint for proxying submitToken on an account_threepid_handler. ([\#6078](https://github.com/matrix-org/synapse/issues/6078)) + - Add `POST /add_threepid/msisdn/submit_token` endpoint for proxying submitToken on an `account_threepid_handler`. ([\#6078](https://github.com/matrix-org/synapse/issues/6078)) - Add `submit_url` response parameter to `*/msisdn/requestToken` endpoints. ([\#6079](https://github.com/matrix-org/synapse/issues/6079)) - Add `m.require_identity_server` flag to /version's unstable_features. ([\#5972](https://github.com/matrix-org/synapse/issues/5972)) - Enhancements to OpenTracing support: From e04c235907968f19788eba1be59169cf08e1df08 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 26 Sep 2019 14:59:01 +0100 Subject: [PATCH 21/21] more changelog updates --- CHANGES.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 86261b27d6..addc4c4b56 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,8 +9,8 @@ Features - Changes to 3pid verification: - Add the ability to send registration emails from the homeserver rather than delegating to an identity server. ([\#5835](https://github.com/matrix-org/synapse/issues/5835), [\#5940](https://github.com/matrix-org/synapse/issues/5940), [\#5993](https://github.com/matrix-org/synapse/issues/5993), [\#5994](https://github.com/matrix-org/synapse/issues/5994), [\#5868](https://github.com/matrix-org/synapse/issues/5868)) - - Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`. ([\#5876](https://github.com/matrix-org/synapse/issues/5876), [\#5969](https://github.com/matrix-org/synapse/issues/5969), [\#6028](https://github.com/matrix-org/synapse/issues/6028)) - - Switch to using the v2 Identity Service `/lookup` API where available, with fallback to v1. (Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) plus id_access_token authentication for v2 Identity Service APIs from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140)). ([\#5897](https://github.com/matrix-org/synapse/issues/5897)) + - Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`, and make the `id_server` parameteter optional on `*/requestToken` endpoints, as per [MSC2263](https://github.com/matrix-org/matrix-doc/pull/2263). ([\#5876](https://github.com/matrix-org/synapse/issues/5876), [\#5969](https://github.com/matrix-org/synapse/issues/5969), [\#6028](https://github.com/matrix-org/synapse/issues/6028)) + - Switch to using the v2 Identity Service `/lookup` API where available, with fallback to v1. (Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) plus `id_access_token authentication` for v2 Identity Service APIs from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140)). ([\#5897](https://github.com/matrix-org/synapse/issues/5897)) - Remove `bind_email` and `bind_msisdn` parameters from `/register` ala [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140). ([\#5964](https://github.com/matrix-org/synapse/issues/5964)) - Add `m.id_access_token` to `unstable_features` in `/versions` as per [MSC2264](https://github.com/matrix-org/matrix-doc/pull/2264). ([\#5974](https://github.com/matrix-org/synapse/issues/5974)) - Use the v2 Identity Service API for 3PID invites. ([\#5979](https://github.com/matrix-org/synapse/issues/5979)) @@ -88,8 +88,8 @@ Bugfixes - Fix logcontext spam on non-Linux platforms. ([\#6059](https://github.com/matrix-org/synapse/issues/6059)) - Ensure query parameters in email validation links are URL-encoded. ([\#6063](https://github.com/matrix-org/synapse/issues/6063)) - Fix a bug which caused SAML attribute maps to be overridden by defaults. ([\#6069](https://github.com/matrix-org/synapse/issues/6069)) -- Fix the logged number of updated items for the users_set_deactivated_flag background update. ([\#6092](https://github.com/matrix-org/synapse/issues/6092)) -- Add sid to `next_link` for email validation. ([\#6097](https://github.com/matrix-org/synapse/issues/6097)) +- Fix the logged number of updated items for the `users_set_deactivated_flag` background update. ([\#6092](https://github.com/matrix-org/synapse/issues/6092)) +- Add `sid` to `next_link` for email validation. ([\#6097](https://github.com/matrix-org/synapse/issues/6097)) - Threepid validity checks on msisdns should not be dependent on `threepid_behaviour_email`. ([\#6104](https://github.com/matrix-org/synapse/issues/6104)) - Ensure that servers which are not configured to support email address verification do not offer it in the registration flows. ([\#6107](https://github.com/matrix-org/synapse/issues/6107)) @@ -97,8 +97,8 @@ Bugfixes Updates to the Docker image --------------------------- -- Avoid changing UID/GID if they are already correct. ([\#5970](https://github.com/matrix-org/synapse/issues/5970)) -- Provide SYNAPSE_WORKER envvar to specify python module. ([\#6058](https://github.com/matrix-org/synapse/issues/6058)) +- Avoid changing `UID/GID` if they are already correct. ([\#5970](https://github.com/matrix-org/synapse/issues/5970)) +- Provide `SYNAPSE_WORKER` envvar to specify python module. ([\#6058](https://github.com/matrix-org/synapse/issues/6058)) Improved Documentation @@ -127,7 +127,7 @@ Internal Changes - Refactor the Appservice scheduler code. ([\#5886](https://github.com/matrix-org/synapse/issues/5886)) - Compatibility with v2 Identity Service APIs other than /lookup. ([\#5892](https://github.com/matrix-org/synapse/issues/5892), [\#6013](https://github.com/matrix-org/synapse/issues/6013)) - Stop populating some unused tables. ([\#5893](https://github.com/matrix-org/synapse/issues/5893), [\#6047](https://github.com/matrix-org/synapse/issues/6047)) -- Add missing index on users_in_public_rooms to improve the performance of directory queries. ([\#5894](https://github.com/matrix-org/synapse/issues/5894)) +- Add missing index on `users_in_public_rooms` to improve the performance of directory queries. ([\#5894](https://github.com/matrix-org/synapse/issues/5894)) - Improve the logging when we have an error when fetching signing keys. ([\#5896](https://github.com/matrix-org/synapse/issues/5896)) - Add support for database engine-specific schema deltas, based on file extension. ([\#5911](https://github.com/matrix-org/synapse/issues/5911)) - Update Buildkite pipeline to use plugins instead of buildkite-agent commands. ([\#5922](https://github.com/matrix-org/synapse/issues/5922))