From 43253c10b894902bcaed90dd2fbf44424b647f45 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Sep 2016 11:47:48 +0100 Subject: [PATCH 1/9] Remove redundant event_auth index --- .../storage/schema/delta/35/remove_auth_idx.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 synapse/storage/schema/delta/35/remove_auth_idx.sql diff --git a/synapse/storage/schema/delta/35/remove_auth_idx.sql b/synapse/storage/schema/delta/35/remove_auth_idx.sql new file mode 100644 index 0000000000..a5bf3895b0 --- /dev/null +++ b/synapse/storage/schema/delta/35/remove_auth_idx.sql @@ -0,0 +1,16 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DROP INDEX IF EXISTS evauth_edges_id; From c57f871184cf669374c07faa6a95eceb8a9321b2 Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Sat, 8 Oct 2016 23:55:20 +0200 Subject: [PATCH 2/9] Use sys.executable instead of hardcoded python. fixes #1161 --- synapse/app/synctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index bb41962d47..c045588866 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -24,7 +24,7 @@ import subprocess import sys import yaml -SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] +SYNAPSE = [sys.executable, "-B", "-m", "synapse.app.homeserver"] GREEN = "\x1b[1;32m" RED = "\x1b[1;31m" From 2869a29fd7d049d7a4dd95f1992e294187884bd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Sep 2016 13:59:07 +0100 Subject: [PATCH 3/9] Drop some unused indices --- synapse/storage/events.py | 1 - .../schema/delta/35/remove_auth_idx.sql | 16 ---- .../schema/delta/36/remove_auth_idx.py | 83 +++++++++++++++++++ 3 files changed, 83 insertions(+), 17 deletions(-) delete mode 100644 synapse/storage/schema/delta/35/remove_auth_idx.sql create mode 100644 synapse/storage/schema/delta/36/remove_auth_idx.py diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 6dc46fa50f..65b3775f1f 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -603,7 +603,6 @@ class EventsStore(SQLBaseStore): "rejections", "redactions", "room_memberships", - "state_events", "topics" ): txn.executemany( diff --git a/synapse/storage/schema/delta/35/remove_auth_idx.sql b/synapse/storage/schema/delta/35/remove_auth_idx.sql deleted file mode 100644 index a5bf3895b0..0000000000 --- a/synapse/storage/schema/delta/35/remove_auth_idx.sql +++ /dev/null @@ -1,16 +0,0 @@ -/* Copyright 2016 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -DROP INDEX IF EXISTS evauth_edges_id; diff --git a/synapse/storage/schema/delta/36/remove_auth_idx.py b/synapse/storage/schema/delta/36/remove_auth_idx.py new file mode 100644 index 0000000000..deac8ea39f --- /dev/null +++ b/synapse/storage/schema/delta/36/remove_auth_idx.py @@ -0,0 +1,83 @@ +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.storage.prepare_database import get_statements +from synapse.storage.engines import PostgresEngine, Sqlite3Engine + +import logging + +logger = logging.getLogger(__name__) + +DROP_INDICES = """ +-- We only ever query based on event_id +DROP INDEX IF EXISTS state_events_room_id; +DROP INDEX IF EXISTS state_events_type; +DROP INDEX IF EXISTS state_events_state_key; + +-- room_id is indexed elsewhere +DROP INDEX IF EXISTS current_state_events_room_id; +DROP INDEX IF EXISTS current_state_events_state_key; +DROP INDEX IF EXISTS current_state_events_type; + +DROP INDEX IF EXISTS transactions_have_ref; + +-- (topological_ordering, stream_ordering, room_id) seems like a strange index, +-- and is used incredibly rarely. +DROP INDEX IF EXISTS events_order_topo_stream_room; + +-- TODO: Drop event_auth stuff. + +DROP INDEX IF EXISTS event_search_ev_idx; +""" + +POSTGRES_DROP_CONSTRAINT = """ +ALTER TABLE event_auth DROP CONSTRAINT IF EXISTS event_auth_event_id_auth_id_room_id_key; +""" + +SQLITE_DROP_CONSTRAINT = """ +DROP INDEX IF EXISTS evauth_edges_id; + +CREATE TABLE IF NOT EXISTS event_auth_new( + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL +); + +INSERT INTO event_auth_new + SELECT event_id, auth_id, room_id + FROM event_auth; + +DROP TABLE event_auth; + +ALTER TABLE event_auth_new RENAME TO event_auth; + +CREATE INDEX evauth_edges_id ON event_auth(event_id); +""" + + +def run_create(cur, database_engine, *args, **kwargs): + for statement in get_statements(DROP_INDICES.splitlines()): + cur.execute(statement) + + if isinstance(database_engine, PostgresEngine): + drop_constraint = POSTGRES_DROP_CONSTRAINT + else: + drop_constraint = SQLITE_DROP_CONSTRAINT + + for statement in get_statements(drop_constraint.splitlines()): + cur.execute(statement) + + +def run_upgrade(cur, database_engine, *args, **kwargs): + pass From b59994b45423d60b44a907e834db7e7702eaa6e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Oct 2016 11:17:02 +0100 Subject: [PATCH 4/9] Remove TODO --- synapse/storage/schema/delta/36/remove_auth_idx.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/schema/delta/36/remove_auth_idx.py b/synapse/storage/schema/delta/36/remove_auth_idx.py index 4be72a86bc..784f3b348f 100644 --- a/synapse/storage/schema/delta/36/remove_auth_idx.py +++ b/synapse/storage/schema/delta/36/remove_auth_idx.py @@ -36,8 +36,6 @@ DROP INDEX IF EXISTS transactions_have_ref; -- and is used incredibly rarely. DROP INDEX IF EXISTS events_order_topo_stream_room; --- TODO: Drop event_auth stuff. - DROP INDEX IF EXISTS event_search_ev_idx; """ From 6942d68247976e572fc4c56171da9303bfd7319f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Oct 2016 11:17:45 +0100 Subject: [PATCH 5/9] Bump schema version --- synapse/storage/prepare_database.py | 2 +- synapse/storage/schema/delta/{36 => 37}/remove_auth_idx.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/delta/{36 => 37}/remove_auth_idx.py (100%) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 08de3cc4c1..d2c0aebe48 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 36 +SCHEMA_VERSION = 37 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/36/remove_auth_idx.py b/synapse/storage/schema/delta/37/remove_auth_idx.py similarity index 100% rename from synapse/storage/schema/delta/36/remove_auth_idx.py rename to synapse/storage/schema/delta/37/remove_auth_idx.py From b95dbdcba4fd79d9a5e3c53075af4838f5816bd2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Oct 2016 15:59:12 +0100 Subject: [PATCH 6/9] Bump version and changelog --- CHANGES.rst | 26 +++++++++++++++++++++++++- synapse/__init__.py | 2 +- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 12abd6cfc4..c5b38b5ca4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,28 @@ -Changes in synapse v0.18.1 (2016-10-0) +Changes in synapse v0.18.2-rc1 (2016-10-17) +=========================================== + +Changes: + +* Remove redundant event_auth index (PR #1113) +* Reduce DB hits for replication (PR #1141) +* Implement pluggable password auth (PR #1155) +* Remove rate limiting from app service senders and fix get_or_create_user + requester (PR #1157) +* window.postmessage for Interactive Auth fallback (PR #1159) +* Use sys.executable instead of hardcoded python. fixes #1161 (PR #1162) +* Add config option for adding additional TLS fingerprints (PR #1167) +* User-interactive auth on delete device (PR #1168) + + +Bug fixes: + +* Fix not being allowed to set your own state_key (PR #1150) +* Fix interactive auth to return 401 from for incorrect password (PR #1160) +* Fix email push notifs being dropped (PR #1169) + + + +Changes in synapse v0.18.1 (2016-10-05) ====================================== No changes since v0.18.1-rc1 diff --git a/synapse/__init__.py b/synapse/__init__.py index 6dbe8fc7e7..6333c1119e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.18.1" +__version__ = "0.18.2-rc1" From 513e600f63847d997e935758b676538710d06ad5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Oct 2016 17:05:59 +0100 Subject: [PATCH 7/9] Update changelog --- CHANGES.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c5b38b5ca4..1f97680f41 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,17 +7,20 @@ Changes: * Reduce DB hits for replication (PR #1141) * Implement pluggable password auth (PR #1155) * Remove rate limiting from app service senders and fix get_or_create_user - requester (PR #1157) + requester, thanks to Patrik Oldsberg (PR #1157) * window.postmessage for Interactive Auth fallback (PR #1159) -* Use sys.executable instead of hardcoded python. fixes #1161 (PR #1162) +* Use sys.executable instead of hardcoded python, thanks to Pedro Larroy + (PR #1162) * Add config option for adding additional TLS fingerprints (PR #1167) * User-interactive auth on delete device (PR #1168) Bug fixes: -* Fix not being allowed to set your own state_key (PR #1150) -* Fix interactive auth to return 401 from for incorrect password (PR #1160) +* Fix not being allowed to set your own state_key, thanks to Patrik Oldsberg + (PR #1150) +* Fix interactive auth to return 401 from for incorrect password (PR #1160, + #1166) * Fix email push notifs being dropped (PR #1169) From f0ca08828082eb1f0a5214b14099034ca53ecbab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 18 Oct 2016 10:52:47 +0100 Subject: [PATCH 8/9] Reduce redundant database work in email pusher Update the last stream ordering if the `get_unread_push_actions_for_user_in_range_for_email` returns no new push actions. This reduces the range that it needs to check next iteration. --- synapse/push/emailpusher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 6600c9cd55..2eb325c7c7 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -150,6 +150,10 @@ class EmailPusher(object): soonest_due_at = None + if not unprocessed: + yield self.save_last_stream_ordering_and_success(self.max_stream_ordering) + return + for push_action in unprocessed: received_at = push_action['received_ts'] if received_at is None: From 8ca05b57550d8d480a1b8cc5a25a6fffb9b09aa9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 18 Oct 2016 10:57:33 +0100 Subject: [PATCH 9/9] Fix push notifications for a single unread message --- synapse/push/mailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 3b63c19eca..53551632b6 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -372,7 +372,7 @@ class Mailer(object): state_event_id = room_state_ids[room_id][ ("m.room.member", event.sender) ] - state_event = yield self.get_event(state_event_id) + state_event = yield self.store.get_event(state_event_id) sender_name = name_from_member_event(state_event) if sender_name is not None and room_name is not None: