From e6acf0c399b1baadc56f57d8398643a4cb1de56a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Feb 2017 16:37:25 +0000 Subject: [PATCH 1/3] Store the default push actions in a more efficient manner --- synapse/storage/event_push_actions.py | 51 +++++++++++++++++++++------ 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 1b7888a914..db20b7de20 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -26,6 +26,32 @@ import ujson as json logger = logging.getLogger(__name__) +DEFAULT_NOTIF_ACITON = ["notify", {"set_tweak": "highlight", "value": False}] +DEFAULT_HIGHLIGHT_ACITON = [ + "notify", {"set_tweak": "sound", "value": "default"}, {"set_tweak": "highlight"} +] + + +def _serialize_action(actions, is_highlight): + if is_highlight: + if actions == DEFAULT_HIGHLIGHT_ACITON: + return "" + else: + if actions == DEFAULT_NOTIF_ACITON: + return "" + return json.dumps(actions) + + +def _deserialize_action(actions, is_highlight): + if actions: + return json.loads(actions) + + if is_highlight: + return DEFAULT_HIGHLIGHT_ACITON + else: + return DEFAULT_NOTIF_ACITON + + class EventPushActionsStore(SQLBaseStore): EPA_HIGHLIGHT_INDEX = "epa_highlight_index" @@ -58,15 +84,17 @@ class EventPushActionsStore(SQLBaseStore): """ values = [] for uid, actions in tuples: + is_highlight = 1 if _action_has_highlight(actions) else 0 + values.append({ 'room_id': event.room_id, 'event_id': event.event_id, 'user_id': uid, - 'actions': json.dumps(actions), + 'actions': _serialize_action(actions, is_highlight), 'stream_ordering': event.internal_metadata.stream_ordering, 'topological_ordering': event.depth, 'notif': 1, - 'highlight': 1 if _action_has_highlight(actions) else 0, + 'highlight': is_highlight, }) for uid, __ in tuples: @@ -202,7 +230,8 @@ class EventPushActionsStore(SQLBaseStore): # find rooms that have a read receipt in them and return the next # push actions sql = ( - "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions" + "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," + " ep.highlight " " FROM (" " SELECT room_id," " MAX(topological_ordering) as topological_ordering," @@ -243,7 +272,7 @@ class EventPushActionsStore(SQLBaseStore): def get_no_receipt(txn): sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," - " e.received_ts" + " ep.highlight " " FROM event_push_actions AS ep" " INNER JOIN events AS e USING (room_id, event_id)" " WHERE" @@ -272,7 +301,7 @@ class EventPushActionsStore(SQLBaseStore): "event_id": row[0], "room_id": row[1], "stream_ordering": row[2], - "actions": json.loads(row[3]), + "actions": _deserialize_action(row[3], row[4]), } for row in after_read_receipt + no_read_receipt ] @@ -311,7 +340,7 @@ class EventPushActionsStore(SQLBaseStore): def get_after_receipt(txn): sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," - " e.received_ts" + " ep.highlight, e.received_ts" " FROM (" " SELECT room_id," " MAX(topological_ordering) as topological_ordering," @@ -353,7 +382,7 @@ class EventPushActionsStore(SQLBaseStore): def get_no_receipt(txn): sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," - " e.received_ts" + " ep.highlight, e.received_ts" " FROM event_push_actions AS ep" " INNER JOIN events AS e USING (room_id, event_id)" " WHERE" @@ -383,8 +412,8 @@ class EventPushActionsStore(SQLBaseStore): "event_id": row[0], "room_id": row[1], "stream_ordering": row[2], - "actions": json.loads(row[3]), - "received_ts": row[4], + "actions": _deserialize_action(row[3], row[4]), + "received_ts": row[5], } for row in after_read_receipt + no_read_receipt ] @@ -418,7 +447,7 @@ class EventPushActionsStore(SQLBaseStore): sql = ( "SELECT epa.event_id, epa.room_id," " epa.stream_ordering, epa.topological_ordering," - " epa.actions, epa.profile_tag, e.received_ts" + " epa.actions, epa.highlight, epa.profile_tag, e.received_ts" " FROM event_push_actions epa, events e" " WHERE epa.event_id = e.event_id" " AND epa.user_id = ? %s" @@ -433,7 +462,7 @@ class EventPushActionsStore(SQLBaseStore): "get_push_actions_for_user", f ) for pa in push_actions: - pa["actions"] = json.loads(pa["actions"]) + pa["actions"] = _deserialize_action(pa["actions"], pa["highlight"]) defer.returnValue(push_actions) @defer.inlineCallbacks From 502ae6c663e9b96950d1b474fbec7cf200e4e8a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Feb 2017 16:54:37 +0000 Subject: [PATCH 2/3] Comment --- synapse/storage/event_push_actions.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index db20b7de20..ea6722df63 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -26,30 +26,37 @@ import ujson as json logger = logging.getLogger(__name__) -DEFAULT_NOTIF_ACITON = ["notify", {"set_tweak": "highlight", "value": False}] -DEFAULT_HIGHLIGHT_ACITON = [ +DEFAULT_NOTIF_ACTION = ["notify", {"set_tweak": "highlight", "value": False}] +DEFAULT_HIGHLIGHT_ACTION = [ "notify", {"set_tweak": "sound", "value": "default"}, {"set_tweak": "highlight"} ] def _serialize_action(actions, is_highlight): + """Custom serializer for actions. This allows us to "compress" common actions. + + We use the fact that most users have the same actions for notifs (and for + highlights). We replaces these default actions with the emtpy string. + """ if is_highlight: - if actions == DEFAULT_HIGHLIGHT_ACITON: - return "" + if actions == DEFAULT_HIGHLIGHT_ACTION: + return "" # We use empty string as the column is non-NULL else: - if actions == DEFAULT_NOTIF_ACITON: + if actions == DEFAULT_NOTIF_ACTION: return "" return json.dumps(actions) def _deserialize_action(actions, is_highlight): + """Custom deserializer for actions. This allows us to "compress" common actions + """ if actions: return json.loads(actions) if is_highlight: - return DEFAULT_HIGHLIGHT_ACITON + return DEFAULT_HIGHLIGHT_ACTION else: - return DEFAULT_NOTIF_ACITON + return DEFAULT_NOTIF_ACTION class EventPushActionsStore(SQLBaseStore): From 138e030cfe6e00956d79eed0702ff3f284692357 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Feb 2017 15:03:36 +0000 Subject: [PATCH 3/3] Comment --- synapse/storage/event_push_actions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index ea6722df63..808c9d22fc 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -36,7 +36,10 @@ def _serialize_action(actions, is_highlight): """Custom serializer for actions. This allows us to "compress" common actions. We use the fact that most users have the same actions for notifs (and for - highlights). We replaces these default actions with the emtpy string. + highlights). + We store these default actions as the empty string rather than the full JSON. + Since the empty string isn't valid JSON there is no risk of this clashing with + any real JSON actions """ if is_highlight: if actions == DEFAULT_HIGHLIGHT_ACTION: