From 369195caa5f52fc67c2496507fc99fccf4b0ede8 Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Wed, 10 May 2017 17:17:12 +0100 Subject: [PATCH 1/5] Modify register/available to be GET with query param - GET is now the method for register/available - a query parameter "username" is now used Also, empty usernames are now handled with an error message on registration or via register/available: `User ID cannot be empty` --- synapse/handlers/register.py | 7 +++++++ synapse/rest/client/v2_alpha/register.py | 9 ++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 03c6a85fc6..dd84c5f5e9 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -54,6 +54,13 @@ class RegistrationHandler(BaseHandler): Codes.INVALID_USERNAME ) + if len(localpart) == 0: + raise SynapseError( + 400, + "User ID cannot be empty", + Codes.INVALID_USERNAME + ) + if localpart[0] == '_': raise SynapseError( 400, diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 6a7cd96ea5..1421c18152 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -21,7 +21,7 @@ from synapse.api.auth import get_access_token_from_request, has_access_token from synapse.api.constants import LoginType from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request, assert_params_in_request + RestServlet, parse_json_object_from_request, assert_params_in_request, parse_string ) from synapse.util.msisdn import phone_number_to_msisdn @@ -142,15 +142,14 @@ class UsernameAvailabilityRestServlet(RestServlet): ) @defer.inlineCallbacks - def on_POST(self, request): + def on_GET(self, request): ip = self.hs.get_ip_from_request(request) with self.ratelimiter.ratelimit(ip) as wait_deferred: yield wait_deferred - body = parse_json_object_from_request(request) - assert_params_in_request(body, ['username']) + username = parse_string(request, "username", required=True) - yield self.registration_handler.check_username(body['username']) + yield self.registration_handler.check_username(username) defer.returnValue((200, {"available": True})) From ccad2ed824f1601d2cc29aafe5dff6ef46668f5e Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Wed, 10 May 2017 17:34:30 +0100 Subject: [PATCH 2/5] Modify condition on empty localpart --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index dd84c5f5e9..ee3a2269a8 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -54,7 +54,7 @@ class RegistrationHandler(BaseHandler): Codes.INVALID_USERNAME ) - if len(localpart) == 0: + if not localpart: raise SynapseError( 400, "User ID cannot be empty", From b64d312421976162a8d41246f11652b5003bb66f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 10 May 2017 17:46:41 +0100 Subject: [PATCH 3/5] add some logging to purge_history --- synapse/storage/events.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2ab44ceaa7..512828cf34 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -2033,6 +2033,8 @@ class EventsStore(SQLBaseStore): for event_id, state_key in event_rows: txn.call_after(self._get_state_group_for_event.invalidate, (event_id,)) + logger.debug("[purge] Finding new backward extremities") + # We calculate the new entries for the backward extremeties by finding # all events that point to events that are to be purged txn.execute( @@ -2045,6 +2047,8 @@ class EventsStore(SQLBaseStore): ) new_backwards_extrems = txn.fetchall() + logger.debug("[purge] replacing backward extremities: %r", new_backwards_extrems) + txn.execute( "DELETE FROM event_backward_extremities WHERE room_id = ?", (room_id,) @@ -2059,6 +2063,8 @@ class EventsStore(SQLBaseStore): ] ) + logger.debug("[purge] finding redundant state groups") + # Get all state groups that are only referenced by events that are # to be deleted. txn.execute( @@ -2076,6 +2082,10 @@ class EventsStore(SQLBaseStore): state_rows = txn.fetchall() state_groups_to_delete = [sg for sg, in state_rows] + logger.debug( + "[purge] finding state groups which depend on redundant state groups" + ) + # Now we get all the state groups that rely on these state groups new_state_edges = [] chunks = [ @@ -2096,6 +2106,8 @@ class EventsStore(SQLBaseStore): # Now we turn the state groups that reference to-be-deleted state groups # to non delta versions. for new_state_edge in new_state_edges: + logger.debug("[purge] de-delta-ing remaining state group %s", + new_state_edge) curr_state = self._get_state_groups_from_groups_txn( txn, [new_state_edge], types=None ) @@ -2132,6 +2144,7 @@ class EventsStore(SQLBaseStore): ], ) + logger.debug("[purge] removing redundant state groups") txn.executemany( "DELETE FROM state_groups_state WHERE state_group = ?", state_rows @@ -2140,12 +2153,15 @@ class EventsStore(SQLBaseStore): "DELETE FROM state_groups WHERE id = ?", state_rows ) + # Delete all non-state + logger.debug("[purge] removing events from event_to_state_groups") txn.executemany( "DELETE FROM event_to_state_groups WHERE event_id = ?", [(event_id,) for event_id, _ in event_rows] ) + logger.debug("[purge] updating room_depth") txn.execute( "UPDATE room_depth SET min_depth = ? WHERE room_id = ?", (topological_ordering, room_id,) @@ -2171,16 +2187,15 @@ class EventsStore(SQLBaseStore): "event_signatures", "rejections", ): + logger.debug("[purge] removing non-state events from %s", table) + txn.executemany( "DELETE FROM %s WHERE event_id = ?" % (table,), to_delete ) - txn.executemany( - "DELETE FROM events WHERE event_id = ?", - to_delete - ) # Mark all state and own events as outliers + logger.debug("[purge] marking events as outliers") txn.executemany( "UPDATE events SET outlier = ?" " WHERE event_id = ?", @@ -2190,6 +2205,8 @@ class EventsStore(SQLBaseStore): ] ) + logger.debug("[purge] done") + @defer.inlineCallbacks def is_event_after(self, event_id1, event_id2): """Returns True if event_id1 is after event_id2 in the stream From 8e345ce46532974aac08c15cf4c90924ec4496d5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 10 May 2017 18:17:41 +0100 Subject: [PATCH 4/5] Don't de-delta state groups we're about to delete --- synapse/storage/events.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 512828cf34..2a37e6f1a8 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -2080,19 +2080,14 @@ class EventsStore(SQLBaseStore): ) state_rows = txn.fetchall() - state_groups_to_delete = [sg for sg, in state_rows] - - logger.debug( - "[purge] finding state groups which depend on redundant state groups" - ) + state_groups_to_delete = set([sg for sg, in state_rows]) # Now we get all the state groups that rely on these state groups + logger.debug("[purge] finding state groups which depend on redundant" + " state groups") new_state_edges = [] - chunks = [ - state_groups_to_delete[i:i + 100] - for i in xrange(0, len(state_groups_to_delete), 100) - ] - for chunk in chunks: + for i in xrange(0, len(state_rows), 100): + chunk = [sg for sg, in state_rows[i:i + 100]] rows = self._simple_select_many_txn( txn, table="state_group_edges", @@ -2101,7 +2096,10 @@ class EventsStore(SQLBaseStore): retcols=["state_group"], keyvalues={}, ) - new_state_edges.extend(row["state_group"] for row in rows) + new_state_edges.extend( + row["state_group"] for row in rows + if row["state_group"] not in state_groups_to_delete + ) # Now we turn the state groups that reference to-be-deleted state groups # to non delta versions. From dc026bb16ff552e9424be217ec5c64104c8b193f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 11 May 2017 10:56:12 +0100 Subject: [PATCH 5/5] Tidy purge code and add some comments Try to make this clearer with more comments and some variable renames --- synapse/storage/events.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2a37e6f1a8..dbd63078c6 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -2080,14 +2080,19 @@ class EventsStore(SQLBaseStore): ) state_rows = txn.fetchall() + + # make a set of the redundant state groups, so that we can look them up + # efficiently state_groups_to_delete = set([sg for sg, in state_rows]) # Now we get all the state groups that rely on these state groups logger.debug("[purge] finding state groups which depend on redundant" " state groups") - new_state_edges = [] + remaining_state_groups = [] for i in xrange(0, len(state_rows), 100): chunk = [sg for sg, in state_rows[i:i + 100]] + # look for state groups whose prev_state_group is one we are about + # to delete rows = self._simple_select_many_txn( txn, table="state_group_edges", @@ -2096,26 +2101,28 @@ class EventsStore(SQLBaseStore): retcols=["state_group"], keyvalues={}, ) - new_state_edges.extend( + remaining_state_groups.extend( row["state_group"] for row in rows + + # exclude state groups we are about to delete: no point in + # updating them if row["state_group"] not in state_groups_to_delete ) - # Now we turn the state groups that reference to-be-deleted state groups - # to non delta versions. - for new_state_edge in new_state_edges: - logger.debug("[purge] de-delta-ing remaining state group %s", - new_state_edge) + # Now we turn the state groups that reference to-be-deleted state + # groups to non delta versions. + for sg in remaining_state_groups: + logger.debug("[purge] de-delta-ing remaining state group %s", sg) curr_state = self._get_state_groups_from_groups_txn( - txn, [new_state_edge], types=None + txn, [sg], types=None ) - curr_state = curr_state[new_state_edge] + curr_state = curr_state[sg] self._simple_delete_txn( txn, table="state_groups_state", keyvalues={ - "state_group": new_state_edge, + "state_group": sg, } ) @@ -2123,7 +2130,7 @@ class EventsStore(SQLBaseStore): txn, table="state_group_edges", keyvalues={ - "state_group": new_state_edge, + "state_group": sg, } ) @@ -2132,7 +2139,7 @@ class EventsStore(SQLBaseStore): table="state_groups_state", values=[ { - "state_group": new_state_edge, + "state_group": sg, "room_id": room_id, "type": key[0], "state_key": key[1],