From 4d53017432e05da621a13b3fe4d9e67108f856fd Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 21 Mar 2019 03:06:36 +1100 Subject: [PATCH] Batching in the user directory import (#4900) --- changelog.d/4900.feature | 1 + synapse/storage/user_directory.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 changelog.d/4900.feature diff --git a/changelog.d/4900.feature b/changelog.d/4900.feature new file mode 100644 index 0000000000..8f792b8890 --- /dev/null +++ b/changelog.d/4900.feature @@ -0,0 +1 @@ +The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index 4ee653210f..d360e857d1 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -32,6 +32,11 @@ TEMP_TABLE = "_temp_populate_user_directory" class UserDirectoryStore(BackgroundUpdateStore): + + # How many records do we calculate before sending it to + # add_users_who_share_private_rooms? + SHARE_PRIVATE_WORKING_SET = 500 + def __init__(self, db_conn, hs): super(UserDirectoryStore, self).__init__(db_conn, hs) @@ -218,6 +223,14 @@ class UserDirectoryStore(BackgroundUpdateStore): user_set = (user_id, other_user_id) to_insert.add(user_set) + # If it gets too big, stop and write to the database + # to prevent storing too much in RAM. + if len(to_insert) >= self.SHARE_PRIVATE_WORKING_SET: + yield self.add_users_who_share_private_room( + room_id, to_insert + ) + to_insert.clear() + if to_insert: yield self.add_users_who_share_private_room(room_id, to_insert) to_insert.clear()