Batching in the user directory import (#4900)

pull/4912/head
Amber Brown 2019-03-21 03:06:36 +11:00 committed by GitHub
parent cdb8036161
commit 4d53017432
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 0 deletions

1
changelog.d/4900.feature Normal file
View File

@ -0,0 +1 @@
The user directory has been rewritten to make it faster, with less chance of falling behind on a large server.

View File

@ -32,6 +32,11 @@ TEMP_TABLE = "_temp_populate_user_directory"
class UserDirectoryStore(BackgroundUpdateStore):
# How many records do we calculate before sending it to
# add_users_who_share_private_rooms?
SHARE_PRIVATE_WORKING_SET = 500
def __init__(self, db_conn, hs):
super(UserDirectoryStore, self).__init__(db_conn, hs)
@ -218,6 +223,14 @@ class UserDirectoryStore(BackgroundUpdateStore):
user_set = (user_id, other_user_id)
to_insert.add(user_set)
# If it gets too big, stop and write to the database
# to prevent storing too much in RAM.
if len(to_insert) >= self.SHARE_PRIVATE_WORKING_SET:
yield self.add_users_who_share_private_room(
room_id, to_insert
)
to_insert.clear()
if to_insert:
yield self.add_users_who_share_private_room(room_id, to_insert)
to_insert.clear()