From e4b078a6004ea34684c27056825346178c22aa0f Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 22 Feb 2019 12:11:43 +0000
Subject: [PATCH 01/22] Config option to prevent showing non-fed rooms in fed
 /publicRooms

---
 synapse/config/room_directory.py | 11 +++++++++++
 synapse/handlers/room_list.py    | 17 +++++++++++++++--
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py
index 9da13ab11b..08d1d26e58 100644
--- a/synapse/config/room_directory.py
+++ b/synapse/config/room_directory.py
@@ -27,6 +27,10 @@ class RoomDirectoryConfig(Config):
             for rule in alias_creation_rules
         ]
 
+        self.allow_non_federated_in_public_rooms = config.get(
+            "allow_non_federated_in_public_rooms", True,
+        )
+
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
         # The `alias_creation` option controls who's allowed to create aliases
@@ -42,6 +46,13 @@ class RoomDirectoryConfig(Config):
             - user_id: "*"
               alias: "*"
               action: allow
+
+        # Specify whether rooms that only allow local users to join should be
+        # shown in the federation public room directory.
+        # 
+        # Note that this does not affect the room directory shown to users on
+        # this homeserver, only those on other homeservers.
+        #allow_non_federated_in_public_rooms: True
         """
 
     def is_alias_creation_allowed(self, user_id, alias):
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index dc88620885..47aeeb1d88 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -47,6 +47,7 @@ class RoomListHandler(BaseHandler):
         self.response_cache = ResponseCache(hs, "room_list")
         self.remote_response_cache = ResponseCache(hs, "remote_room_list",
                                                    timeout_ms=30 * 1000)
+        self.config = hs.get_config()
 
     def get_local_public_room_list(self, limit=None, since_token=None,
                                    search_filter=None,
@@ -286,14 +287,16 @@ class RoomListHandler(BaseHandler):
             # We've already got enough, so lets just drop it.
             return
 
-        result = yield self.generate_room_entry(room_id, num_joined_users)
+        result = yield self.generate_room_entry(room_id, num_joined_users,
+            allow_federated=self.config.allow_non_federated_in_public_rooms)
 
         if result and _matches_room_entry(result, search_filter):
             chunk.append(result)
 
     @cachedInlineCallbacks(num_args=1, cache_context=True)
     def generate_room_entry(self, room_id, num_joined_users, cache_context,
-                            with_alias=True, allow_private=False):
+                            with_alias=True, allow_private=False,
+                            allow_federated=True):
         """Returns the entry for a room
         """
         result = {
@@ -308,6 +311,7 @@ class RoomListHandler(BaseHandler):
         event_map = yield self.store.get_events([
             event_id for key, event_id in iteritems(current_state_ids)
             if key[0] in (
+                EventTypes.Create,
                 EventTypes.JoinRules,
                 EventTypes.Name,
                 EventTypes.Topic,
@@ -324,12 +328,21 @@ class RoomListHandler(BaseHandler):
         }
 
         # Double check that this is actually a public room.
+
         join_rules_event = current_state.get((EventTypes.JoinRules, ""))
         if join_rules_event:
             join_rule = join_rules_event.content.get("join_rule", None)
             if not allow_private and join_rule and join_rule != JoinRules.PUBLIC:
                 defer.returnValue(None)
 
+        if not allow_federated:
+            # Disallow non-federated from appearing
+            create_event = current_state.get((EventTypes.Create, ""))
+            if create_event:
+                federate = create_event.content.get("m.federate", True)
+                if federate == False:
+                    defer.returnValue(None)
+
         if with_alias:
             aliases = yield self.store.get_aliases_for_room(
                 room_id, on_invalidate=cache_context.invalidate

From bd398b874eb375e92939a087a64258a4feb91f49 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 22 Feb 2019 15:43:11 +0000
Subject: [PATCH 02/22] Don't restrict non-fed rooms over client APIs

---
 synapse/federation/transport/server.py |  3 ++-
 synapse/groups/groups_server.py        |  4 ++--
 synapse/handlers/room_list.py          | 29 ++++++++++++++++----------
 3 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 7288d49074..6d4a26f595 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -697,7 +697,8 @@ class PublicRoomList(BaseFederationServlet):
 
         data = yield self.handler.get_local_public_room_list(
             limit, since_token,
-            network_tuple=network_tuple
+            network_tuple=network_tuple,
+            from_federation=True,
         )
         defer.returnValue((200, data))
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 633c865ed8..691752a30c 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -113,7 +113,7 @@ class GroupsServerHandler(object):
             room_id = room_entry["room_id"]
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users),
+                room_id, True, len(joined_users),
                 with_alias=False, allow_private=True,
             )
             entry = dict(entry)  # so we don't change whats cached
@@ -544,7 +544,7 @@ class GroupsServerHandler(object):
 
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users),
+                room_id, True, len(joined_users),
                 with_alias=False, allow_private=True,
             )
 
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 47aeeb1d88..bde9ec744f 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -51,7 +51,8 @@ class RoomListHandler(BaseHandler):
 
     def get_local_public_room_list(self, limit=None, since_token=None,
                                    search_filter=None,
-                                   network_tuple=EMPTY_THIRD_PARTY_ID,):
+                                   network_tuple=EMPTY_THIRD_PARTY_ID,
+                                   from_federation=False):
         """Generate a local public room list.
 
         There are multiple different lists: the main one plus one per third
@@ -82,13 +83,15 @@ class RoomListHandler(BaseHandler):
         return self.response_cache.wrap(
             key,
             self._get_public_room_list,
-            limit, since_token, network_tuple=network_tuple,
+            limit, since_token,
+            network_tuple=network_tuple, from_federation=from_federation,
         )
 
     @defer.inlineCallbacks
     def _get_public_room_list(self, limit=None, since_token=None,
                               search_filter=None,
-                              network_tuple=EMPTY_THIRD_PARTY_ID,):
+                              network_tuple=EMPTY_THIRD_PARTY_ID,
+                              from_federation=False,):
         if since_token and since_token != "END":
             since_token = RoomListNextBatch.from_token(since_token)
         else:
@@ -208,7 +211,8 @@ class RoomListHandler(BaseHandler):
             yield concurrently_execute(
                 lambda r: self._append_room_entry_to_chunk(
                     r, rooms_to_num_joined[r],
-                    chunk, limit, search_filter
+                    chunk, limit, search_filter,
+                    from_federation=from_federation,
                 ),
                 batch, 5,
             )
@@ -279,7 +283,7 @@ class RoomListHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit,
-                                    search_filter):
+                                    search_filter, from_federation=False):
         """Generate the entry for a room in the public room list and append it
         to the `chunk` if it matches the search filter
         """
@@ -287,16 +291,19 @@ class RoomListHandler(BaseHandler):
             # We've already got enough, so lets just drop it.
             return
 
-        result = yield self.generate_room_entry(room_id, num_joined_users,
-            allow_federated=self.config.allow_non_federated_in_public_rooms)
+        if from_federation:
+            result = yield self.generate_room_entry(room_id,
+                self.config.allow_non_federated_in_public_rooms,
+                num_joined_users)
+        else:
+            result = yield self.generate_room_entry(room_id, True, num_joined_users)
 
         if result and _matches_room_entry(result, search_filter):
             chunk.append(result)
 
-    @cachedInlineCallbacks(num_args=1, cache_context=True)
-    def generate_room_entry(self, room_id, num_joined_users, cache_context,
-                            with_alias=True, allow_private=False,
-                            allow_federated=True):
+    @cachedInlineCallbacks(num_args=2, cache_context=True)
+    def generate_room_entry(self, room_id, allow_federated, num_joined_users, 
+                            cache_context, with_alias=True, allow_private=False):
         """Returns the entry for a room
         """
         result = {

From 07493607a84665be006ec52f897eb904d923721c Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 25 Feb 2019 12:42:30 +0000
Subject: [PATCH 03/22] Docs and arg name clarification

---
 synapse/handlers/room_list.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index bde9ec744f..0636e5b8eb 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -302,9 +302,22 @@ class RoomListHandler(BaseHandler):
             chunk.append(result)
 
     @cachedInlineCallbacks(num_args=2, cache_context=True)
-    def generate_room_entry(self, room_id, allow_federated, num_joined_users, 
+    def generate_room_entry(self, room_id, allow_non_federated, num_joined_users,
                             cache_context, with_alias=True, allow_private=False):
         """Returns the entry for a room
+
+        Args:
+            room_id (str): The room's ID.
+            allow_non_federated (bool): Whether rooms with federation
+            disabled should be shown.
+            num_joined_users (int): Number of users in the room.
+            cache_context: Information for cached responses.
+            with_alias (bool): Whether to return the room's aliases in the result.
+            allow_private (bool): Whether invite-only rooms should be shown.
+
+        Returns:
+            Deferred[dict|None]: Returns a room entry as a dictionary, or None if this
+            room was determined not to be shown publicly.
         """
         result = {
             "room_id": room_id,
@@ -342,7 +355,7 @@ class RoomListHandler(BaseHandler):
             if not allow_private and join_rule and join_rule != JoinRules.PUBLIC:
                 defer.returnValue(None)
 
-        if not allow_federated:
+        if not allow_non_federated:
             # Disallow non-federated from appearing
             create_event = current_state.get((EventTypes.Create, ""))
             if create_event:

From 84c0a20dfeb043f1113a6e213bc7488ef21a0432 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 25 Feb 2019 15:23:27 +0000
Subject: [PATCH 04/22] Simplify call to generate_room_entry

---
 synapse/handlers/room_list.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 810c624c2b..69b0bc3dd1 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -300,24 +300,24 @@ class RoomListHandler(BaseHandler):
             # We've already got enough, so lets just drop it.
             return
 
-        if from_federation:
-            result = yield self.generate_room_entry(room_id,
-                self.config.allow_non_federated_in_public_rooms,
-                num_joined_users)
-        else:
-            result = yield self.generate_room_entry(room_id, True, num_joined_users)
+        result = yield self.generate_room_entry(room_id,
+            num_joined_users)
 
-        if result and _matches_room_entry(result, search_filter):
+        if from_federation and not self.config.allow_non_federated_in_public_rooms:
+            if result["m.federate"] = False:
+                # This is a non-federating room and the config has chosen not
+                # to show these rooms to other servers
+                chunk.append(None)
+        else if result and _matches_room_entry(result, search_filter):
             chunk.append(result)
 
     @cachedInlineCallbacks(num_args=2, cache_context=True)
-    def generate_room_entry(self, room_id, allow_non_federated, num_joined_users,
+    def generate_room_entry(self, room_id, num_joined_users,
                             cache_context, with_alias=True, allow_private=False):
         """Returns the entry for a room
 
         Args:
             room_id (str): The room's ID.
-            allow_non_federated (bool): Whether rooms with federation
             disabled should be shown.
             num_joined_users (int): Number of users in the room.
             cache_context: Information for cached responses.
@@ -364,13 +364,9 @@ class RoomListHandler(BaseHandler):
             if not allow_private and join_rule and join_rule != JoinRules.PUBLIC:
                 defer.returnValue(None)
 
-        if not allow_non_federated:
-            # Disallow non-federated from appearing
-            create_event = current_state.get((EventTypes.Create, ""))
-            if create_event:
-                federate = create_event.content.get("m.federate", True)
-                if federate == False:
-                    defer.returnValue(None)
+        # Return whether this room is open to federation users or not
+        create_event = current_state.get((EventTypes.Create, ""))
+        result["m.federate"] = create_event.content.get("m.federate", True)
 
         if with_alias:
             aliases = yield self.store.get_aliases_for_room(

From 8aaf7ffc4415e9f7da2d34448b29fb2d0cb80574 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 25 Feb 2019 15:27:17 +0000
Subject: [PATCH 05/22] syntax derp

---
 synapse/handlers/room_list.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 69b0bc3dd1..b7f450e719 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -304,7 +304,7 @@ class RoomListHandler(BaseHandler):
             num_joined_users)
 
         if from_federation and not self.config.allow_non_federated_in_public_rooms:
-            if result["m.federate"] = False:
+            if result["m.federate"] == False:
                 # This is a non-federating room and the config has chosen not
                 # to show these rooms to other servers
                 chunk.append(None)

From 1330aa4a8fdff5f6d1ff4bc5d27de674dfcd67e7 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 25 Feb 2019 17:28:19 +0000
Subject: [PATCH 06/22] elif not else if

---
 synapse/handlers/room_list.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index b7f450e719..94cabc2a2e 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -308,7 +308,7 @@ class RoomListHandler(BaseHandler):
                 # This is a non-federating room and the config has chosen not
                 # to show these rooms to other servers
                 chunk.append(None)
-        else if result and _matches_room_entry(result, search_filter):
+        elif result and _matches_room_entry(result, search_filter):
             chunk.append(result)
 
     @cachedInlineCallbacks(num_args=2, cache_context=True)

From 96c408273e3ed8fd18ca3f7530820a02c06c42c1 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Mon, 25 Feb 2019 18:00:17 +0000
Subject: [PATCH 07/22] Fix group's call to generate_room_entry

---
 synapse/groups/groups_server.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 691752a30c..a7eaead56b 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -113,8 +113,7 @@ class GroupsServerHandler(object):
             room_id = room_entry["room_id"]
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, True, len(joined_users),
-                with_alias=False, allow_private=True,
+                room_id, len(joined_users), with_alias=False, allow_private=True,
             )
             entry = dict(entry)  # so we don't change whats cached
             entry.pop("room_id", None)
@@ -544,8 +543,7 @@ class GroupsServerHandler(object):
 
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, True, len(joined_users),
-                with_alias=False, allow_private=True,
+                room_id, len(joined_users), with_alias=False, allow_private=True,
             )
 
             if not entry:

From 71669a0fba524e4d8cdb9e0076dca5d2770ef788 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 11:25:00 +0000
Subject: [PATCH 08/22] Address rich comments

---
 synapse/handlers/room_list.py | 36 +++++++++++++++++++++++++++++------
 synapse/server.pyi            |  2 +-
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 94cabc2a2e..a9725ade37 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -97,7 +97,23 @@ class RoomListHandler(BaseHandler):
     def _get_public_room_list(self, limit=None, since_token=None,
                               search_filter=None,
                               network_tuple=EMPTY_THIRD_PARTY_ID,
-                              from_federation=False, timeout=None,):
+                              from_federation=False,
+                              timeout=None,):
+        """Generate a public room list.
+
+        Args:
+            limit (int): Maximum amount of rooms to return.
+            since_token (str)
+            search_filter (dict): Dictionary to filter rooms by.
+            network_tuple (ThirdPartyInstanceID): Which public list to use.
+                This can be (None, None) to indicate the main list, or a particular
+                appservice and network id to use an appservice specific one.
+                Setting to None returns all public rooms across all lists.
+            from_federation (bool): Whether this request originated from a
+                federating server or a client. Used for room filtering.
+            timeout (int): Amount of seconds to wait for a response before
+                timing out.
+        """
         if since_token and since_token != "END":
             since_token = RoomListNextBatch.from_token(since_token)
         else:
@@ -295,19 +311,28 @@ class RoomListHandler(BaseHandler):
                                     search_filter, from_federation=False):
         """Generate the entry for a room in the public room list and append it
         to the `chunk` if it matches the search filter
+
+        Args:
+            room_id (str): The ID of the room.
+            num_joined_users (int): The number of joined users in the room.
+            chunk (list)
+            limit (int): Maximum amount of rooms to display. Function will
+                return if length of chunk is greater than limit + 1.
+            search_filter (dict)
+            from_federation (bool): Whether this request originated from a
+                federating server or a client. Used for room filtering.
         """
         if limit and len(chunk) > limit + 1:
             # We've already got enough, so lets just drop it.
             return
 
-        result = yield self.generate_room_entry(room_id,
-            num_joined_users)
+        result = yield self.generate_room_entry(room_id, num_joined_users)
 
         if from_federation and not self.config.allow_non_federated_in_public_rooms:
-            if result["m.federate"] == False:
+            if result["m.federate"] is False:
                 # This is a non-federating room and the config has chosen not
                 # to show these rooms to other servers
-                chunk.append(None)
+                return
         elif result and _matches_room_entry(result, search_filter):
             chunk.append(result)
 
@@ -318,7 +343,6 @@ class RoomListHandler(BaseHandler):
 
         Args:
             room_id (str): The room's ID.
-            disabled should be shown.
             num_joined_users (int): Number of users in the room.
             cache_context: Information for cached responses.
             with_alias (bool): Whether to return the room's aliases in the result.
diff --git a/synapse/server.pyi b/synapse/server.pyi
index 06cd083a74..fb8df56cd5 100644
--- a/synapse/server.pyi
+++ b/synapse/server.pyi
@@ -7,9 +7,9 @@ import synapse.handlers.auth
 import synapse.handlers.deactivate_account
 import synapse.handlers.device
 import synapse.handlers.e2e_keys
+import synapse.handlers.message
 import synapse.handlers.room
 import synapse.handlers.room_member
-import synapse.handlers.message
 import synapse.handlers.set_password
 import synapse.rest.media.v1.media_repository
 import synapse.server_notices.server_notices_manager

From 6946c20111cb7c460deb340b61236dce9f4fd878 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 11:27:19 +0000
Subject: [PATCH 09/22] Result may be None

---
 synapse/handlers/room_list.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index a9725ade37..7c21f142a0 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -329,7 +329,7 @@ class RoomListHandler(BaseHandler):
         result = yield self.generate_room_entry(room_id, num_joined_users)
 
         if from_federation and not self.config.allow_non_federated_in_public_rooms:
-            if result["m.federate"] is False:
+            if not result or result["m.federate"] is False:
                 # This is a non-federating room and the config has chosen not
                 # to show these rooms to other servers
                 return

From 6728bf39405c52e4f2473c1e19a7648134361c15 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 11:52:52 +0000
Subject: [PATCH 10/22] Make not showing non-federated rooms the default

---
 synapse/config/room_directory.py | 12 ------------
 synapse/handlers/room_list.py    |  2 +-
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py
index 3322cf2eea..9b897abe3c 100644
--- a/synapse/config/room_directory.py
+++ b/synapse/config/room_directory.py
@@ -52,10 +52,6 @@ class RoomDirectoryConfig(Config):
                 )
             ]
 
-        self.allow_non_federated_in_public_rooms = config.get(
-            "allow_non_federated_in_public_rooms", True,
-        )
-
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
         # The `alias_creation` option controls who's allowed to create aliases
@@ -114,14 +110,6 @@ class RoomDirectoryConfig(Config):
         #    alias: "*"
         #    room_id: "*"
         #    action: allow
-
-        # Specify whether rooms that only allow local users to join should be
-        # shown in the federation public room directory.
-        #
-        # Note that this does not affect the room directory shown to users on
-        # this homeserver, only those on other homeservers.
-        #
-        #allow_non_federated_in_public_rooms: True
         """
 
     def is_alias_creation_allowed(self, user_id, room_id, alias):
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 7c21f142a0..f1a51a7ca9 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -328,7 +328,7 @@ class RoomListHandler(BaseHandler):
 
         result = yield self.generate_room_entry(room_id, num_joined_users)
 
-        if from_federation and not self.config.allow_non_federated_in_public_rooms:
+        if from_federation:
             if not result or result["m.federate"] is False:
                 # This is a non-federating room and the config has chosen not
                 # to show these rooms to other servers

From 40c2271680501c5ca2c29349d62df42d80b953db Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 12:04:34 +0000
Subject: [PATCH 11/22] Clean up room chunk logic

---
 synapse/handlers/room_list.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index f1a51a7ca9..e6038a1069 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -327,13 +327,15 @@ class RoomListHandler(BaseHandler):
             return
 
         result = yield self.generate_room_entry(room_id, num_joined_users)
+        if not result:
+            return
 
-        if from_federation:
-            if not result or result["m.federate"] is False:
-                # This is a non-federating room and the config has chosen not
-                # to show these rooms to other servers
-                return
-        elif result and _matches_room_entry(result, search_filter):
+        if from_federation and result["m.federate"] is False:
+            # This is a room that other servers cannot join. Do not show them
+            # this room.
+            return
+
+        if _matches_room_entry(result, search_filter):
             chunk.append(result)
 
     @cachedInlineCallbacks(num_args=2, cache_context=True)

From 16565e67dbc81ddf5fc16d39f82accbd508ac13f Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 12:12:48 +0000
Subject: [PATCH 12/22] Correct docstring types and chunk logic

---
 synapse/handlers/room_list.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index e6038a1069..7ab8476680 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -59,9 +59,9 @@ class RoomListHandler(BaseHandler):
         party network. A client can ask for a specific list or to return all.
 
         Args:
-            limit (int)
-            since_token (str)
-            search_filter (dict)
+            limit (int|None)
+            since_token (str|None)
+            search_filter (dict|None)
             network_tuple (ThirdPartyInstanceID): Which public list to use.
                 This can be (None, None) to indicate the main list, or a particular
                 appservice and network id to use an appservice specific one.
@@ -102,16 +102,16 @@ class RoomListHandler(BaseHandler):
         """Generate a public room list.
 
         Args:
-            limit (int): Maximum amount of rooms to return.
-            since_token (str)
-            search_filter (dict): Dictionary to filter rooms by.
+            limit (int|None): Maximum amount of rooms to return.
+            since_token (str|None)
+            search_filter (dict|None): Dictionary to filter rooms by.
             network_tuple (ThirdPartyInstanceID): Which public list to use.
                 This can be (None, None) to indicate the main list, or a particular
                 appservice and network id to use an appservice specific one.
                 Setting to None returns all public rooms across all lists.
             from_federation (bool): Whether this request originated from a
                 federating server or a client. Used for room filtering.
-            timeout (int): Amount of seconds to wait for a response before
+            timeout (int|None): Amount of seconds to wait for a response before
                 timing out.
         """
         if since_token and since_token != "END":
@@ -316,9 +316,9 @@ class RoomListHandler(BaseHandler):
             room_id (str): The ID of the room.
             num_joined_users (int): The number of joined users in the room.
             chunk (list)
-            limit (int): Maximum amount of rooms to display. Function will
+            limit (int|None): Maximum amount of rooms to display. Function will
                 return if length of chunk is greater than limit + 1.
-            search_filter (dict)
+            search_filter (dict|None)
             from_federation (bool): Whether this request originated from a
                 federating server or a client. Used for room filtering.
         """
@@ -330,7 +330,8 @@ class RoomListHandler(BaseHandler):
         if not result:
             return
 
-        if from_federation and result["m.federate"] is False:
+        if from_federation:
+            if "m.federate" in result and not result["m.federate"]:
             # This is a room that other servers cannot join. Do not show them
             # this room.
             return

From a712aa3a9c5c58eca6ecfa0cb23636035fbbf449 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 12:13:55 +0000
Subject: [PATCH 13/22] Correct indent

---
 synapse/handlers/room_list.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 7ab8476680..0ce274e9ae 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -332,9 +332,9 @@ class RoomListHandler(BaseHandler):
 
         if from_federation:
             if "m.federate" in result and not result["m.federate"]:
-            # This is a room that other servers cannot join. Do not show them
-            # this room.
-            return
+                # This is a room that other servers cannot join. Do not show them
+                # this room.
+                return
 
         if _matches_room_entry(result, search_filter):
             chunk.append(result)

From c4414768af5e2213b1ad2b16bc8e1b8062fd1e49 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 12:22:34 +0000
Subject: [PATCH 14/22] Cleaner chunk logic

---
 synapse/handlers/room_list.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index 0ce274e9ae..c02fedcfe4 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -330,11 +330,10 @@ class RoomListHandler(BaseHandler):
         if not result:
             return
 
-        if from_federation:
-            if "m.federate" in result and not result["m.federate"]:
-                # This is a room that other servers cannot join. Do not show them
-                # this room.
-                return
+        if from_federation and not result.get("m.federate", True):
+            # This is a room that other servers cannot join. Do not show them
+            # this room.
+            return
 
         if _matches_room_entry(result, search_filter):
             chunk.append(result)

From c74624a633ae883474a5ec3137bddd009165144c Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Tue, 26 Feb 2019 13:20:38 +0000
Subject: [PATCH 15/22] Revert "Prevent showing non-fed rooms in fed
 /publicRooms"

---
 synapse/federation/transport/server.py |  3 +-
 synapse/groups/groups_server.py        |  6 +-
 synapse/handlers/room_list.py          | 76 ++++----------------------
 synapse/server.pyi                     |  2 +-
 4 files changed, 17 insertions(+), 70 deletions(-)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 5ba94be2ec..a2396ab466 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -736,8 +736,7 @@ class PublicRoomList(BaseFederationServlet):
 
         data = yield self.handler.get_local_public_room_list(
             limit, since_token,
-            network_tuple=network_tuple,
-            from_federation=True,
+            network_tuple=network_tuple
         )
         defer.returnValue((200, data))
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index a7eaead56b..633c865ed8 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -113,7 +113,8 @@ class GroupsServerHandler(object):
             room_id = room_entry["room_id"]
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users), with_alias=False, allow_private=True,
+                room_id, len(joined_users),
+                with_alias=False, allow_private=True,
             )
             entry = dict(entry)  # so we don't change whats cached
             entry.pop("room_id", None)
@@ -543,7 +544,8 @@ class GroupsServerHandler(object):
 
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users), with_alias=False, allow_private=True,
+                room_id, len(joined_users),
+                with_alias=False, allow_private=True,
             )
 
             if not entry:
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index c02fedcfe4..13e212d669 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -47,21 +47,19 @@ class RoomListHandler(BaseHandler):
         self.response_cache = ResponseCache(hs, "room_list")
         self.remote_response_cache = ResponseCache(hs, "remote_room_list",
                                                    timeout_ms=30 * 1000)
-        self.config = hs.get_config()
 
     def get_local_public_room_list(self, limit=None, since_token=None,
                                    search_filter=None,
-                                   network_tuple=EMPTY_THIRD_PARTY_ID,
-                                   from_federation=False):
+                                   network_tuple=EMPTY_THIRD_PARTY_ID,):
         """Generate a local public room list.
 
         There are multiple different lists: the main one plus one per third
         party network. A client can ask for a specific list or to return all.
 
         Args:
-            limit (int|None)
-            since_token (str|None)
-            search_filter (dict|None)
+            limit (int)
+            since_token (str)
+            search_filter (dict)
             network_tuple (ThirdPartyInstanceID): Which public list to use.
                 This can be (None, None) to indicate the main list, or a particular
                 appservice and network id to use an appservice specific one.
@@ -89,31 +87,14 @@ class RoomListHandler(BaseHandler):
         return self.response_cache.wrap(
             key,
             self._get_public_room_list,
-            limit, since_token,
-            network_tuple=network_tuple, from_federation=from_federation,
+            limit, since_token, network_tuple=network_tuple,
         )
 
     @defer.inlineCallbacks
     def _get_public_room_list(self, limit=None, since_token=None,
                               search_filter=None,
                               network_tuple=EMPTY_THIRD_PARTY_ID,
-                              from_federation=False,
                               timeout=None,):
-        """Generate a public room list.
-
-        Args:
-            limit (int|None): Maximum amount of rooms to return.
-            since_token (str|None)
-            search_filter (dict|None): Dictionary to filter rooms by.
-            network_tuple (ThirdPartyInstanceID): Which public list to use.
-                This can be (None, None) to indicate the main list, or a particular
-                appservice and network id to use an appservice specific one.
-                Setting to None returns all public rooms across all lists.
-            from_federation (bool): Whether this request originated from a
-                federating server or a client. Used for room filtering.
-            timeout (int|None): Amount of seconds to wait for a response before
-                timing out.
-        """
         if since_token and since_token != "END":
             since_token = RoomListNextBatch.from_token(since_token)
         else:
@@ -236,8 +217,7 @@ class RoomListHandler(BaseHandler):
             yield concurrently_execute(
                 lambda r: self._append_room_entry_to_chunk(
                     r, rooms_to_num_joined[r],
-                    chunk, limit, search_filter,
-                    from_federation=from_federation,
+                    chunk, limit, search_filter
                 ),
                 batch, 5,
             )
@@ -308,51 +288,23 @@ class RoomListHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit,
-                                    search_filter, from_federation=False):
+                                    search_filter):
         """Generate the entry for a room in the public room list and append it
         to the `chunk` if it matches the search filter
-
-        Args:
-            room_id (str): The ID of the room.
-            num_joined_users (int): The number of joined users in the room.
-            chunk (list)
-            limit (int|None): Maximum amount of rooms to display. Function will
-                return if length of chunk is greater than limit + 1.
-            search_filter (dict|None)
-            from_federation (bool): Whether this request originated from a
-                federating server or a client. Used for room filtering.
         """
         if limit and len(chunk) > limit + 1:
             # We've already got enough, so lets just drop it.
             return
 
         result = yield self.generate_room_entry(room_id, num_joined_users)
-        if not result:
-            return
 
-        if from_federation and not result.get("m.federate", True):
-            # This is a room that other servers cannot join. Do not show them
-            # this room.
-            return
-
-        if _matches_room_entry(result, search_filter):
+        if result and _matches_room_entry(result, search_filter):
             chunk.append(result)
 
-    @cachedInlineCallbacks(num_args=2, cache_context=True)
-    def generate_room_entry(self, room_id, num_joined_users,
-                            cache_context, with_alias=True, allow_private=False):
+    @cachedInlineCallbacks(num_args=1, cache_context=True)
+    def generate_room_entry(self, room_id, num_joined_users, cache_context,
+                            with_alias=True, allow_private=False):
         """Returns the entry for a room
-
-        Args:
-            room_id (str): The room's ID.
-            num_joined_users (int): Number of users in the room.
-            cache_context: Information for cached responses.
-            with_alias (bool): Whether to return the room's aliases in the result.
-            allow_private (bool): Whether invite-only rooms should be shown.
-
-        Returns:
-            Deferred[dict|None]: Returns a room entry as a dictionary, or None if this
-            room was determined not to be shown publicly.
         """
         result = {
             "room_id": room_id,
@@ -366,7 +318,6 @@ class RoomListHandler(BaseHandler):
         event_map = yield self.store.get_events([
             event_id for key, event_id in iteritems(current_state_ids)
             if key[0] in (
-                EventTypes.Create,
                 EventTypes.JoinRules,
                 EventTypes.Name,
                 EventTypes.Topic,
@@ -383,17 +334,12 @@ class RoomListHandler(BaseHandler):
         }
 
         # Double check that this is actually a public room.
-
         join_rules_event = current_state.get((EventTypes.JoinRules, ""))
         if join_rules_event:
             join_rule = join_rules_event.content.get("join_rule", None)
             if not allow_private and join_rule and join_rule != JoinRules.PUBLIC:
                 defer.returnValue(None)
 
-        # Return whether this room is open to federation users or not
-        create_event = current_state.get((EventTypes.Create, ""))
-        result["m.federate"] = create_event.content.get("m.federate", True)
-
         if with_alias:
             aliases = yield self.store.get_aliases_for_room(
                 room_id, on_invalidate=cache_context.invalidate
diff --git a/synapse/server.pyi b/synapse/server.pyi
index fb8df56cd5..06cd083a74 100644
--- a/synapse/server.pyi
+++ b/synapse/server.pyi
@@ -7,9 +7,9 @@ import synapse.handlers.auth
 import synapse.handlers.deactivate_account
 import synapse.handlers.device
 import synapse.handlers.e2e_keys
-import synapse.handlers.message
 import synapse.handlers.room
 import synapse.handlers.room_member
+import synapse.handlers.message
 import synapse.handlers.set_password
 import synapse.rest.media.v1.media_repository
 import synapse.server_notices.server_notices_manager

From 7a4632af9c4a26604b9d30f69eb81cb4630a82e7 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 22 Feb 2019 12:11:43 +0000
Subject: [PATCH 16/22] Prevent showing non-fed rooms in fed /publicRooms

---
 synapse/federation/transport/server.py |  3 +-
 synapse/groups/groups_server.py        |  6 +--
 synapse/handlers/room_list.py          | 62 +++++++++++++++++++++-----
 3 files changed, 54 insertions(+), 17 deletions(-)

diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 7288d49074..6d4a26f595 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -697,7 +697,8 @@ class PublicRoomList(BaseFederationServlet):
 
         data = yield self.handler.get_local_public_room_list(
             limit, since_token,
-            network_tuple=network_tuple
+            network_tuple=network_tuple,
+            from_federation=True,
         )
         defer.returnValue((200, data))
 
diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py
index 633c865ed8..a7eaead56b 100644
--- a/synapse/groups/groups_server.py
+++ b/synapse/groups/groups_server.py
@@ -113,8 +113,7 @@ class GroupsServerHandler(object):
             room_id = room_entry["room_id"]
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users),
-                with_alias=False, allow_private=True,
+                room_id, len(joined_users), with_alias=False, allow_private=True,
             )
             entry = dict(entry)  # so we don't change whats cached
             entry.pop("room_id", None)
@@ -544,8 +543,7 @@ class GroupsServerHandler(object):
 
             joined_users = yield self.store.get_users_in_room(room_id)
             entry = yield self.room_list_handler.generate_room_entry(
-                room_id, len(joined_users),
-                with_alias=False, allow_private=True,
+                room_id, len(joined_users), with_alias=False, allow_private=True,
             )
 
             if not entry:
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index dc88620885..b722e2175f 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -50,16 +50,17 @@ class RoomListHandler(BaseHandler):
 
     def get_local_public_room_list(self, limit=None, since_token=None,
                                    search_filter=None,
-                                   network_tuple=EMPTY_THIRD_PARTY_ID,):
+                                   network_tuple=EMPTY_THIRD_PARTY_ID,
+                                   from_federation=False):
         """Generate a local public room list.
 
         There are multiple different lists: the main one plus one per third
         party network. A client can ask for a specific list or to return all.
 
         Args:
-            limit (int)
-            since_token (str)
-            search_filter (dict)
+            limit (int|None)
+            since_token (str|None)
+            search_filter (dict|None)
             network_tuple (ThirdPartyInstanceID): Which public list to use.
                 This can be (None, None) to indicate the main list, or a particular
                 appservice and network id to use an appservice specific one.
@@ -81,13 +82,15 @@ class RoomListHandler(BaseHandler):
         return self.response_cache.wrap(
             key,
             self._get_public_room_list,
-            limit, since_token, network_tuple=network_tuple,
+            limit, since_token,
+            network_tuple=network_tuple, from_federation=from_federation,
         )
 
     @defer.inlineCallbacks
     def _get_public_room_list(self, limit=None, since_token=None,
                               search_filter=None,
-                              network_tuple=EMPTY_THIRD_PARTY_ID,):
+                              network_tuple=EMPTY_THIRD_PARTY_ID,
+                              from_federation=False,):
         if since_token and since_token != "END":
             since_token = RoomListNextBatch.from_token(since_token)
         else:
@@ -207,7 +210,8 @@ class RoomListHandler(BaseHandler):
             yield concurrently_execute(
                 lambda r: self._append_room_entry_to_chunk(
                     r, rooms_to_num_joined[r],
-                    chunk, limit, search_filter
+                    chunk, limit, search_filter,
+                    from_federation=from_federation,
                 ),
                 batch, 5,
             )
@@ -278,23 +282,51 @@ class RoomListHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _append_room_entry_to_chunk(self, room_id, num_joined_users, chunk, limit,
-                                    search_filter):
+                                    search_filter, from_federation=False):
         """Generate the entry for a room in the public room list and append it
         to the `chunk` if it matches the search filter
+
+        Args:
+            room_id (str): The ID of the room.
+            num_joined_users (int): The number of joined users in the room.
+            chunk (list)
+            limit (int|None): Maximum amount of rooms to display. Function will
+                return if length of chunk is greater than limit + 1.
+            search_filter (dict|None)
+            from_federation (bool): Whether this request originated from a
+                federating server or a client. Used for room filtering.
         """
         if limit and len(chunk) > limit + 1:
             # We've already got enough, so lets just drop it.
             return
 
         result = yield self.generate_room_entry(room_id, num_joined_users)
+        if not result:
+            return
 
-        if result and _matches_room_entry(result, search_filter):
+        if from_federation and not result.get("m.federate", True):
+            # This is a room that other servers cannot join. Do not show them
+            # this room.
+            return
+
+        if _matches_room_entry(result, search_filter):
             chunk.append(result)
 
-    @cachedInlineCallbacks(num_args=1, cache_context=True)
-    def generate_room_entry(self, room_id, num_joined_users, cache_context,
-                            with_alias=True, allow_private=False):
+    @cachedInlineCallbacks(num_args=2, cache_context=True)
+    def generate_room_entry(self, room_id, num_joined_users,
+                            cache_context, with_alias=True, allow_private=False):
         """Returns the entry for a room
+
+        Args:
+            room_id (str): The room's ID.
+            num_joined_users (int): Number of users in the room.
+            cache_context: Information for cached responses.
+            with_alias (bool): Whether to return the room's aliases in the result.
+            allow_private (bool): Whether invite-only rooms should be shown.
+
+        Returns:
+            Deferred[dict|None]: Returns a room entry as a dictionary, or None if this
+            room was determined not to be shown publicly.
         """
         result = {
             "room_id": room_id,
@@ -308,6 +340,7 @@ class RoomListHandler(BaseHandler):
         event_map = yield self.store.get_events([
             event_id for key, event_id in iteritems(current_state_ids)
             if key[0] in (
+                EventTypes.Create,
                 EventTypes.JoinRules,
                 EventTypes.Name,
                 EventTypes.Topic,
@@ -324,12 +357,17 @@ class RoomListHandler(BaseHandler):
         }
 
         # Double check that this is actually a public room.
+
         join_rules_event = current_state.get((EventTypes.JoinRules, ""))
         if join_rules_event:
             join_rule = join_rules_event.content.get("join_rule", None)
             if not allow_private and join_rule and join_rule != JoinRules.PUBLIC:
                 defer.returnValue(None)
 
+        # Return whether this room is open to federation users or not
+        create_event = current_state.get((EventTypes.Create, ""))
+        result["m.federate"] = create_event.content.get("m.federate", True)
+
         if with_alias:
             aliases = yield self.store.get_aliases_for_room(
                 room_id, on_invalidate=cache_context.invalidate

From 6fcb25202f34348e3411460496296252a2491bce Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 14:13:38 +0000
Subject: [PATCH 17/22] Put function def back to the way it was

---
 synapse/handlers/room_list.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
index b722e2175f..4f51a464e7 100644
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -312,9 +312,9 @@ class RoomListHandler(BaseHandler):
         if _matches_room_entry(result, search_filter):
             chunk.append(result)
 
-    @cachedInlineCallbacks(num_args=2, cache_context=True)
-    def generate_room_entry(self, room_id, num_joined_users,
-                            cache_context, with_alias=True, allow_private=False):
+    @cachedInlineCallbacks(num_args=1, cache_context=True)
+    def generate_room_entry(self, room_id, num_joined_users, cache_context,
+                            with_alias=True, allow_private=False):
         """Returns the entry for a room
 
         Args:

From b8e6ed36c1d58039db18b8cfa6a41389e25c159c Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 26 Feb 2019 13:46:45 +0000
Subject: [PATCH 18/22] Add changelog

---
 changelog.d/4746.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4746.feature

diff --git a/changelog.d/4746.feature b/changelog.d/4746.feature
new file mode 100644
index 0000000000..97c253eccf
--- /dev/null
+++ b/changelog.d/4746.feature
@@ -0,0 +1 @@
+Prevent showing rooms to other servers that were set to not federate.
\ No newline at end of file

From 4bc74835185a42fae0004c30e55f61f5bb8c78d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Paul=20T=C3=B6tterman?= <ptman@users.noreply.github.com>
Date: Tue, 26 Feb 2019 20:01:45 +0200
Subject: [PATCH 19/22] Fix apache reverse proxy example (#4742)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

So that it actually works. See https://httpd.apache.org/docs/2.4/mod/mod_proxy.html#proxypass

Signed-off-by: Paul Tötterman <paul.totterman@iki.fi>
---
 docs/reverse_proxy.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reverse_proxy.rst b/docs/reverse_proxy.rst
index 242935a62f..4706061eba 100644
--- a/docs/reverse_proxy.rst
+++ b/docs/reverse_proxy.rst
@@ -79,7 +79,7 @@ Let's assume that we expect clients to connect to our server at
           SSLEngine on
           ServerName example.com;
 
-          <Location />
+          <Location /_matrix>
               ProxyPass http://127.0.0.1:8008/_matrix nocanon
               ProxyPassReverse http://127.0.0.1:8008/_matrix
           </Location>

From 57426ec6a377015925d643576efa9ef3efd63fda Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 26 Feb 2019 21:13:41 +0000
Subject: [PATCH 20/22] Fix check-newsfragment script (#4750)

* Fix check-newsfragment script

I previously broke this so that it always succeeded...

* more fixes

* fix newsfiles
---
 changelog.d/4698.misc          |  2 +-
 changelog.d/4750.misc          |  1 +
 scripts-dev/check-newsfragment | 11 ++++++++---
 3 files changed, 10 insertions(+), 4 deletions(-)
 create mode 100644 changelog.d/4750.misc

diff --git a/changelog.d/4698.misc b/changelog.d/4698.misc
index d17b19bec5..9dea5dd2be 100644
--- a/changelog.d/4698.misc
+++ b/changelog.d/4698.misc
@@ -1 +1 @@
-Better checks on newsfragments
+Better checks on newsfragments.
diff --git a/changelog.d/4750.misc b/changelog.d/4750.misc
new file mode 100644
index 0000000000..3bb9c48f1a
--- /dev/null
+++ b/changelog.d/4750.misc
@@ -0,0 +1 @@
+Better checks on newsfragments.
\ No newline at end of file
diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
index 5da093e168..e4a22bae61 100755
--- a/scripts-dev/check-newsfragment
+++ b/scripts-dev/check-newsfragment
@@ -6,7 +6,8 @@
 set -e
 
 # make sure that origin/develop is up to date
-git fetch origin develop
+git remote set-branches --add origin develop
+git fetch --depth=1 origin develop
 
 UPSTREAM=origin/develop
 
@@ -25,11 +26,15 @@ if git diff --name-only $UPSTREAM... | grep -qv '^develop/'; then
     tox -e check-newsfragment
 fi
 
+echo
+echo "--------------------------"
+echo
+
 # check that any new newsfiles on this branch end with a full stop.
-for f in git diff --name-only $UPSTREAM... -- changelog.d; do
+for f in `git diff --name-only $UPSTREAM... -- changelog.d`; do
     lastchar=`tr -d '\n' < $f | tail -c 1`
     if [ $lastchar != '.' ]; then
-        echo "Newsfragment $f does not end with a '.'" >&2
+        echo -e "\e[31mERROR: newsfragment $f does not end with a '.'\e[39m" >&2
         exit 1
     fi
 done

From f191be822b265b50339e17b2d95125f43dc700b3 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Wed, 27 Feb 2019 10:21:49 +0000
Subject: [PATCH 21/22] Add database version to phonehome stats. (#4753)

---
 changelog.d/4753.misc               |  1 +
 synapse/app/homeserver.py           |  3 +++
 synapse/storage/_base.py            |  8 ++++++++
 synapse/storage/engines/postgres.py | 25 +++++++++++++++++++++++++
 synapse/storage/engines/sqlite.py   |  9 +++++++++
 5 files changed, 46 insertions(+)
 create mode 100644 changelog.d/4753.misc

diff --git a/changelog.d/4753.misc b/changelog.d/4753.misc
new file mode 100644
index 0000000000..98532cc971
--- /dev/null
+++ b/changelog.d/4753.misc
@@ -0,0 +1 @@
+Add database version to phonehome stats.
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 05a97979ec..e8b6cc3114 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -555,6 +555,9 @@ def run(hs):
                 stats["memory_rss"] += process.memory_info().rss
                 stats["cpu_average"] += int(process.cpu_percent(interval=None))
 
+        stats["database_engine"] = hs.get_datastore().database_engine_name
+        stats["database_server_version"] = hs.get_datastore().get_server_version()
+
         logger.info("Reporting stats to matrix.org: %s" % (stats,))
         try:
             yield hs.get_simple_http_client().put_json(
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 5a80eef211..190be34fb1 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -1596,6 +1596,14 @@ class SQLBaseStore(object):
 
         return cls.cursor_to_dict(txn)
 
+    @property
+    def database_engine_name(self):
+        return self.database_engine.module.__name__
+
+    def get_server_version(self):
+        """Returns a string describing the server version number"""
+        return self.database_engine.server_version
+
 
 class _RollbackButIsFineException(Exception):
     """ This exception is used to rollback a transaction without implying
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 4004427c7b..dc3238501c 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -23,6 +23,7 @@ class PostgresEngine(object):
         self.module = database_module
         self.module.extensions.register_type(self.module.extensions.UNICODE)
         self.synchronous_commit = database_config.get("synchronous_commit", True)
+        self._version = None   # unknown as yet
 
     def check_database(self, txn):
         txn.execute("SHOW SERVER_ENCODING")
@@ -87,3 +88,27 @@ class PostgresEngine(object):
         """
         txn.execute("SELECT nextval('state_group_id_seq')")
         return txn.fetchone()[0]
+
+    @property
+    def server_version(self):
+        """Returns a string giving the server version. For example: '8.1.5'
+
+        Returns:
+            string
+        """
+        # note that this is a bit of a hack because it relies on on_new_connection
+        # having been called at least once. Still, that should be a safe bet here.
+        numver = self._version
+        assert numver is not None
+
+        # https://www.postgresql.org/docs/current/libpq-status.html#LIBPQ-PQSERVERVERSION
+        if numver >= 100000:
+            return "%i.%i" % (
+                numver / 10000, numver % 10000,
+            )
+        else:
+            return "%i.%i.%i" % (
+                numver / 10000,
+                (numver % 10000) / 100,
+                numver % 100,
+            )
diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py
index 059ab81055..1bcd5b99a4 100644
--- a/synapse/storage/engines/sqlite.py
+++ b/synapse/storage/engines/sqlite.py
@@ -70,6 +70,15 @@ class Sqlite3Engine(object):
             self._current_state_group_id += 1
             return self._current_state_group_id
 
+    @property
+    def server_version(self):
+        """Gets a string giving the server version. For example: '3.22.0'
+
+        Returns:
+            string
+        """
+        return "%i.%i.%i" % self.module.sqlite_version_info
+
 
 # Following functions taken from: https://github.com/coleifer/peewee
 

From 6bb1c028f190d8ba561a6deaa474c060efb7c502 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erikj@jki.re>
Date: Wed, 27 Feb 2019 10:28:37 +0000
Subject: [PATCH 22/22] Limit cache invalidation replication line length
 (#4748)

---
 changelog.d/4748.misc               |  1 +
 synapse/replication/tcp/protocol.py | 17 ++++++++++++++++-
 synapse/storage/_base.py            | 15 +++++++++++----
 3 files changed, 28 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/4748.misc

diff --git a/changelog.d/4748.misc b/changelog.d/4748.misc
new file mode 100644
index 0000000000..4dc18378e7
--- /dev/null
+++ b/changelog.d/4748.misc
@@ -0,0 +1 @@
+Improve replication performance by reducing cache invalidation traffic.
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index 0b3fe6cbf5..429471c345 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -268,7 +268,17 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
         if "\n" in string:
             raise Exception("Unexpected newline in command: %r", string)
 
-        self.sendLine(string.encode("utf-8"))
+        encoded_string = string.encode("utf-8")
+
+        if len(encoded_string) > self.MAX_LENGTH:
+            raise Exception(
+                "Failed to send command %s as too long (%d > %d)" % (
+                    cmd.NAME,
+                    len(encoded_string), self.MAX_LENGTH,
+                )
+            )
+
+        self.sendLine(encoded_string)
 
         self.last_sent_command = self.clock.time_msec()
 
@@ -361,6 +371,11 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
     def id(self):
         return "%s-%s" % (self.name, self.conn_id)
 
+    def lineLengthExceeded(self, line):
+        """Called when we receive a line that is above the maximum line length
+        """
+        self.send_error("Line length exceeded")
+
 
 class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol):
     VALID_INBOUND_COMMANDS = VALID_CLIENT_COMMANDS
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 190be34fb1..a0333d5309 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -30,6 +30,7 @@ from synapse.api.errors import StoreError
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.engines import PostgresEngine, Sqlite3Engine
 from synapse.types import get_domain_from_id
+from synapse.util import batch_iter
 from synapse.util.caches.descriptors import Cache
 from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
 from synapse.util.stringutils import exception_to_unicode
@@ -1327,10 +1328,16 @@ class SQLBaseStore(object):
         """
         txn.call_after(self._invalidate_state_caches, room_id, members_changed)
 
-        keys = itertools.chain([room_id], members_changed)
-        self._send_invalidation_to_replication(
-            txn, _CURRENT_STATE_CACHE_NAME, keys,
-        )
+        # We need to be careful that the size of the `members_changed` list
+        # isn't so large that it causes problems sending over replication, so we
+        # send them in chunks.
+        # Max line length is 16K, and max user ID length is 255, so 50 should
+        # be safe.
+        for chunk in batch_iter(members_changed, 50):
+            keys = itertools.chain([room_id], chunk)
+            self._send_invalidation_to_replication(
+                txn, _CURRENT_STATE_CACHE_NAME, keys,
+            )
 
     def _invalidate_state_caches(self, room_id, members_changed):
         """Invalidates caches that are based on the current state, but does