Merge branch 'develop' of github.com:matrix-org/synapse into matrix-org-hotfixes

CHANGES.md

@@ -1,5 +1,15 @@
-Synapse 0.99.1rc1 (2019-02-12)
+Synapse 0.99.1.1 (2019-02-14)
-==============================
+=============================
+
+Bugfixes
+--------
+
+- Fix "TypeError: '>' not supported" when starting without an existing certificate.
+  Fix a bug where an existing certificate would be reprovisoned every day. ([\#4648](https://github.com/matrix-org/synapse/issues/4648))
+
+
+Synapse 0.99.1 (2019-02-14)
+===========================
 
 Features
 --------
@@ -10,7 +20,7 @@ Features
 - Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580))
 - Allow the "unavailable" presence status for /sync.
   This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592))
-- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617))
+- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617), [\#4636](https://github.com/matrix-org/synapse/issues/4636))
 - The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614))
 
 

changelog.d/4450.bugfix

@@ -0,0 +1,2 @@
+The dependency checker now correctly reports a version mismatch for optional
+dependencies, instead of reporting the dependency missing.

changelog.d/4635.misc

@@ -0,0 +1 @@
+Run `black` to reformat user directory code.

changelog.d/4636.bugfix

@@ -1 +0,0 @@
-Fix errors when using default bind_addresses with replication/metrics listeners.

changelog.d/4647.feature

@@ -0,0 +1 @@
+Add configurable room list publishing rules

debian/changelog

@@ -1,3 +1,19 @@
+matrix-synapse-py3 (0.99.1.1) stable; urgency=medium
+
+  * New synapse release 0.99.1.1
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 17:19:44 +0000
+
+matrix-synapse-py3 (0.99.1) stable; urgency=medium
+
+  [ Damjan Georgievski ]
+  * Added ExecReload= in service unit file to send a HUP signal
+
+  [ Synapse Packaging team ]
+  * New synapse release 0.99.1
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 14:12:26 +0000
+
 matrix-synapse-py3 (0.99.0) stable; urgency=medium
 
   * New synapse release 0.99.0

debian/matrix-synapse.service

@@ -8,6 +8,7 @@ WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse
 ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
 ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
+ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
 RestartSec=3
 

docker/Dockerfile-dhvirtualenv

@@ -58,7 +58,11 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         sqlite3
 
 COPY --from=builder /dh-virtualenv_1.1-1_all.deb /
-RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb
+
+# install dhvirtualenv. Update the apt cache again first, in case we got a
+# cached cache from docker the first time.
+RUN apt-get update -qq -o Acquire::Languages=none \
+    && apt-get install -yq /dh-virtualenv_1.1-1_all.deb
 
 WORKDIR /synapse/source
 ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]

docs/MSC1711_certificates_FAQ.md

@@ -125,7 +125,7 @@ doing one of the following:
  * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the
    `server_name` domain to your Synapse instance.
 
-### Option 2: run Synapse behind a reverse proxy
+#### Option 2: run Synapse behind a reverse proxy
 
 If you have an existing reverse proxy set up with correct TLS certificates for
 your domain, you can simply route all traffic through the reverse proxy by

synapse/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
+# Copyright 2018-9 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,4 +27,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "0.99.1rc1"
+__version__ = "0.99.1.1"

synapse/app/homeserver.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
+# Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 import gc
 import logging
 import os
 import sys
-import traceback
 
 from six import iteritems
 
@@ -27,6 +29,7 @@ from prometheus_client import Gauge
 
 from twisted.application import service
 from twisted.internet import defer, reactor
+from twisted.python.failure import Failure
 from twisted.web.resource import EncodingResourceWrapper, NoResource
 from twisted.web.server import GzipEncoderFactory
 from twisted.web.static import File
@@ -394,10 +397,10 @@ def setup(config_options):
         # is less than our re-registration threshold.
         provision = False
 
-        if (cert_days_remaining is None):
+        if (
-            provision = True
+            cert_days_remaining is None or
-
+            cert_days_remaining < hs.config.acme_reprovision_threshold
-        if cert_days_remaining > hs.config.acme_reprovision_threshold:
+        ):
             provision = True
 
         if provision:
@@ -438,7 +441,11 @@ def setup(config_options):
             hs.get_datastore().start_doing_background_updates()
         except Exception:
             # Print the exception and bail out.
-            traceback.print_exc(file=sys.stderr)
+            print("Error during startup:", file=sys.stderr)
+
+            # this gives better tracebacks than traceback.print_exc()
+            Failure().printTraceback(file=sys.stderr)
+
             if reactor.running:
                 reactor.stop()
             sys.exit(1)

synapse/config/logger.py

@@ -242,3 +242,5 @@ def setup_logging(config, use_worker_options=False):
         [_log],
         redirectStandardIO=not config.no_redirect_stdio,
     )
+    if not config.no_redirect_stdio:
+        print("Redirected stdout/stderr to logs")

synapse/config/room_directory.py

@@ -20,12 +20,37 @@ from ._base import Config, ConfigError
 
 class RoomDirectoryConfig(Config):
     def read_config(self, config):
-        alias_creation_rules = config["alias_creation_rules"]
+        alias_creation_rules = config.get("alias_creation_rules")
 
+        if alias_creation_rules is not None:
             self._alias_creation_rules = [
-            _AliasRule(rule)
+                _RoomDirectoryRule("alias_creation_rules", rule)
                 for rule in alias_creation_rules
             ]
+        else:
+            self._alias_creation_rules = [
+                _RoomDirectoryRule(
+                    "alias_creation_rules", {
+                        "action": "allow",
+                    }
+                )
+            ]
+
+        room_list_publication_rules = config.get("room_list_publication_rules")
+
+        if room_list_publication_rules is not None:
+            self._room_list_publication_rules = [
+                _RoomDirectoryRule("room_list_publication_rules", rule)
+                for rule in room_list_publication_rules
+            ]
+        else:
+            self._room_list_publication_rules = [
+                _RoomDirectoryRule(
+                    "room_list_publication_rules", {
+                        "action": "allow",
+                    }
+                )
+            ]
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
@@ -33,60 +58,138 @@ class RoomDirectoryConfig(Config):
         # on this server.
         #
         # The format of this option is a list of rules that contain globs that
-        # match against user_id and the new alias (fully qualified with server
+        # match against user_id, room_id and the new alias (fully qualified with
-        # name). The action in the first rule that matches is taken, which can
+        # server name). The action in the first rule that matches is taken,
-        # currently either be "allow" or "deny".
+        # which can currently either be "allow" or "deny".
         #
-        # If no rules match the request is denied.
+        # Missing user_id/room_id/alias fields default to "*".
-        alias_creation_rules:
+        #
-            - user_id: "*"
+        # If no rules match the request is denied. An empty list means no one
-              alias: "*"
+        # can create aliases.
-              action: allow
+        #
+        # Options for the rules include:
+        #
+        #   user_id: Matches against the creator of the alias
+        #   alias: Matches against the alias being created
+        #   room_id: Matches against the room ID the alias is being pointed at
+        #   action: Whether to "allow" or "deny" the request if the rule matches
+        #
+        # The default is:
+        #
+        # alias_creation_rules:
+        #     - user_id: "*"
+        #       alias: "*"
+        #       room_id: "*"
+        #       action: allow
+
+        # The `room_list_publication_rules` option controls who can publish and
+        # which rooms can be published in the public room list.
+        #
+        # The format of this option is the same as that for
+        # `alias_creation_rules`.
+        #
+        # If the room has one or more aliases associated with it, only one of
+        # the aliases needs to match the alias rule. If there are no aliases
+        # then only rules with `alias: *` match.
+        #
+        # If no rules match the request is denied. An empty list means no one
+        # can publish rooms.
+        #
+        # Options for the rules include:
+        #
+        #   user_id: Matches agaisnt the creator of the alias
+        #   room_id: Matches against the room ID being published
+        #   alias: Matches against any current local or canonical aliases
+        #            associated with the room
+        #   action: Whether to "allow" or "deny" the request if the rule matches
+        #
+        # The default is:
+        #
+        # room_list_publication_rules:
+        #     - user_id: "*"
+        #       alias: "*"
+        #       room_id: "*"
+        #       action: allow
         """
 
-    def is_alias_creation_allowed(self, user_id, alias):
+    def is_alias_creation_allowed(self, user_id, room_id, alias):
         """Checks if the given user is allowed to create the given alias
 
         Args:
             user_id (str)
+            room_id (str)
             alias (str)
 
         Returns:
             boolean: True if user is allowed to crate the alias
         """
         for rule in self._alias_creation_rules:
-            if rule.matches(user_id, alias):
+            if rule.matches(user_id, room_id, [alias]):
+                return rule.action == "allow"
+
+        return False
+
+    def is_publishing_room_allowed(self, user_id, room_id, aliases):
+        """Checks if the given user is allowed to publish the room
+
+        Args:
+            user_id (str)
+            room_id (str)
+            aliases (list[str]): any local aliases associated with the room
+
+        Returns:
+            boolean: True if user can publish room
+        """
+        for rule in self._room_list_publication_rules:
+            if rule.matches(user_id, room_id, aliases):
                 return rule.action == "allow"
 
         return False
 
 
-class _AliasRule(object):
+class _RoomDirectoryRule(object):
-    def __init__(self, rule):
+    """Helper class to test whether a room directory action is allowed, like
+    creating an alias or publishing a room.
+    """
+
+    def __init__(self, option_name, rule):
+        """
+        Args:
+            option_name (str): Name of the config option this rule belongs to
+            rule (dict): The rule as specified in the config
+        """
+
         action = rule["action"]
-        user_id = rule["user_id"]
+        user_id = rule.get("user_id", "*")
-        alias = rule["alias"]
+        room_id = rule.get("room_id", "*")
+        alias = rule.get("alias", "*")
 
         if action in ("allow", "deny"):
             self.action = action
         else:
             raise ConfigError(
-                "alias_creation_rules rules can only have action of 'allow'"
+                "%s rules can only have action of 'allow'"
-                " or 'deny'"
+                " or 'deny'" % (option_name,)
             )
 
+        self._alias_matches_all = alias == "*"
+
         try:
             self._user_id_regex = glob_to_regex(user_id)
             self._alias_regex = glob_to_regex(alias)
+            self._room_id_regex = glob_to_regex(room_id)
         except Exception as e:
             raise ConfigError("Failed to parse glob into regex: %s", e)
 
-    def matches(self, user_id, alias):
+    def matches(self, user_id, room_id, aliases):
-        """Tests if this rule matches the given user_id and alias.
+        """Tests if this rule matches the given user_id, room_id and aliases.
 
         Args:
             user_id (str)
-            alias (str)
+            room_id (str)
+            aliases (list[str]): The associated aliases to the room. Will be a
+                single element for testing alias creation, and can be empty for
+                testing room publishing.
 
         Returns:
             boolean
@@ -96,7 +199,22 @@ class _AliasRule(object):
         if not self._user_id_regex.match(user_id):
             return False
 
-        if not self._alias_regex.match(alias):
+        if not self._room_id_regex.match(room_id):
             return False
 
+        # We only have alias checks left, so we can short circuit if the alias
+        # rule matches everything.
+        if self._alias_matches_all:
             return True
+
+        # If we are not given any aliases then this rule only matches if the
+        # alias glob matches all aliases, which we checked above.
+        if not aliases:
+            return False
+
+        # Otherwise, we just need one alias to match
+        for alias in aliases:
+            if self._alias_regex.match(alias):
+                return True
+
+        return False

synapse/handlers/directory.py

@@ -112,7 +112,9 @@ class DirectoryHandler(BaseHandler):
                     403, "This user is not permitted to create this alias",
                 )
 
-            if not self.config.is_alias_creation_allowed(user_id, room_alias.to_string()):
+            if not self.config.is_alias_creation_allowed(
+                user_id, room_id, room_alias.to_string(),
+            ):
                 # Lets just return a generic message, as there may be all sorts of
                 # reasons why we said no. TODO: Allow configurable error messages
                 # per alias creation rule?
@@ -395,9 +397,9 @@ class DirectoryHandler(BaseHandler):
         room_id (str)
         visibility (str): "public" or "private"
         """
-        if not self.spam_checker.user_may_publish_room(
+        user_id = requester.user.to_string()
-            requester.user.to_string(), room_id
+
-        ):
+        if not self.spam_checker.user_may_publish_room(user_id, room_id):
             raise AuthError(
                 403,
                 "This user is not permitted to publish rooms to the room list"
@@ -415,7 +417,24 @@ class DirectoryHandler(BaseHandler):
 
         yield self.auth.check_can_change_room_list(room_id, requester.user)
 
-        yield self.store.set_room_is_public(room_id, visibility == "public")
+        making_public = visibility == "public"
+        if making_public:
+            room_aliases = yield self.store.get_aliases_for_room(room_id)
+            canonical_alias = yield self.store.get_canonical_alias_for_room(room_id)
+            if canonical_alias:
+                room_aliases.append(canonical_alias)
+
+            if not self.config.is_publishing_room_allowed(
+                user_id, room_id, room_aliases,
+            ):
+                # Lets just return a generic message, as there may be all sorts of
+                # reasons why we said no. TODO: Allow configurable error messages
+                # per alias creation rule?
+                raise SynapseError(
+                    403, "Not allowed to publish room",
+                )
+
+        yield self.store.set_room_is_public(room_id, making_public)
 
     @defer.inlineCallbacks
     def edit_published_appservice_room_list(self, appservice_id, network_id,

synapse/python_dependencies.py

@@ -143,9 +143,12 @@ def check_requirements(for_feature=None, _get_distribution=get_distribution):
         for dependency in OPTS:
             try:
                 _get_distribution(dependency)
-            except VersionConflict:
+            except VersionConflict as e:
                 deps_needed.append(dependency)
-                errors.append("Needed %s but it was not installed" % (dependency,))
+                errors.append(
+                    "Needed optional %s, got %s==%s"
+                    % (dependency, e.dist.project_name, e.dist.version)
+                )
             except DistributionNotFound:
                 # If it's not found, we don't care
                 pass

synapse/storage/state.py

@@ -548,6 +548,31 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
             _get_filtered_current_state_ids_txn,
         )
 
+    @defer.inlineCallbacks
+    def get_canonical_alias_for_room(self, room_id):
+        """Get canonical alias for room, if any
+
+        Args:
+            room_id (str)
+
+        Returns:
+            Deferred[str|None]: The canonical alias, if any
+        """
+
+        state = yield self.get_filtered_current_state_ids(room_id, StateFilter.from_types(
+            [(EventTypes.CanonicalAlias, "")]
+        ))
+
+        event_id = state.get((EventTypes.CanonicalAlias, ""))
+        if not event_id:
+            return
+
+        event = yield self.get_event(event_id, allow_none=True)
+        if not event:
+            return
+
+        defer.returnValue(event.content.get("canonical_alias"))
+
     @cached(max_entries=10000, iterable=True)
     def get_state_group_delta(self, state_group):
         """Given a state group try to return a previous group and a delta between

synapse/storage/user_directory.py

@@ -44,7 +44,7 @@ class UserDirectoryStore(SQLBaseStore):
         )
 
         current_state_ids = yield self.get_filtered_current_state_ids(
-            room_id, StateFilter.from_types(types_to_filter),
+            room_id, StateFilter.from_types(types_to_filter)
         )
 
         join_rules_id = current_state_ids.get((EventTypes.JoinRules, ""))
@@ -74,14 +74,8 @@ class UserDirectoryStore(SQLBaseStore):
         """
         yield self._simple_insert_many(
             table="users_in_public_rooms",
-            values=[
+            values=[{"user_id": user_id, "room_id": room_id} for user_id in user_ids],
-                {
+            desc="add_users_to_public_room",
-                    "user_id": user_id,
-                    "room_id": room_id,
-                }
-                for user_id in user_ids
-            ],
-            desc="add_users_to_public_room"
         )
         for user_id in user_ids:
             self.get_user_in_public_room.invalidate((user_id,))
@@ -107,7 +101,9 @@ class UserDirectoryStore(SQLBaseStore):
             """
             args = (
                 (
-                    user_id, get_localpart_from_id(user_id), get_domain_from_id(user_id),
+                    user_id,
+                    get_localpart_from_id(user_id),
+                    get_domain_from_id(user_id),
                     profile.display_name,
                 )
                 for user_id, profile in iteritems(users_with_profile)
@@ -120,7 +116,7 @@ class UserDirectoryStore(SQLBaseStore):
             args = (
                 (
                     user_id,
-                    "%s %s" % (user_id, p.display_name,) if p.display_name else user_id
+                    "%s %s" % (user_id, p.display_name) if p.display_name else user_id,
                 )
                 for user_id, p in iteritems(users_with_profile)
             )
@@ -141,12 +137,10 @@ class UserDirectoryStore(SQLBaseStore):
                         "avatar_url": profile.avatar_url,
                     }
                     for user_id, profile in iteritems(users_with_profile)
-                ]
+                ],
             )
             for user_id in users_with_profile:
-                txn.call_after(
+                txn.call_after(self.get_user_in_directory.invalidate, (user_id,))
-                    self.get_user_in_directory.invalidate, (user_id,)
-                )
 
         return self.runInteraction(
             "add_profiles_to_user_dir", _add_profiles_to_user_dir_txn
@@ -188,9 +182,11 @@ class UserDirectoryStore(SQLBaseStore):
                     txn.execute(
                         sql,
                         (
-                            user_id, get_localpart_from_id(user_id),
+                            user_id,
-                            get_domain_from_id(user_id), display_name,
+                            get_localpart_from_id(user_id),
-                        )
+                            get_domain_from_id(user_id),
+                            display_name,
+                        ),
                     )
                 else:
                     # TODO: Remove this code after we've bumped the minimum version
@@ -208,9 +204,11 @@ class UserDirectoryStore(SQLBaseStore):
                         txn.execute(
                             sql,
                             (
-                                user_id, get_localpart_from_id(user_id),
+                                user_id,
-                                get_domain_from_id(user_id), display_name,
+                                get_localpart_from_id(user_id),
-                            )
+                                get_domain_from_id(user_id),
+                                display_name,
+                            ),
                         )
                     elif new_entry is False:
                         sql = """
@@ -225,15 +223,16 @@ class UserDirectoryStore(SQLBaseStore):
                             (
                                 get_localpart_from_id(user_id),
                                 get_domain_from_id(user_id),
-                                display_name, user_id,
+                                display_name,
-                            )
+                                user_id,
+                            ),
                         )
                     else:
                         raise RuntimeError(
                             "upsert returned None when 'can_native_upsert' is False"
                         )
             elif isinstance(self.database_engine, Sqlite3Engine):
-                value = "%s %s" % (user_id, display_name,) if display_name else user_id
+                value = "%s %s" % (user_id, display_name) if display_name else user_id
                 self._simple_upsert_txn(
                     txn,
                     table="user_directory_search",
@@ -264,29 +263,18 @@ class UserDirectoryStore(SQLBaseStore):
     def remove_from_user_dir(self, user_id):
         def _remove_from_user_dir_txn(txn):
             self._simple_delete_txn(
-                txn,
+                txn, table="user_directory", keyvalues={"user_id": user_id}
-                table="user_directory",
-                keyvalues={"user_id": user_id},
             )
             self._simple_delete_txn(
-                txn,
+                txn, table="user_directory_search", keyvalues={"user_id": user_id}
-                table="user_directory_search",
-                keyvalues={"user_id": user_id},
             )
             self._simple_delete_txn(
-                txn,
+                txn, table="users_in_public_rooms", keyvalues={"user_id": user_id}
-                table="users_in_public_rooms",
-                keyvalues={"user_id": user_id},
-            )
-            txn.call_after(
-                self.get_user_in_directory.invalidate, (user_id,)
-            )
-            txn.call_after(
-                self.get_user_in_public_room.invalidate, (user_id,)
-            )
-        return self.runInteraction(
-            "remove_from_user_dir", _remove_from_user_dir_txn,
             )
+            txn.call_after(self.get_user_in_directory.invalidate, (user_id,))
+            txn.call_after(self.get_user_in_public_room.invalidate, (user_id,))
+
+        return self.runInteraction("remove_from_user_dir", _remove_from_user_dir_txn)
 
     @defer.inlineCallbacks
     def remove_from_user_in_public_room(self, user_id):
@@ -371,6 +359,7 @@ class UserDirectoryStore(SQLBaseStore):
             share_private (bool): Is the room private
             user_id_tuples([(str, str)]): iterable of 2-tuple of user IDs.
         """
+
         def _add_users_who_share_room_txn(txn):
             self._simple_insert_many_txn(
                 txn,
@@ -387,13 +376,12 @@ class UserDirectoryStore(SQLBaseStore):
             )
             for user_id, other_user_id in user_id_tuples:
                 txn.call_after(
-                    self.get_users_who_share_room_from_dir.invalidate,
+                    self.get_users_who_share_room_from_dir.invalidate, (user_id,)
-                    (user_id,),
                 )
                 txn.call_after(
-                    self.get_if_users_share_a_room.invalidate,
+                    self.get_if_users_share_a_room.invalidate, (user_id, other_user_id)
-                    (user_id, other_user_id),
                 )
+
         return self.runInteraction(
             "add_users_who_share_room", _add_users_who_share_room_txn
         )
@@ -407,6 +395,7 @@ class UserDirectoryStore(SQLBaseStore):
             share_private (bool): Is the room private
             user_id_tuples([(str, str)]): iterable of 2-tuple of user IDs.
         """
+
         def _update_users_who_share_room_txn(txn):
             sql = """
                 UPDATE users_who_share_rooms
@@ -414,21 +403,16 @@ class UserDirectoryStore(SQLBaseStore):
                 WHERE user_id = ? AND other_user_id = ?
             """
             txn.executemany(
-                sql,
+                sql, ((room_id, share_private, uid, oid) for uid, oid in user_id_sets)
-                (
-                    (room_id, share_private, uid, oid)
-                    for uid, oid in user_id_sets
-                )
             )
             for user_id, other_user_id in user_id_sets:
                 txn.call_after(
-                    self.get_users_who_share_room_from_dir.invalidate,
+                    self.get_users_who_share_room_from_dir.invalidate, (user_id,)
-                    (user_id,),
                 )
                 txn.call_after(
-                    self.get_if_users_share_a_room.invalidate,
+                    self.get_if_users_share_a_room.invalidate, (user_id, other_user_id)
-                    (user_id, other_user_id),
                 )
+
         return self.runInteraction(
             "update_users_who_share_room", _update_users_who_share_room_txn
         )
@@ -442,22 +426,18 @@ class UserDirectoryStore(SQLBaseStore):
             share_private (bool): Is the room private
             user_id_tuples([(str, str)]): iterable of 2-tuple of user IDs.
         """
+
         def _remove_user_who_share_room_txn(txn):
             self._simple_delete_txn(
                 txn,
                 table="users_who_share_rooms",
-                keyvalues={
+                keyvalues={"user_id": user_id, "other_user_id": other_user_id},
-                    "user_id": user_id,
-                    "other_user_id": other_user_id,
-                },
             )
             txn.call_after(
-                self.get_users_who_share_room_from_dir.invalidate,
+                self.get_users_who_share_room_from_dir.invalidate, (user_id,)
-                (user_id,),
             )
             txn.call_after(
-                self.get_if_users_share_a_room.invalidate,
+                self.get_if_users_share_a_room.invalidate, (user_id, other_user_id)
-                (user_id, other_user_id),
             )
 
         return self.runInteraction(
@@ -478,10 +458,7 @@ class UserDirectoryStore(SQLBaseStore):
         """
         return self._simple_select_one_onecol(
             table="users_who_share_rooms",
-            keyvalues={
+            keyvalues={"user_id": user_id, "other_user_id": other_user_id},
-                "user_id": user_id,
-                "other_user_id": other_user_id,
-            },
             retcol="share_private",
             allow_none=True,
             desc="get_if_users_share_a_room",
@@ -499,17 +476,12 @@ class UserDirectoryStore(SQLBaseStore):
         """
         rows = yield self._simple_select_list(
             table="users_who_share_rooms",
-            keyvalues={
+            keyvalues={"user_id": user_id},
-                "user_id": user_id,
+            retcols=("other_user_id", "share_private"),
-            },
-            retcols=("other_user_id", "share_private",),
             desc="get_users_who_share_room_with_user",
         )
 
-        defer.returnValue({
+        defer.returnValue({row["other_user_id"]: row["share_private"] for row in rows})
-            row["other_user_id"]: row["share_private"]
-            for row in rows
-        })
 
     def get_users_in_share_dir_with_room_id(self, user_id, room_id):
         """Get all user tuples that are in the users_who_share_rooms due to the
@@ -556,6 +528,7 @@ class UserDirectoryStore(SQLBaseStore):
     def delete_all_from_user_dir(self):
         """Delete the entire user directory
         """
+
         def _delete_all_from_user_dir_txn(txn):
             txn.execute("DELETE FROM user_directory")
             txn.execute("DELETE FROM user_directory_search")
@@ -565,6 +538,7 @@ class UserDirectoryStore(SQLBaseStore):
             txn.call_after(self.get_user_in_public_room.invalidate_all)
             txn.call_after(self.get_users_who_share_room_from_dir.invalidate_all)
             txn.call_after(self.get_if_users_share_a_room.invalidate_all)
+
         return self.runInteraction(
             "delete_all_from_user_dir", _delete_all_from_user_dir_txn
         )
@@ -574,7 +548,7 @@ class UserDirectoryStore(SQLBaseStore):
         return self._simple_select_one(
             table="user_directory",
             keyvalues={"user_id": user_id},
-            retcols=("room_id", "display_name", "avatar_url",),
+            retcols=("room_id", "display_name", "avatar_url"),
             allow_none=True,
             desc="get_user_in_directory",
         )
@@ -607,7 +581,9 @@ class UserDirectoryStore(SQLBaseStore):
 
     def get_current_state_deltas(self, prev_stream_id):
         prev_stream_id = int(prev_stream_id)
-        if not self._curr_state_delta_stream_cache.has_any_entity_changed(prev_stream_id):
+        if not self._curr_state_delta_stream_cache.has_any_entity_changed(
+            prev_stream_id
+        ):
             return []
 
         def get_current_state_deltas_txn(txn):
@@ -641,7 +617,7 @@ class UserDirectoryStore(SQLBaseStore):
                 WHERE ? < stream_id AND stream_id <= ?
                 ORDER BY stream_id ASC
             """
-            txn.execute(sql, (prev_stream_id, max_stream_id,))
+            txn.execute(sql, (prev_stream_id, max_stream_id))
             return self.cursor_to_dict(txn)
 
         return self.runInteraction(
@@ -731,8 +707,11 @@ class UserDirectoryStore(SQLBaseStore):
                     display_name IS NULL,
                     avatar_url IS NULL
                 LIMIT ?
-            """ % (join_clause, where_clause)
+            """ % (
-            args = join_args + (full_query, exact_query, prefix_query, limit + 1,)
+                join_clause,
+                where_clause,
+            )
+            args = join_args + (full_query, exact_query, prefix_query, limit + 1)
         elif isinstance(self.database_engine, Sqlite3Engine):
             search_query = _parse_query_sqlite(search_term)
 
@@ -749,7 +728,10 @@ class UserDirectoryStore(SQLBaseStore):
                     display_name IS NULL,
                     avatar_url IS NULL
                 LIMIT ?
-            """ % (join_clause, where_clause)
+            """ % (
+                join_clause,
+                where_clause,
+            )
             args = join_args + (search_query, limit + 1)
         else:
             # This should be unreachable.
@@ -761,10 +743,7 @@ class UserDirectoryStore(SQLBaseStore):
 
         limited = len(results) > limit
 
-        defer.returnValue({
+        defer.returnValue({"limited": limited, "results": results})
-            "limited": limited,
-            "results": results,
-        })
 
 
 def _parse_query_sqlite(search_term):
@@ -779,7 +758,7 @@ def _parse_query_sqlite(search_term):
 
     # Pull out the individual words, discarding any non-word characters.
     results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
-    return " & ".join("(%s* OR %s)" % (result, result,) for result in results)
+    return " & ".join("(%s* OR %s)" % (result, result) for result in results)
 
 
 def _parse_query_postgres(search_term):
@@ -792,7 +771,7 @@ def _parse_query_postgres(search_term):
     # Pull out the individual words, discarding any non-word characters.
     results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
 
-    both = " & ".join("(%s:* | %s)" % (result, result,) for result in results)
+    both = " & ".join("(%s:* | %s)" % (result, result) for result in results)
     exact = " & ".join("%s" % (result,) for result in results)
     prefix = " & ".join("%s:*" % (result,) for result in results)
 

tests/config/test_room_directory.py

@@ -36,6 +36,8 @@ class RoomDirectoryConfigTestCase(unittest.TestCase):
             - user_id: "@gah:example.com"
               alias: "#goo:example.com"
               action: "allow"
+
+        room_list_publication_rules: []
         """)
 
         rd_config = RoomDirectoryConfig()
@@ -43,25 +45,102 @@ class RoomDirectoryConfigTestCase(unittest.TestCase):
 
         self.assertFalse(rd_config.is_alias_creation_allowed(
             user_id="@bob:example.com",
+            room_id="!test",
             alias="#test:example.com",
         ))
 
         self.assertTrue(rd_config.is_alias_creation_allowed(
             user_id="@test:example.com",
+            room_id="!test",
             alias="#unofficial_st:example.com",
         ))
 
         self.assertTrue(rd_config.is_alias_creation_allowed(
             user_id="@foobar:example.com",
+            room_id="!test",
             alias="#test:example.com",
         ))
 
         self.assertTrue(rd_config.is_alias_creation_allowed(
             user_id="@gah:example.com",
+            room_id="!test",
             alias="#goo:example.com",
         ))
 
         self.assertFalse(rd_config.is_alias_creation_allowed(
             user_id="@test:example.com",
+            room_id="!test",
             alias="#test:example.com",
         ))
+
+    def test_room_publish_acl(self):
+        config = yaml.load("""
+        alias_creation_rules: []
+
+        room_list_publication_rules:
+            - user_id: "*bob*"
+              alias: "*"
+              action: "deny"
+            - user_id: "*"
+              alias: "#unofficial_*"
+              action: "allow"
+            - user_id: "@foo*:example.com"
+              alias: "*"
+              action: "allow"
+            - user_id: "@gah:example.com"
+              alias: "#goo:example.com"
+              action: "allow"
+            - room_id: "!test-deny"
+              action: "deny"
+        """)
+
+        rd_config = RoomDirectoryConfig()
+        rd_config.read_config(config)
+
+        self.assertFalse(rd_config.is_publishing_room_allowed(
+            user_id="@bob:example.com",
+            room_id="!test",
+            aliases=["#test:example.com"],
+        ))
+
+        self.assertTrue(rd_config.is_publishing_room_allowed(
+            user_id="@test:example.com",
+            room_id="!test",
+            aliases=["#unofficial_st:example.com"],
+        ))
+
+        self.assertTrue(rd_config.is_publishing_room_allowed(
+            user_id="@foobar:example.com",
+            room_id="!test",
+            aliases=[],
+        ))
+
+        self.assertTrue(rd_config.is_publishing_room_allowed(
+            user_id="@gah:example.com",
+            room_id="!test",
+            aliases=["#goo:example.com"],
+        ))
+
+        self.assertFalse(rd_config.is_publishing_room_allowed(
+            user_id="@test:example.com",
+            room_id="!test",
+            aliases=["#test:example.com"],
+        ))
+
+        self.assertTrue(rd_config.is_publishing_room_allowed(
+            user_id="@foobar:example.com",
+            room_id="!test-deny",
+            aliases=[],
+        ))
+
+        self.assertFalse(rd_config.is_publishing_room_allowed(
+            user_id="@gah:example.com",
+            room_id="!test-deny",
+            aliases=[],
+        ))
+
+        self.assertTrue(rd_config.is_publishing_room_allowed(
+            user_id="@test:example.com",
+            room_id="!test",
+            aliases=["#unofficial_st:example.com", "#blah:example.com"],
+        ))

tests/handlers/test_directory.py

@@ -121,6 +121,7 @@ class TestCreateAliasACL(unittest.HomeserverTestCase):
                 "action": "allow",
             }
         ]
+        config["room_list_publication_rules"] = []
 
         rd_config = RoomDirectoryConfig()
         rd_config.read_config(config)