Support "identifier" dicts in UIA (#8848)

The spec requires synapse to support `identifier` dicts for `m.login.password`
user-interactive auth, which it did not (instead, it required an undocumented
`user` parameter.)

To fix this properly, we need to pull the code that interprets `identifier`
into `AuthHandler.validate_login` so that it can be called from the UIA code.

Fixes #5665.

changelog.d/8804.feature

@@ -0,0 +1 @@
+Allow Date header through CORS. Contributed by Nicolas Chamo.

changelog.d/8835.bugfix

@@ -0,0 +1 @@
+Fix minor long-standing bug in login, where we would offer the `password` login type if a custom auth provider supported it, even if password login was disabled.

changelog.d/8848.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug which caused Synapse to require unspecified parameters during user-interactive authentication.

changelog.d/8850.misc

@@ -0,0 +1 @@
+Add missing `ordering` to background database updates.

synapse/handlers/auth.py

@@ -205,15 +205,23 @@ class AuthHandler(BaseHandler):
         # type in the list. (NB that the spec doesn't require us to do so and
         # clients which favour types that they don't understand over those that
         # they do are technically broken)
+
+        # start out by assuming PASSWORD is enabled; we will remove it later if not.
         login_types = []
-        if self._password_enabled:
+        if hs.config.password_localdb_enabled:
             login_types.append(LoginType.PASSWORD)
+
         for provider in self.password_providers:
             if hasattr(provider, "get_supported_login_types"):
                 for t in provider.get_supported_login_types().keys():
                     if t not in login_types:
                         login_types.append(t)
+
+        if not self._password_enabled:
+            login_types.remove(LoginType.PASSWORD)
+
         self._supported_login_types = login_types
+
         # Login types and UI Auth types have a heavy overlap, but are not
         # necessarily identical. Login types have SSO (and other login types)
         # added in the rest layer, see synapse.rest.client.v1.login.LoginRestServerlet.on_GET.
@@ -230,6 +238,13 @@ class AuthHandler(BaseHandler):
             burst_count=self.hs.config.rc_login_failed_attempts.burst_count,
         )
 
+        # Ratelimitier for failed /login attempts
+        self._failed_login_attempts_ratelimiter = Ratelimiter(
+            clock=hs.get_clock(),
+            rate_hz=self.hs.config.rc_login_failed_attempts.per_second,
+            burst_count=self.hs.config.rc_login_failed_attempts.burst_count,
+        )
+
         self._clock = self.hs.get_clock()
 
         # Expire old UI auth sessions after a period of time.
@@ -642,14 +657,8 @@ class AuthHandler(BaseHandler):
             res = await checker.check_auth(authdict, clientip=clientip)
             return res
 
-        # build a v1-login-style dict out of the authdict and fall back to the
-        # v1 code
-        user_id = authdict.get("user")
-
-        if user_id is None:
-            raise SynapseError(400, "", Codes.MISSING_PARAM)
-
-        (canonical_id, callback) = await self.validate_login(user_id, authdict)
+        # fall back to the v1 login flow
+        canonical_id, _ = await self.validate_login(authdict)
         return canonical_id
 
     def _get_params_recaptcha(self) -> dict:
@@ -824,15 +833,155 @@ class AuthHandler(BaseHandler):
         return self._supported_login_types
 
     async def validate_login(
-        self, username: str, login_submission: Dict[str, Any]
+        self, login_submission: Dict[str, Any], ratelimit: bool = False,
     ) -> Tuple[str, Optional[Callable[[Dict[str, str]], None]]]:
         """Authenticates the user for the /login API
 
-        Also used by the user-interactive auth flow to validate
-        m.login.password auth types.
+        Also used by the user-interactive auth flow to validate auth types which don't
+        have an explicit UIA handler, including m.password.auth.
 
         Args:
-            username: username supplied by the user
+            login_submission: the whole of the login submission
+                (including 'type' and other relevant fields)
+            ratelimit: whether to apply the failed_login_attempt ratelimiter
+        Returns:
+            A tuple of the canonical user id, and optional callback
+                to be called once the access token and device id are issued
+        Raises:
+            StoreError if there was a problem accessing the database
+            SynapseError if there was a problem with the request
+            LoginError if there was an authentication problem.
+        """
+        login_type = login_submission.get("type")
+
+        # ideally, we wouldn't be checking the identifier unless we know we have a login
+        # method which uses it (https://github.com/matrix-org/synapse/issues/8836)
+        #
+        # But the auth providers' check_auth interface requires a username, so in
+        # practice we can only support login methods which we can map to a username
+        # anyway.
+
+        # special case to check for "password" for the check_password interface
+        # for the auth providers
+        password = login_submission.get("password")
+        if login_type == LoginType.PASSWORD:
+            if not self._password_enabled:
+                raise SynapseError(400, "Password login has been disabled.")
+            if not isinstance(password, str):
+                raise SynapseError(400, "Bad parameter: password", Codes.INVALID_PARAM)
+
+        # map old-school login fields into new-school "identifier" fields.
+        identifier_dict = convert_client_dict_legacy_fields_to_identifier(
+            login_submission
+        )
+
+        # convert phone type identifiers to generic threepids
+        if identifier_dict["type"] == "m.id.phone":
+            identifier_dict = login_id_phone_to_thirdparty(identifier_dict)
+
+        # convert threepid identifiers to user IDs
+        if identifier_dict["type"] == "m.id.thirdparty":
+            address = identifier_dict.get("address")
+            medium = identifier_dict.get("medium")
+
+            if medium is None or address is None:
+                raise SynapseError(400, "Invalid thirdparty identifier")
+
+            # For emails, canonicalise the address.
+            # We store all email addresses canonicalised in the DB.
+            # (See add_threepid in synapse/handlers/auth.py)
+            if medium == "email":
+                try:
+                    address = canonicalise_email(address)
+                except ValueError as e:
+                    raise SynapseError(400, str(e))
+
+            # We also apply account rate limiting using the 3PID as a key, as
+            # otherwise using 3PID bypasses the ratelimiting based on user ID.
+            if ratelimit:
+                self._failed_login_attempts_ratelimiter.ratelimit(
+                    (medium, address), update=False
+                )
+
+            # Check for login providers that support 3pid login types
+            if login_type == LoginType.PASSWORD:
+                # we've already checked that there is a (valid) password field
+                assert isinstance(password, str)
+                (
+                    canonical_user_id,
+                    callback_3pid,
+                ) = await self.check_password_provider_3pid(medium, address, password)
+                if canonical_user_id:
+                    # Authentication through password provider and 3pid succeeded
+                    return canonical_user_id, callback_3pid
+
+            # No password providers were able to handle this 3pid
+            # Check local store
+            user_id = await self.hs.get_datastore().get_user_id_by_threepid(
+                medium, address
+            )
+            if not user_id:
+                logger.warning(
+                    "unknown 3pid identifier medium %s, address %r", medium, address
+                )
+                # We mark that we've failed to log in here, as
+                # `check_password_provider_3pid` might have returned `None` due
+                # to an incorrect password, rather than the account not
+                # existing.
+                #
+                # If it returned None but the 3PID was bound then we won't hit
+                # this code path, which is fine as then the per-user ratelimit
+                # will kick in below.
+                if ratelimit:
+                    self._failed_login_attempts_ratelimiter.can_do_action(
+                        (medium, address)
+                    )
+                raise LoginError(403, "", errcode=Codes.FORBIDDEN)
+
+            identifier_dict = {"type": "m.id.user", "user": user_id}
+
+        # by this point, the identifier should be an m.id.user: if it's anything
+        # else, we haven't understood it.
+        if identifier_dict["type"] != "m.id.user":
+            raise SynapseError(400, "Unknown login identifier type")
+
+        username = identifier_dict.get("user")
+        if not username:
+            raise SynapseError(400, "User identifier is missing 'user' key")
+
+        if username.startswith("@"):
+            qualified_user_id = username
+        else:
+            qualified_user_id = UserID(username, self.hs.hostname).to_string()
+
+        # Check if we've hit the failed ratelimit (but don't update it)
+        if ratelimit:
+            self._failed_login_attempts_ratelimiter.ratelimit(
+                qualified_user_id.lower(), update=False
+            )
+
+        try:
+            return await self._validate_userid_login(username, login_submission)
+        except LoginError:
+            # The user has failed to log in, so we need to update the rate
+            # limiter. Using `can_do_action` avoids us raising a ratelimit
+            # exception and masking the LoginError. The actual ratelimiting
+            # should have happened above.
+            if ratelimit:
+                self._failed_login_attempts_ratelimiter.can_do_action(
+                    qualified_user_id.lower()
+                )
+            raise
+
+    async def _validate_userid_login(
+        self, username: str, login_submission: Dict[str, Any],
+    ) -> Tuple[str, Optional[Callable[[Dict[str, str]], None]]]:
+        """Helper for validate_login
+
+        Handles login, once we've mapped 3pids onto userids
+
+        Args:
+            username: the username, from the identifier dict
             login_submission: the whole of the login submission
                 (including 'type' and other relevant fields)
         Returns:
@@ -843,7 +992,6 @@ class AuthHandler(BaseHandler):
             SynapseError if there was a problem with the request
             LoginError if there was an authentication problem.
         """
-
         if username.startswith("@"):
             qualified_user_id = username
         else:
@@ -852,20 +1000,13 @@ class AuthHandler(BaseHandler):
         login_type = login_submission.get("type")
         known_login_type = False
 
-        # special case to check for "password" for the check_password interface
-        # for the auth providers
-        password = login_submission.get("password")
-
-        if login_type == LoginType.PASSWORD:
-            if not self._password_enabled:
-                raise SynapseError(400, "Password login has been disabled.")
-            if not password:
-                raise SynapseError(400, "Missing parameter: password")
-
         for provider in self.password_providers:
             if hasattr(provider, "check_password") and login_type == LoginType.PASSWORD:
                 known_login_type = True
-                is_valid = await provider.check_password(qualified_user_id, password)
+                # we've already checked that there is a (valid) password field
+                is_valid = await provider.check_password(
+                    qualified_user_id, login_submission["password"]
+                )
                 if is_valid:
                     return qualified_user_id, None
 
@@ -906,8 +1047,12 @@ class AuthHandler(BaseHandler):
         if login_type == LoginType.PASSWORD and self.hs.config.password_localdb_enabled:
             known_login_type = True
 
+            # we've already checked that there is a (valid) password field
+            password = login_submission["password"]
+            assert isinstance(password, str)
+
             canonical_user_id = await self._check_local_password(
-                qualified_user_id, password  # type: ignore
+                qualified_user_id, password
             )
 
             if canonical_user_id:

synapse/http/server.py

@@ -674,7 +674,7 @@ def set_cors_headers(request: Request):
     )
     request.setHeader(
         b"Access-Control-Allow-Headers",
-        b"Origin, X-Requested-With, Content-Type, Accept, Authorization",
+        b"Origin, X-Requested-With, Content-Type, Accept, Authorization, Date",
     )
 
 

synapse/rest/client/v1/login.py

@@ -19,10 +19,6 @@ from typing import Awaitable, Callable, Dict, Optional
 from synapse.api.errors import Codes, LoginError, SynapseError
 from synapse.api.ratelimiting import Ratelimiter
 from synapse.appservice import ApplicationService
-from synapse.handlers.auth import (
-    convert_client_dict_legacy_fields_to_identifier,
-    login_id_phone_to_thirdparty,
-)
 from synapse.http.server import finish_request
 from synapse.http.servlet import (
     RestServlet,
@@ -33,7 +29,6 @@ from synapse.http.site import SynapseRequest
 from synapse.rest.client.v2_alpha._base import client_patterns
 from synapse.rest.well_known import WellKnownBuilder
 from synapse.types import JsonDict, UserID
-from synapse.util.threepids import canonicalise_email
 
 logger = logging.getLogger(__name__)
 
@@ -78,11 +73,6 @@ class LoginRestServlet(RestServlet):
             rate_hz=self.hs.config.rc_login_account.per_second,
             burst_count=self.hs.config.rc_login_account.burst_count,
         )
-        self._failed_attempts_ratelimiter = Ratelimiter(
-            clock=hs.get_clock(),
-            rate_hz=self.hs.config.rc_login_failed_attempts.per_second,
-            burst_count=self.hs.config.rc_login_failed_attempts.burst_count,
-        )
 
     def on_GET(self, request: SynapseRequest):
         flows = []
@@ -140,17 +130,6 @@ class LoginRestServlet(RestServlet):
             result["well_known"] = well_known_data
         return 200, result
 
-    def _get_qualified_user_id(self, identifier):
-        if identifier["type"] != "m.id.user":
-            raise SynapseError(400, "Unknown login identifier type")
-        if "user" not in identifier:
-            raise SynapseError(400, "User identifier is missing 'user' key")
-
-        if identifier["user"].startswith("@"):
-            return identifier["user"]
-        else:
-            return UserID(identifier["user"], self.hs.hostname).to_string()
-
     async def _do_appservice_login(
         self, login_submission: JsonDict, appservice: ApplicationService
     ):
@@ -201,91 +180,9 @@ class LoginRestServlet(RestServlet):
             login_submission.get("address"),
             login_submission.get("user"),
         )
-        identifier = convert_client_dict_legacy_fields_to_identifier(login_submission)
-
-        # convert phone type identifiers to generic threepids
-        if identifier["type"] == "m.id.phone":
-            identifier = login_id_phone_to_thirdparty(identifier)
-
-        # convert threepid identifiers to user IDs
-        if identifier["type"] == "m.id.thirdparty":
-            address = identifier.get("address")
-            medium = identifier.get("medium")
-
-            if medium is None or address is None:
-                raise SynapseError(400, "Invalid thirdparty identifier")
-
-            # For emails, canonicalise the address.
-            # We store all email addresses canonicalised in the DB.
-            # (See add_threepid in synapse/handlers/auth.py)
-            if medium == "email":
-                try:
-                    address = canonicalise_email(address)
-                except ValueError as e:
-                    raise SynapseError(400, str(e))
-
-            # We also apply account rate limiting using the 3PID as a key, as
-            # otherwise using 3PID bypasses the ratelimiting based on user ID.
-            self._failed_attempts_ratelimiter.ratelimit((medium, address), update=False)
-
-            # Check for login providers that support 3pid login types
-            (
-                canonical_user_id,
-                callback_3pid,
-            ) = await self.auth_handler.check_password_provider_3pid(
-                medium, address, login_submission["password"]
-            )
-            if canonical_user_id:
-                # Authentication through password provider and 3pid succeeded
-
-                result = await self._complete_login(
-                    canonical_user_id, login_submission, callback_3pid
-                )
-                return result
-
-            # No password providers were able to handle this 3pid
-            # Check local store
-            user_id = await self.hs.get_datastore().get_user_id_by_threepid(
-                medium, address
-            )
-            if not user_id:
-                logger.warning(
-                    "unknown 3pid identifier medium %s, address %r", medium, address
-                )
-                # We mark that we've failed to log in here, as
-                # `check_password_provider_3pid` might have returned `None` due
-                # to an incorrect password, rather than the account not
-                # existing.
-                #
-                # If it returned None but the 3PID was bound then we won't hit
-                # this code path, which is fine as then the per-user ratelimit
-                # will kick in below.
-                self._failed_attempts_ratelimiter.can_do_action((medium, address))
-                raise LoginError(403, "", errcode=Codes.FORBIDDEN)
-
-            identifier = {"type": "m.id.user", "user": user_id}
-
-        # by this point, the identifier should be an m.id.user: if it's anything
-        # else, we haven't understood it.
-        qualified_user_id = self._get_qualified_user_id(identifier)
-
-        # Check if we've hit the failed ratelimit (but don't update it)
-        self._failed_attempts_ratelimiter.ratelimit(
-            qualified_user_id.lower(), update=False
+        canonical_user_id, callback = await self.auth_handler.validate_login(
+            login_submission, ratelimit=True
         )
-
-        try:
-            canonical_user_id, callback = await self.auth_handler.validate_login(
-                identifier["user"], login_submission
-            )
-        except LoginError:
-            # The user has failed to log in, so we need to update the rate
-            # limiter. Using `can_do_action` avoids us raising a ratelimit
-            # exception and masking the LoginError. The actual ratelimiting
-            # should have happened above.
-            self._failed_attempts_ratelimiter.can_do_action(qualified_user_id.lower())
-            raise
-
         result = await self._complete_login(
             canonical_user_id, login_submission, callback
         )

synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres

@@ -20,14 +20,14 @@
  */
 
 -- add new index that includes method to local media
-INSERT INTO background_updates (update_name, progress_json) VALUES
-  ('local_media_repository_thumbnails_method_idx', '{}');
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (5807, 'local_media_repository_thumbnails_method_idx', '{}');
 
 -- add new index that includes method to remote media
-INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
-  ('remote_media_repository_thumbnails_method_idx', '{}', 'local_media_repository_thumbnails_method_idx');
+INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
+  (5807, 'remote_media_repository_thumbnails_method_idx', '{}', 'local_media_repository_thumbnails_method_idx');
 
 -- drop old index
-INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES
-  ('media_repository_drop_index_wo_method', '{}', 'remote_media_repository_thumbnails_method_idx');
+INSERT INTO background_updates (ordering, update_name, progress_json, depends_on) VALUES
+  (5807, 'media_repository_drop_index_wo_method', '{}', 'remote_media_repository_thumbnails_method_idx');
 

synapse/storage/databases/main/schema/delta/58/12room_stats.sql

@@ -28,5 +28,5 @@
 -- functionality as the old one. This effectively restarts the background job
 -- from the beginning, without running it twice in a row, supporting both
 -- upgrade usecases.
-INSERT INTO background_updates (update_name, progress_json) VALUES
-    ('populate_stats_process_rooms_2', '{}');
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+    (5812, 'populate_stats_process_rooms_2', '{}');

synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql

@@ -1,2 +1,2 @@
-INSERT INTO background_updates (update_name, progress_json) VALUES
-  ('users_have_local_media', '{}');
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (5822, 'users_have_local_media', '{}');

synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql

@@ -13,5 +13,5 @@
  * limitations under the License.
  */
 
-INSERT INTO background_updates (update_name, progress_json) VALUES
-  ('e2e_cross_signing_keys_idx', '{}');
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (5823, 'e2e_cross_signing_keys_idx', '{}');

tests/handlers/test_password_providers.py

@@ -70,6 +70,24 @@ class CustomAuthProvider:
         return mock_password_provider.check_auth(*args)
 
 
+class PasswordCustomAuthProvider:
+    """A password_provider which implements password login via `check_auth`, as well
+    as a custom type."""
+
+    @staticmethod
+    def parse_config(self):
+        pass
+
+    def __init__(self, config, account_handler):
+        pass
+
+    def get_supported_login_types(self):
+        return {"m.login.password": ["password"], "test.login_type": ["test_field"]}
+
+    def check_auth(self, *args):
+        return mock_password_provider.check_auth(*args)
+
+
 def providers_config(*providers: Type[Any]) -> dict:
     """Returns a config dict that will enable the given password auth providers"""
     return {
@@ -246,7 +264,11 @@ class PasswordAuthProviderTests(unittest.HomeserverTestCase):
         mock_password_provider.check_password.reset_mock()
 
         # first delete should give a 401
-        session = self._start_delete_device_session(tok1, "dev2")
+        channel = self._delete_device(tok1, "dev2")
+        self.assertEqual(channel.code, 401)
+        # there are no valid flows here!
+        self.assertEqual(channel.json_body["flows"], [])
+        session = channel.json_body["session"]
         mock_password_provider.check_password.assert_not_called()
 
         # now try deleting with the local password
@@ -336,9 +358,6 @@ class PasswordAuthProviderTests(unittest.HomeserverTestCase):
             "auth": {
                 "type": "test.login_type",
                 "identifier": {"type": "m.id.user", "user": "localuser"},
-                # FIXME "identifier" is ignored
-                #   https://github.com/matrix-org/synapse/issues/5665
-                "user": "localuser",
                 "session": session,
             },
         }
@@ -410,6 +429,85 @@ class PasswordAuthProviderTests(unittest.HomeserverTestCase):
         self.assertEqual(channel.code, 400, channel.result)
         mock_password_provider.check_auth.assert_not_called()
 
+    @override_config(
+        {
+            **providers_config(PasswordCustomAuthProvider),
+            "password_config": {"enabled": False},
+        }
+    )
+    def test_password_custom_auth_password_disabled_login(self):
+        """log in with a custom auth provider which implements password, but password
+        login is disabled"""
+        self.register_user("localuser", "localpass")
+
+        flows = self._get_login_flows()
+        self.assertEqual(flows, [{"type": "test.login_type"}] + ADDITIONAL_LOGIN_FLOWS)
+
+        # login shouldn't work and should be rejected with a 400 ("unknown login type")
+        channel = self._send_password_login("localuser", "localpass")
+        self.assertEqual(channel.code, 400, channel.result)
+        mock_password_provider.check_auth.assert_not_called()
+
+    @override_config(
+        {
+            **providers_config(PasswordCustomAuthProvider),
+            "password_config": {"enabled": False},
+        }
+    )
+    def test_password_custom_auth_password_disabled_ui_auth(self):
+        """UI Auth with a custom auth provider which implements password, but password
+        login is disabled"""
+        # register the user and log in twice via the test login type to get two devices,
+        self.register_user("localuser", "localpass")
+        mock_password_provider.check_auth.return_value = defer.succeed(
+            "@localuser:test"
+        )
+        channel = self._send_login("test.login_type", "localuser", test_field="")
+        self.assertEqual(channel.code, 200, channel.result)
+        tok1 = channel.json_body["access_token"]
+
+        channel = self._send_login(
+            "test.login_type", "localuser", test_field="", device_id="dev2"
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+
+        # make the initial request which returns a 401
+        channel = self._delete_device(tok1, "dev2")
+        self.assertEqual(channel.code, 401)
+        # Ensure that flows are what is expected. In particular, "password" should *not*
+        # be present.
+        self.assertIn({"stages": ["test.login_type"]}, channel.json_body["flows"])
+        session = channel.json_body["session"]
+
+        mock_password_provider.reset_mock()
+
+        # check that auth with password is rejected
+        body = {
+            "auth": {
+                "type": "m.login.password",
+                "identifier": {"type": "m.id.user", "user": "localuser"},
+                "password": "localpass",
+                "session": session,
+            },
+        }
+
+        channel = self._delete_device(tok1, "dev2", body)
+        self.assertEqual(channel.code, 400)
+        self.assertEqual(
+            "Password login has been disabled.", channel.json_body["error"]
+        )
+        mock_password_provider.check_auth.assert_not_called()
+        mock_password_provider.reset_mock()
+
+        # successful auth
+        body["auth"]["type"] = "test.login_type"
+        body["auth"]["test_field"] = "x"
+        channel = self._delete_device(tok1, "dev2", body)
+        self.assertEqual(channel.code, 200)
+        mock_password_provider.check_auth.assert_called_once_with(
+            "localuser", "test.login_type", {"test_field": "x"}
+        )
+
     @override_config(
         {
             **providers_config(CustomAuthProvider),
@@ -428,8 +526,6 @@ class PasswordAuthProviderTests(unittest.HomeserverTestCase):
         channel = self._send_password_login("localuser", "localpass")
         self.assertEqual(channel.code, 400, channel.result)
 
-    test_custom_auth_no_local_user_fallback.skip = "currently broken"
-
     def _get_login_flows(self) -> JsonDict:
         _, channel = self.make_request("GET", "/_matrix/client/r0/login")
         self.assertEqual(channel.code, 200, channel.result)
@@ -439,7 +535,7 @@ class PasswordAuthProviderTests(unittest.HomeserverTestCase):
         return self._send_login(type="m.login.password", user=user, password=password)
 
     def _send_login(self, type<