From 317248d42cb05ffa39119d6fefb7da286cb46225 Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Thu, 26 May 2022 16:07:27 +0100
Subject: [PATCH 1/4] Improve URL previews by not including the content of
 media tags in the generated description. (#12887)

---
 changelog.d/12887.misc                |  1 +
 synapse/rest/media/v1/preview_html.py | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/12887.misc

diff --git a/changelog.d/12887.misc b/changelog.d/12887.misc
new file mode 100644
index 0000000000..7f6f731832
--- /dev/null
+++ b/changelog.d/12887.misc
@@ -0,0 +1 @@
+Improve URL previews by not including the content of media tags in the generated description.
\ No newline at end of file
diff --git a/synapse/rest/media/v1/preview_html.py b/synapse/rest/media/v1/preview_html.py
index ca73965fc2..0358c68a64 100644
--- a/synapse/rest/media/v1/preview_html.py
+++ b/synapse/rest/media/v1/preview_html.py
@@ -246,7 +246,9 @@ def parse_html_description(tree: "etree.Element") -> Optional[str]:
 
     Grabs any text nodes which are inside the <body/> tag, unless they are within
     an HTML5 semantic markup tag (<header/>, <nav/>, <aside/>, <footer/>), or
-    if they are within a <script/> or <style/> tag.
+    if they are within a <script/>, <svg/> or <style/> tag, or if they are within
+    a tag whose content is usually only shown to old browsers
+    (<iframe/>, <video/>, <canvas/>, <picture/>).
 
     This is a very very very coarse approximation to a plain text render of the page.
 
@@ -268,6 +270,12 @@ def parse_html_description(tree: "etree.Element") -> Optional[str]:
         "script",
         "noscript",
         "style",
+        "svg",
+        "iframe",
+        "video",
+        "canvas",
+        "img",
+        "picture",
         etree.Comment,
     )
 

From 7b88f5a107ce9751365f9f2393521ef3d62afde8 Mon Sep 17 00:00:00 2001
From: reivilibre <oliverw@matrix.org>
Date: Fri, 27 May 2022 10:44:51 +0100
Subject: [PATCH 2/4] Add an option allowing users to use their password to
 reauthenticate even though password authentication is disabled. (#12883)

---
 changelog.d/12883.feature                     |  1 +
 docs/sample_config.yaml                       |  4 +-
 .../configuration/config_documentation.md     |  3 ++
 synapse/config/auth.py                        | 17 +++++++-
 synapse/handlers/auth.py                      | 29 +++++++++----
 tests/rest/client/test_auth.py                | 41 +++++++++++++++++++
 6 files changed, 83 insertions(+), 12 deletions(-)
 create mode 100644 changelog.d/12883.feature

diff --git a/changelog.d/12883.feature b/changelog.d/12883.feature
new file mode 100644
index 0000000000..83626771c0
--- /dev/null
+++ b/changelog.d/12883.feature
@@ -0,0 +1 @@
+Add an option allowing users to use their password to reauthenticate for privileged actions even though password login is disabled.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index ee98d193cb..e0abcd3b03 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -2216,7 +2216,9 @@ sso:
 
 
 password_config:
-   # Uncomment to disable password login
+   # Uncomment to disable password login.
+   # Set to `only_for_reauth` to permit reauthentication for users that
+   # have passwords and are already logged in.
    #
    #enabled: false
 
diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md
index 0f5bda32b9..295cece7e8 100644
--- a/docs/usage/configuration/config_documentation.md
+++ b/docs/usage/configuration/config_documentation.md
@@ -2930,6 +2930,9 @@ Use this setting to enable password-based logins.
 
 This setting has the following sub-options:
 * `enabled`: Defaults to true.
+   Set to false to disable password authentication.
+   Set to `only_for_reauth` to allow users with existing passwords to use them
+   to log in and reauthenticate, whilst preventing new users from setting passwords.
 * `localdb_enabled`: Set to false to disable authentication against the local password
    database. This is ignored if `enabled` is false, and is only useful
    if you have other `password_providers`. Defaults to true. 
diff --git a/synapse/config/auth.py b/synapse/config/auth.py
index bb417a2359..265a554a5d 100644
--- a/synapse/config/auth.py
+++ b/synapse/config/auth.py
@@ -29,7 +29,18 @@ class AuthConfig(Config):
         if password_config is None:
             password_config = {}
 
-        self.password_enabled = password_config.get("enabled", True)
+        passwords_enabled = password_config.get("enabled", True)
+        # 'only_for_reauth' allows users who have previously set a password to use it,
+        # even though passwords would otherwise be disabled.
+        passwords_for_reauth_only = passwords_enabled == "only_for_reauth"
+
+        self.password_enabled_for_login = (
+            passwords_enabled and not passwords_for_reauth_only
+        )
+        self.password_enabled_for_reauth = (
+            passwords_for_reauth_only or passwords_enabled
+        )
+
         self.password_localdb_enabled = password_config.get("localdb_enabled", True)
         self.password_pepper = password_config.get("pepper", "")
 
@@ -46,7 +57,9 @@ class AuthConfig(Config):
     def generate_config_section(self, **kwargs: Any) -> str:
         return """\
         password_config:
-           # Uncomment to disable password login
+           # Uncomment to disable password login.
+           # Set to `only_for_reauth` to permit reauthentication for users that
+           # have passwords and are already logged in.
            #
            #enabled: false
 
diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py
index 1b9050ea96..fbafbbee6b 100644
--- a/synapse/handlers/auth.py
+++ b/synapse/handlers/auth.py
@@ -210,7 +210,8 @@ class AuthHandler:
 
         self.hs = hs  # FIXME better possibility to access registrationHandler later?
         self.macaroon_gen = hs.get_macaroon_generator()
-        self._password_enabled = hs.config.auth.password_enabled
+        self._password_enabled_for_login = hs.config.auth.password_enabled_for_login
+        self._password_enabled_for_reauth = hs.config.auth.password_enabled_for_reauth
         self._password_localdb_enabled = hs.config.auth.password_localdb_enabled
         self._third_party_rules = hs.get_third_party_event_rules()
 
@@ -387,13 +388,13 @@ class AuthHandler:
         return params, session_id
 
     async def _get_available_ui_auth_types(self, user: UserID) -> Iterable[str]:
-        """Get a list of the authentication types this user can use"""
+        """Get a list of the user-interactive authentication types this user can use."""
 
         ui_auth_types = set()
 
         # if the HS supports password auth, and the user has a non-null password, we
         # support password auth
-        if self._password_localdb_enabled and self._password_enabled:
+        if self._password_localdb_enabled and self._password_enabled_for_reauth:
             lookupres = await self._find_user_id_and_pwd_hash(user.to_string())
             if lookupres:
                 _, password_hash = lookupres
@@ -402,7 +403,7 @@ class AuthHandler:
 
         # also allow auth from password providers
         for t in self.password_auth_provider.get_supported_login_types().keys():
-            if t == LoginType.PASSWORD and not self._password_enabled:
+            if t == LoginType.PASSWORD and not self._password_enabled_for_reauth:
                 continue
             ui_auth_types.add(t)
 
@@ -710,7 +711,7 @@ class AuthHandler:
             return res
 
         # fall back to the v1 login flow
-        canonical_id, _ = await self.validate_login(authdict)
+        canonical_id, _ = await self.validate_login(authdict, is_reauth=True)
         return canonical_id
 
     def _get_params_recaptcha(self) -> dict:
@@ -1064,7 +1065,7 @@ class AuthHandler:
         Returns:
             Whether users on this server are allowed to change or set a password
         """
-        return self._password_enabled and self._password_localdb_enabled
+        return self._password_enabled_for_login and self._password_localdb_enabled
 
     def get_supported_login_types(self) -> Iterable[str]:
         """Get a the login types supported for the /login API
@@ -1089,9 +1090,9 @@ class AuthHandler:
         # that comes first, where it's present.
         if LoginType.PASSWORD in types:
             types.remove(LoginType.PASSWORD)
-            if self._password_enabled:
+            if self._password_enabled_for_login:
                 types.insert(0, LoginType.PASSWORD)
-        elif self._password_localdb_enabled and self._password_enabled:
+        elif self._password_localdb_enabled and self._password_enabled_for_login:
             types.insert(0, LoginType.PASSWORD)
 
         return types
@@ -1100,6 +1101,7 @@ class AuthHandler:
         self,
         login_submission: Dict[str, Any],
         ratelimit: bool = False,
+        is_reauth: bool = False,
     ) -> Tuple[str, Optional[Callable[["LoginResponse"], Awaitable[None]]]]:
         """Authenticates the user for the /login API
 
@@ -1110,6 +1112,9 @@ class AuthHandler:
             login_submission: the whole of the login submission
                 (including 'type' and other relevant fields)
             ratelimit: whether to apply the failed_login_attempt ratelimiter
+            is_reauth: whether this is part of a User-Interactive Authorisation
+                flow to reauthenticate for a privileged action (rather than a
+                new login)
         Returns:
             A tuple of the canonical user id, and optional callback
                 to be called once the access token and device id are issued
@@ -1132,8 +1137,14 @@ class AuthHandler:
         # special case to check for "password" for the check_password interface
         # for the auth providers
         password = login_submission.get("password")
+
         if login_type == LoginType.PASSWORD:
-            if not self._password_enabled:
+            if is_reauth:
+                passwords_allowed_here = self._password_enabled_for_reauth
+            else:
+                passwords_allowed_here = self._password_enabled_for_login
+
+            if not passwords_allowed_here:
                 raise SynapseError(400, "Password login has been disabled.")
             if not isinstance(password, str):
                 raise SynapseError(400, "Bad parameter: password", Codes.INVALID_PARAM)
diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py
index 9653f45837..05355c7fb6 100644
--- a/tests/rest/client/test_auth.py
+++ b/tests/rest/client/test_auth.py
@@ -195,8 +195,17 @@ class UIAuthTests(unittest.HomeserverTestCase):
         self.user_pass = "pass"
         self.user = self.register_user("test", self.user_pass)
         self.device_id = "dev1"
+
+        # Force-enable password login for just long enough to log in.
+        auth_handler = self.hs.get_auth_handler()
+        allow_auth_for_login = auth_handler._password_enabled_for_login
+        auth_handler._password_enabled_for_login = True
+
         self.user_tok = self.login("test", self.user_pass, self.device_id)
 
+        # Restore password login to however it was.
+        auth_handler._password_enabled_for_login = allow_auth_for_login
+
     def delete_device(
         self,
         access_token: str,
@@ -263,6 +272,38 @@ class UIAuthTests(unittest.HomeserverTestCase):
             },
         )
 
+    @override_config({"password_config": {"enabled": "only_for_reauth"}})
+    def test_ui_auth_with_passwords_for_reauth_only(self) -> None:
+        """
+        Test user interactive authentication outside of registration.
+        """
+
+        # Attempt to delete this device.
+        # Returns a 401 as per the spec
+        channel = self.delete_device(
+            self.user_tok, self.device_id, HTTPStatus.UNAUTHORIZED
+        )
+
+        # Grab the session
+        session = channel.json_body["session"]
+        # Ensure that flows are what is expected.
+        self.assertIn({"stages": ["m.login.password"]}, channel.json_body["flows"])
+
+        # Make another request providing the UI auth flow.
+        self.delete_device(
+            self.user_tok,
+            self.device_id,
+            HTTPStatus.OK,
+            {
+                "auth": {
+                    "type": "m.login.password",
+                    "identifier": {"type": "m.id.user", "user": self.user},
+                    "password": self.user_pass,
+                    "session": session,
+                },
+            },
+        )
+
     def test_grandfathered_identifier(self) -> None:
         """Check behaviour without "identifier" dict
 

From bb7a6377652755584c327d28b131c467d999280d Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Fri, 27 May 2022 11:03:05 +0100
Subject: [PATCH 3/4] Close `ijson` coroutines ourselves instead of letting the
 GC close them (#12875)

Hopefully this means that exceptions raised due to truncated JSON
get a sensible logging context and stack.

Signed-off-by: Sean Quah <seanq@matrix.org>
---
 changelog.d/12875.bugfix               |  1 +
 synapse/federation/transport/client.py |  9 +++++++--
 synapse/http/matrixfederationclient.py | 11 +++++++++++
 3 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/12875.bugfix

diff --git a/changelog.d/12875.bugfix b/changelog.d/12875.bugfix
new file mode 100644
index 0000000000..7b011e0f90
--- /dev/null
+++ b/changelog.d/12875.bugfix
@@ -0,0 +1 @@
+Explicitly close `ijson` coroutines once we are done with them, instead of leaving the garbage collector to close them.
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 9ce06dfa28..2686ee2e51 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -1363,7 +1363,7 @@ class SendJoinParser(ByteParser[SendJoinResponse]):
     def __init__(self, room_version: RoomVersion, v1_api: bool):
         self._response = SendJoinResponse([], [], event_dict={})
         self._room_version = room_version
-        self._coros = []
+        self._coros: List[Generator[None, bytes, None]] = []
 
         # The V1 API has the shape of `[200, {...}]`, which we handle by
         # prefixing with `item.*`.
@@ -1411,6 +1411,9 @@ class SendJoinParser(ByteParser[SendJoinResponse]):
         return len(data)
 
     def finish(self) -> SendJoinResponse:
+        for c in self._coros:
+            c.close()
+
         if self._response.event_dict:
             self._response.event = make_event_from_dict(
                 self._response.event_dict, self._room_version
@@ -1430,7 +1433,7 @@ class _StateParser(ByteParser[StateRequestResponse]):
     def __init__(self, room_version: RoomVersion):
         self._response = StateRequestResponse([], [])
         self._room_version = room_version
-        self._coros = [
+        self._coros: List[Generator[None, bytes, None]] = [
             ijson.items_coro(
                 _event_list_parser(room_version, self._response.state),
                 "pdus.item",
@@ -1449,4 +1452,6 @@ class _StateParser(ByteParser[StateRequestResponse]):
         return len(data)
 
     def finish(self) -> StateRequestResponse:
+        for c in self._coros:
+            c.close()
         return self._response
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 0b9475debd..901c47f756 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -225,6 +225,7 @@ async def _handle_response(
     if max_response_size is None:
         max_response_size = MAX_RESPONSE_SIZE
 
+    finished = False
     try:
         check_content_type_is(response.headers, parser.CONTENT_TYPE)
 
@@ -233,6 +234,7 @@ async def _handle_response(
 
         length = await make_deferred_yieldable(d)
 
+        finished = True
         value = parser.finish()
     except BodyExceededMaxSize as e:
         # The response was too big.
@@ -283,6 +285,15 @@ async def _handle_response(
             e,
         )
         raise
+    finally:
+        if not finished:
+            # There was an exception and we didn't `finish()` the parse.
+            # Let the parser know that it can free up any resources.
+            try:
+                parser.finish()
+            except Exception:
+                # Ignore any additional exceptions.
+                pass
 
     time_taken_secs = reactor.seconds() - start_ms / 1000
 

From e409ab8e9209cb599d7e9a4b3f8bfbd59eb839d3 Mon Sep 17 00:00:00 2001
From: Sean Quah <seanq@matrix.org>
Date: Fri, 27 May 2022 11:06:45 +0100
Subject: [PATCH 4/4] 1.60.0rc2

---
 CHANGES.md                | 23 ++++++++++++++++++++++-
 changelog.d/12875.bugfix  |  1 -
 changelog.d/12883.feature |  1 -
 changelog.d/12887.misc    |  1 -
 debian/changelog          |  6 ++++++
 pyproject.toml            |  2 +-
 6 files changed, 29 insertions(+), 5 deletions(-)
 delete mode 100644 changelog.d/12875.bugfix
 delete mode 100644 changelog.d/12883.feature
 delete mode 100644 changelog.d/12887.misc

diff --git a/CHANGES.md b/CHANGES.md
index 46ac3fce7a..40e362e920 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,4 +1,4 @@
-Synapse 1.60.0rc1 (2022-05-24)
+Synapse 1.60.0rc2 (2022-05-27)
 ==============================
 
 This release of Synapse adds a unique index to the `state_group_edges` table, in
@@ -17,6 +17,27 @@ for more details.
 Features
 --------
 
+- Add an option allowing users to use their password to reauthenticate for privileged actions even though password login is disabled. ([\#12883](https://github.com/matrix-org/synapse/issues/12883))
+
+
+Bugfixes
+--------
+
+- Explicitly close `ijson` coroutines once we are done with them, instead of leaving the garbage collector to close them. ([\#12875](https://github.com/matrix-org/synapse/issues/12875))
+
+
+Internal Changes
+----------------
+
+- Improve URL previews by not including the content of media tags in the generated description. ([\#12887](https://github.com/matrix-org/synapse/issues/12887))
+
+
+Synapse 1.60.0rc1 (2022-05-24)
+==============================
+
+Features
+--------
+
 - Measure the time taken in spam-checking callbacks and expose those measurements as metrics. ([\#12513](https://github.com/matrix-org/synapse/issues/12513))
 - Add a `default_power_level_content_override` config option to set default room power levels per room preset. ([\#12618](https://github.com/matrix-org/synapse/issues/12618))
 - Add support for [MSC3787: Allowing knocks to restricted rooms](https://github.com/matrix-org/matrix-spec-proposals/pull/3787). ([\#12623](https://github.com/matrix-org/synapse/issues/12623))
diff --git a/changelog.d/12875.bugfix b/changelog.d/12875.bugfix
deleted file mode 100644
index 7b011e0f90..0000000000
--- a/changelog.d/12875.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Explicitly close `ijson` coroutines once we are done with them, instead of leaving the garbage collector to close them.
diff --git a/changelog.d/12883.feature b/changelog.d/12883.feature
deleted file mode 100644
index 83626771c0..0000000000
--- a/changelog.d/12883.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add an option allowing users to use their password to reauthenticate for privileged actions even though password login is disabled.
diff --git a/changelog.d/12887.misc b/changelog.d/12887.misc
deleted file mode 100644
index 7f6f731832..0000000000
--- a/changelog.d/12887.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve URL previews by not including the content of media tags in the generated description.
\ No newline at end of file
diff --git a/debian/changelog b/debian/changelog
index 6eba9b3a1b..b6a51d6903 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.60.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.60.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 27 May 2022 11:04:55 +0100
+
 matrix-synapse-py3 (1.60.0~rc1) stable; urgency=medium
 
   * New Synapse release 1.60.0rc1.
diff --git a/pyproject.toml b/pyproject.toml
index 9359d211f7..59cff590b5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,7 +54,7 @@ skip_gitignore = true
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.60.0rc1"
+version = "1.60.0rc2"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"