From 69147ed15897f66838ed4224048dda606cb529e2 Mon Sep 17 00:00:00 2001 From: chagai95 <31655082+chagai95@users.noreply.github.com> Date: Fri, 13 Nov 2020 13:07:09 +0100 Subject: [PATCH 01/57] Updating README.rst (#8746) Minor corrections and advice... Should help beginners. --- README.rst | 16 ++++++++++------ changelog.d/8746.doc | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 changelog.d/8746.doc diff --git a/README.rst b/README.rst index 59d5a4389b..d724cf97da 100644 --- a/README.rst +++ b/README.rst @@ -261,18 +261,22 @@ to install using pip and a virtualenv:: pip install -e ".[all,test]" This will run a process of downloading and installing all the needed -dependencies into a virtual env. +dependencies into a virtual env. If any dependencies fail to install, +try installing the failing modules individually:: -Once this is done, you may wish to run Synapse's unit tests, to -check that everything is installed as it should be:: + pip install -e "module-name" + +Once this is done, you may wish to run Synapse's unit tests to +check that everything is installed correctly:: python -m twisted.trial tests -This should end with a 'PASSED' result:: +This should end with a 'PASSED' result (note that exact numbers will +differ):: - Ran 1266 tests in 643.930s + Ran 1337 tests in 716.064s - PASSED (skips=15, successes=1251) + PASSED (skips=15, successes=1322) Running the Integration Tests ============================= diff --git a/changelog.d/8746.doc b/changelog.d/8746.doc new file mode 100644 index 0000000000..6baf58ba7a --- /dev/null +++ b/changelog.d/8746.doc @@ -0,0 +1 @@ +Add some helpful hints to the README for new Synapse developers. Contributed by @chagai95. \ No newline at end of file From 68fc0dcb5a3612c869259573f50bee5d388a8a66 Mon Sep 17 00:00:00 2001 From: Marcus Schopen Date: Fri, 13 Nov 2020 13:07:50 +0100 Subject: [PATCH 02/57] SAML: add element examples (#8718) add some mdui:UIInfo element examples for saml2_config in homeserver.yaml --- changelog.d/8718.misc | 1 + docs/sample_config.yaml | 22 ++++++++++++++++++++++ synapse/config/saml2_config.py | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 changelog.d/8718.misc diff --git a/changelog.d/8718.misc b/changelog.d/8718.misc new file mode 100644 index 0000000000..f1868ce709 --- /dev/null +++ b/changelog.d/8718.misc @@ -0,0 +1 @@ +Add some `mdui:UIInfo` element examples for `saml2_config` in the homeserver config. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 7e2cf97c3e..c0cd009230 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1560,6 +1560,28 @@ saml2_config: #description: ["My awesome SP", "en"] #name: ["Test SP", "en"] + #ui_info: + # display_name: + # - lang: en + # text: "Display Name is the descriptive name of your service." + # description: + # - lang: en + # text: "Description should be a short paragraph explaining the purpose of the service." + # information_url: + # - lang: en + # text: "https://example.com/terms-of-service" + # privacy_statement_url: + # - lang: en + # text: "https://example.com/privacy-policy" + # keywords: + # - lang: en + # text: ["Matrix", "Element"] + # logo: + # - lang: en + # text: "https://example.com/logo.svg" + # width: "200" + # height: "80" + #organization: # name: Example com # display_name: diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py index 778750f43b..2ff7dfb311 100644 --- a/synapse/config/saml2_config.py +++ b/synapse/config/saml2_config.py @@ -271,6 +271,28 @@ class SAML2Config(Config): #description: ["My awesome SP", "en"] #name: ["Test SP", "en"] + #ui_info: + # display_name: + # - lang: en + # text: "Display Name is the descriptive name of your service." + # description: + # - lang: en + # text: "Description should be a short paragraph explaining the purpose of the service." + # information_url: + # - lang: en + # text: "https://example.com/terms-of-service" + # privacy_statement_url: + # - lang: en + # text: "https://example.com/privacy-policy" + # keywords: + # - lang: en + # text: ["Matrix", "Element"] + # logo: + # - lang: en + # text: "https://example.com/logo.svg" + # width: "200" + # height: "80" + #organization: # name: Example com # display_name: From 023f791143fbce28386011e8ce667389476588d6 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Fri, 13 Nov 2020 14:57:55 +0100 Subject: [PATCH 03/57] Migrate documentation `docs/admin_api/event_reports` to markdown (#8742) Related to #8714. `event_reports.rst` was introduced in Synapse 1.21.0. --- changelog.d/8742.doc | 1 + docs/admin_api/event_reports.md | 172 +++++++++++++++++++++++++++++++ docs/admin_api/event_reports.rst | 165 ----------------------------- 3 files changed, 173 insertions(+), 165 deletions(-) create mode 100644 changelog.d/8742.doc create mode 100644 docs/admin_api/event_reports.md delete mode 100644 docs/admin_api/event_reports.rst diff --git a/changelog.d/8742.doc b/changelog.d/8742.doc new file mode 100644 index 0000000000..cbae6cfdc6 --- /dev/null +++ b/changelog.d/8742.doc @@ -0,0 +1 @@ +Migrate documentation `docs/admin_api/event_reports` to markdown. \ No newline at end of file diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md new file mode 100644 index 0000000000..0159098138 --- /dev/null +++ b/docs/admin_api/event_reports.md @@ -0,0 +1,172 @@ +# Show reported events + +This API returns information about reported events. + +The api is: +``` +GET /_synapse/admin/v1/event_reports?from=0&limit=10 +``` +To use it, you will need to authenticate by providing an `access_token` for a +server admin: see [README.rst](README.rst). + +It returns a JSON body like the following: + +```json +{ + "event_reports": [ + { + "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY", + "id": 2, + "reason": "foo", + "score": -100, + "received_ts": 1570897107409, + "canonical_alias": "#alias1:matrix.org", + "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org", + "name": "Matrix HQ", + "sender": "@foobar:matrix.org", + "user_id": "@foo:matrix.org" + }, + { + "event_id": "$3IcdZsDaN_En-S1DF4EMCy3v4gNRKeOJs8W5qTOKj4I", + "id": 3, + "reason": "bar", + "score": -100, + "received_ts": 1598889612059, + "canonical_alias": "#alias2:matrix.org", + "room_id": "!eGvUQuTCkHGVwNMOjv:matrix.org", + "name": "Your room name here", + "sender": "@foobar:matrix.org", + "user_id": "@bar:matrix.org" + } + ], + "next_token": 2, + "total": 4 +} +``` + +To paginate, check for `next_token` and if present, call the endpoint again with `from` +set to the value of `next_token`. This will return a new page. + +If the endpoint does not return a `next_token` then there are no more reports to +paginate through. + +**URL parameters:** + +* `limit`: integer - Is optional but is used for pagination, denoting the maximum number + of items to return in this call. Defaults to `100`. +* `from`: integer - Is optional but used for pagination, denoting the offset in the + returned results. This should be treated as an opaque value and not explicitly set to + anything other than the return value of `next_token` from a previous call. Defaults to `0`. +* `dir`: string - Direction of event report order. Whether to fetch the most recent + first (`b`) or the oldest first (`f`). Defaults to `b`. +* `user_id`: string - Is optional and filters to only return users with user IDs that + contain this value. This is the user who reported the event and wrote the reason. +* `room_id`: string - Is optional and filters to only return rooms with room IDs that + contain this value. + +**Response** + +The following fields are returned in the JSON response body: + +* `id`: integer - ID of event report. +* `received_ts`: integer - The timestamp (in milliseconds since the unix epoch) when this + report was sent. +* `room_id`: string - The ID of the room in which the event being reported is located. +* `name`: string - The name of the room. +* `event_id`: string - The ID of the reported event. +* `user_id`: string - This is the user who reported the event and wrote the reason. +* `reason`: string - Comment made by the `user_id` in this report. May be blank. +* `score`: integer - Content is reported based upon a negative score, where -100 is + "most offensive" and 0 is "inoffensive". +* `sender`: string - This is the ID of the user who sent the original message/event that + was reported. +* `canonical_alias`: string - The canonical alias of the room. `null` if the room does not + have a canonical alias set. +* `next_token`: integer - Indication for pagination. See above. +* `total`: integer - Total number of event reports related to the query + (`user_id` and `room_id`). + +# Show details of a specific event report + +This API returns information about a specific event report. + +The api is: +``` +GET /_synapse/admin/v1/event_reports/ +``` +To use it, you will need to authenticate by providing an `access_token` for a +server admin: see [README.rst](README.rst). + +It returns a JSON body like the following: + +```jsonc +{ + "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY", + "event_json": { + "auth_events": [ + "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M", + "$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws" + ], + "content": { + "body": "matrix.org: This Week in Matrix", + "format": "org.matrix.custom.html", + "formatted_body": "matrix.org:
This Week in Matrix", + "msgtype": "m.notice" + }, + "depth": 546, + "hashes": { + "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw" + }, + "origin": "matrix.org", + "origin_server_ts": 1592291711430, + "prev_events": [ + "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M" + ], + "prev_state": [], + "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org", + "sender": "@foobar:matrix.org", + "signatures": { + "matrix.org": { + "ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg" + } + }, + "type": "m.room.message", + "unsigned": { + "age_ts": 1592291711430, + } + }, + "id": , + "reason": "foo", + "score": -100, + "received_ts": 1570897107409, + "canonical_alias": "#alias1:matrix.org", + "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org", + "name": "Matrix HQ", + "sender": "@foobar:matrix.org", + "user_id": "@foo:matrix.org" +} +``` + +**URL parameters:** + +* `report_id`: string - The ID of the event report. + +**Response** + +The following fields are returned in the JSON response body: + +* `id`: integer - ID of event report. +* `received_ts`: integer - The timestamp (in milliseconds since the unix epoch) when this + report was sent. +* `room_id`: string - The ID of the room in which the event being reported is located. +* `name`: string - The name of the room. +* `event_id`: string - The ID of the reported event. +* `user_id`: string - This is the user who reported the event and wrote the reason. +* `reason`: string - Comment made by the `user_id` in this report. May be blank. +* `score`: integer - Content is reported based upon a negative score, where -100 is + "most offensive" and 0 is "inoffensive". +* `sender`: string - This is the ID of the user who sent the original message/event that + was reported. +* `canonical_alias`: string - The canonical alias of the room. `null` if the room does not + have a canonical alias set. +* `event_json`: object - Details of the original event that was reported. diff --git a/docs/admin_api/event_reports.rst b/docs/admin_api/event_reports.rst deleted file mode 100644 index 5f7b0fa6bb..0000000000 --- a/docs/admin_api/event_reports.rst +++ /dev/null @@ -1,165 +0,0 @@ -Show reported events -==================== - -This API returns information about reported events. - -The api is:: - - GET /_synapse/admin/v1/event_reports?from=0&limit=10 - -To use it, you will need to authenticate by providing an ``access_token`` for a -server admin: see `README.rst `_. - -It returns a JSON body like the following: - -.. code:: jsonc - - { - "event_reports": [ - { - "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY", - "id": 2, - "reason": "foo", - "score": -100, - "received_ts": 1570897107409, - "canonical_alias": "#alias1:matrix.org", - "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org", - "name": "Matrix HQ", - "sender": "@foobar:matrix.org", - "user_id": "@foo:matrix.org" - }, - { - "event_id": "$3IcdZsDaN_En-S1DF4EMCy3v4gNRKeOJs8W5qTOKj4I", - "id": 3, - "reason": "bar", - "score": -100, - "received_ts": 1598889612059, - "canonical_alias": "#alias2:matrix.org", - "room_id": "!eGvUQuTCkHGVwNMOjv:matrix.org", - "name": "Your room name here", - "sender": "@foobar:matrix.org", - "user_id": "@bar:matrix.org" - } - ], - "next_token": 2, - "total": 4 - } - -To paginate, check for ``next_token`` and if present, call the endpoint again -with ``from`` set to the value of ``next_token``. This will return a new page. - -If the endpoint does not return a ``next_token`` then there are no more -reports to paginate through. - -**URL parameters:** - -- ``limit``: integer - Is optional but is used for pagination, - denoting the maximum number of items to return in this call. Defaults to ``100``. -- ``from``: integer - Is optional but used for pagination, - denoting the offset in the returned results. This should be treated as an opaque value and - not explicitly set to anything other than the return value of ``next_token`` from a previous call. - Defaults to ``0``. -- ``dir``: string - Direction of event report order. Whether to fetch the most recent first (``b``) or the - oldest first (``f``). Defaults to ``b``. -- ``user_id``: string - Is optional and filters to only return users with user IDs that contain this value. - This is the user who reported the event and wrote the reason. -- ``room_id``: string - Is optional and filters to only return rooms with room IDs that contain this value. - -**Response** - -The following fields are returned in the JSON response body: - -- ``id``: integer - ID of event report. -- ``received_ts``: integer - The timestamp (in milliseconds since the unix epoch) when this report was sent. -- ``room_id``: string - The ID of the room in which the event being reported is located. -- ``name``: string - The name of the room. -- ``event_id``: string - The ID of the reported event. -- ``user_id``: string - This is the user who reported the event and wrote the reason. -- ``reason``: string - Comment made by the ``user_id`` in this report. May be blank. -- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive". -- ``sender``: string - This is the ID of the user who sent the original message/event that was reported. -- ``canonical_alias``: string - The canonical alias of the room. ``null`` if the room does not have a canonical alias set. -- ``next_token``: integer - Indication for pagination. See above. -- ``total``: integer - Total number of event reports related to the query (``user_id`` and ``room_id``). - -Show details of a specific event report -======================================= - -This API returns information about a specific event report. - -The api is:: - - GET /_synapse/admin/v1/event_reports/ - -To use it, you will need to authenticate by providing an ``access_token`` for a -server admin: see `README.rst `_. - -It returns a JSON body like the following: - -.. code:: jsonc - - { - "event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY", - "event_json": { - "auth_events": [ - "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M", - "$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws" - ], - "content": { - "body": "matrix.org: This Week in Matrix", - "format": "org.matrix.custom.html", - "formatted_body": "matrix.org:
This Week in Matrix", - "msgtype": "m.notice" - }, - "depth": 546, - "hashes": { - "sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw" - }, - "origin": "matrix.org", - "origin_server_ts": 1592291711430, - "prev_events": [ - "$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M" - ], - "prev_state": [], - "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org", - "sender": "@foobar:matrix.org", - "signatures": { - "matrix.org": { - "ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg" - } - }, - "type": "m.room.message", - "unsigned": { - "age_ts": 1592291711430, - } - }, - "id": , - "reason": "foo", - "score": -100, - "received_ts": 1570897107409, - "canonical_alias": "#alias1:matrix.org", - "room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org", - "name": "Matrix HQ", - "sender": "@foobar:matrix.org", - "user_id": "@foo:matrix.org" - } - -**URL parameters:** - -- ``report_id``: string - The ID of the event report. - -**Response** - -The following fields are returned in the JSON response body: - -- ``id``: integer - ID of event report. -- ``received_ts``: integer - The timestamp (in milliseconds since the unix epoch) when this report was sent. -- ``room_id``: string - The ID of the room in which the event being reported is located. -- ``name``: string - The name of the room. -- ``event_id``: string - The ID of the reported event. -- ``user_id``: string - This is the user who reported the event and wrote the reason. -- ``reason``: string - Comment made by the ``user_id`` in this report. May be blank. -- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive". -- ``sender``: string - This is the ID of the user who sent the original message/event that was reported. -- ``canonical_alias``: string - The canonical alias of the room. ``null`` if the room does not have a canonical alias set. -- ``event_json``: object - Details of the original event that was reported. From 4a54b821bba9d8faff401771c8ba2a20a78c5f65 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Nov 2020 13:59:58 +0000 Subject: [PATCH 04/57] 1.23.0rc1 --- CHANGES.md | 86 ++++++++++++++++++++++++++++++++++++++++ changelog.d/8286.feature | 1 - changelog.d/8455.bugfix | 1 - changelog.d/8519.feature | 1 - changelog.d/8539.feature | 1 - changelog.d/8559.misc | 1 - changelog.d/8580.bugfix | 1 - changelog.d/8582.doc | 1 - changelog.d/8595.misc | 1 - changelog.d/8607.feature | 1 - changelog.d/8610.feature | 1 - changelog.d/8614.misc | 1 - changelog.d/8615.misc | 1 - changelog.d/8616.misc | 1 - changelog.d/8620.bugfix | 1 - changelog.d/8621.misc | 1 - changelog.d/8627.bugfix | 1 - changelog.d/8628.bugfix | 1 - changelog.d/8632.bugfix | 1 - changelog.d/8633.misc | 1 - changelog.d/8634.misc | 1 - changelog.d/8635.doc | 1 - changelog.d/8636.misc | 1 - changelog.d/8639.misc | 1 - changelog.d/8640.misc | 1 - changelog.d/8643.bugfix | 1 - changelog.d/8644.misc | 1 - changelog.d/8647.feature | 1 - changelog.d/8655.misc | 1 - changelog.d/8657.doc | 1 - changelog.d/8664.misc | 1 - changelog.d/8665.doc | 1 - changelog.d/8666.doc | 1 - changelog.d/8667.doc | 1 - changelog.d/8668.misc | 1 - changelog.d/8669.misc | 1 - changelog.d/8670.misc | 1 - changelog.d/8671.misc | 1 - changelog.d/8679.misc | 1 - changelog.d/8680.misc | 1 - changelog.d/8682.bugfix | 1 - changelog.d/8684.misc | 1 - changelog.d/8685.feature | 1 - changelog.d/8688.misc | 1 - changelog.d/8689.feature | 1 - changelog.d/8690.misc | 1 - changelog.d/8693.misc | 1 - changelog.d/8694.misc | 1 - changelog.d/8697.misc | 1 - changelog.d/8698.misc | 1 - changelog.d/8700.feature | 1 - changelog.d/8701.doc | 1 - changelog.d/8702.misc | 1 - changelog.d/8705.misc | 1 - changelog.d/8706.doc | 1 - changelog.d/8708.misc | 1 - changelog.d/8712.misc | 1 - changelog.d/8713.misc | 1 - changelog.d/8714.doc | 1 - changelog.d/8718.misc | 1 - changelog.d/8719.misc | 1 - changelog.d/8722.feature | 1 - changelog.d/8726.bugfix | 1 - changelog.d/8728.bugfix | 1 - changelog.d/8729.bugfix | 1 - changelog.d/8730.bugfix | 1 - changelog.d/8742.doc | 1 - changelog.d/8746.doc | 1 - changelog.d/8752.misc | 1 - changelog.d/8755.bugfix | 1 - synapse/__init__.py | 2 +- 71 files changed, 87 insertions(+), 70 deletions(-) delete mode 100644 changelog.d/8286.feature delete mode 100644 changelog.d/8455.bugfix delete mode 100644 changelog.d/8519.feature delete mode 100644 changelog.d/8539.feature delete mode 100644 changelog.d/8559.misc delete mode 100644 changelog.d/8580.bugfix delete mode 100644 changelog.d/8582.doc delete mode 100644 changelog.d/8595.misc delete mode 100644 changelog.d/8607.feature delete mode 100644 changelog.d/8610.feature delete mode 100644 changelog.d/8614.misc delete mode 100644 changelog.d/8615.misc delete mode 100644 changelog.d/8616.misc delete mode 100644 changelog.d/8620.bugfix delete mode 100644 changelog.d/8621.misc delete mode 100644 changelog.d/8627.bugfix delete mode 100644 changelog.d/8628.bugfix delete mode 100644 changelog.d/8632.bugfix delete mode 100644 changelog.d/8633.misc delete mode 100644 changelog.d/8634.misc delete mode 100644 changelog.d/8635.doc delete mode 100644 changelog.d/8636.misc delete mode 100644 changelog.d/8639.misc delete mode 100644 changelog.d/8640.misc delete mode 100644 changelog.d/8643.bugfix delete mode 100644 changelog.d/8644.misc delete mode 100644 changelog.d/8647.feature delete mode 100644 changelog.d/8655.misc delete mode 100644 changelog.d/8657.doc delete mode 100644 changelog.d/8664.misc delete mode 100644 changelog.d/8665.doc delete mode 100644 changelog.d/8666.doc delete mode 100644 changelog.d/8667.doc delete mode 100644 changelog.d/8668.misc delete mode 100644 changelog.d/8669.misc delete mode 100644 changelog.d/8670.misc delete mode 100644 changelog.d/8671.misc delete mode 100644 changelog.d/8679.misc delete mode 100644 changelog.d/8680.misc delete mode 100644 changelog.d/8682.bugfix delete mode 100644 changelog.d/8684.misc delete mode 100644 changelog.d/8685.feature delete mode 100644 changelog.d/8688.misc delete mode 100644 changelog.d/8689.feature delete mode 100644 changelog.d/8690.misc delete mode 100644 changelog.d/8693.misc delete mode 100644 changelog.d/8694.misc delete mode 100644 changelog.d/8697.misc delete mode 100644 changelog.d/8698.misc delete mode 100644 changelog.d/8700.feature delete mode 100644 changelog.d/8701.doc delete mode 100644 changelog.d/8702.misc delete mode 100644 changelog.d/8705.misc delete mode 100644 changelog.d/8706.doc delete mode 100644 changelog.d/8708.misc delete mode 100644 changelog.d/8712.misc delete mode 100644 changelog.d/8713.misc delete mode 100644 changelog.d/8714.doc delete mode 100644 changelog.d/8718.misc delete mode 100644 changelog.d/8719.misc delete mode 100644 changelog.d/8722.feature delete mode 100644 changelog.d/8726.bugfix delete mode 100644 changelog.d/8728.bugfix delete mode 100644 changelog.d/8729.bugfix delete mode 100644 changelog.d/8730.bugfix delete mode 100644 changelog.d/8742.doc delete mode 100644 changelog.d/8746.doc delete mode 100644 changelog.d/8752.misc delete mode 100644 changelog.d/8755.bugfix diff --git a/CHANGES.md b/CHANGES.md index 8bd7825089..dc7d245812 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,89 @@ +Synapse 1.23.0rc1 (2020-11-13) +============================== + +Features +-------- + +- Add a push rule that highlights when a jitsi conference is created in a room. ([\#8286](https://github.com/matrix-org/synapse/issues/8286)) +- Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel. ([\#8519](https://github.com/matrix-org/synapse/issues/8519)) +- Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel. ([\#8539](https://github.com/matrix-org/synapse/issues/8539)) +- Support generating structured logs via the standard logging configuration. ([\#8607](https://github.com/matrix-org/synapse/issues/8607), [\#8685](https://github.com/matrix-org/synapse/issues/8685)) +- Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel. ([\#8610](https://github.com/matrix-org/synapse/issues/8610), [\#8689](https://github.com/matrix-org/synapse/issues/8689)) +- Add an admin API `GET /_synapse/admin/v1/users//media` to get information about uploaded media. Contributed by @dklimpel. ([\#8647](https://github.com/matrix-org/synapse/issues/8647)) +- Add an admin API for local user media statistics. Contributed by @dklimpel. ([\#8700](https://github.com/matrix-org/synapse/issues/8700)) +- Add `displayname` to Shared-Secret Registration for admins. ([\#8722](https://github.com/matrix-org/synapse/issues/8722)) + + +Bugfixes +-------- + +- Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already. ([\#8455](https://github.com/matrix-org/synapse/issues/8455)) +- Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information. ([\#8580](https://github.com/matrix-org/synapse/issues/8580)) +- Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error. ([\#8620](https://github.com/matrix-org/synapse/issues/8620)) +- Fix email notifications for invites without local state. ([\#8627](https://github.com/matrix-org/synapse/issues/8627)) +- Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500. ([\#8628](https://github.com/matrix-org/synapse/issues/8628)) +- Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded. ([\#8632](https://github.com/matrix-org/synapse/issues/8632)) +- Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0. ([\#8643](https://github.com/matrix-org/synapse/issues/8643)) +- Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories. ([\#8682](https://github.com/matrix-org/synapse/issues/8682)) +- Fix bug where Synapse would not recover after losing connection to the database. ([\#8726](https://github.com/matrix-org/synapse/issues/8726)) +- Fix bug where the `/_synapse/admin/v1/send_server_notice` API could send notices to non-notice rooms. ([\#8728](https://github.com/matrix-org/synapse/issues/8728)) +- Fix port script fails when DB has no backfilled events. Broke in v1.21.0. ([\#8729](https://github.com/matrix-org/synapse/issues/8729)) +- Fix port script to correctly handle foreign key constraints. Broke in v1.21.0. ([\#8730](https://github.com/matrix-org/synapse/issues/8730)) +- Fix port script so that it can be run again after a failure. Broke in v1.21.0. ([\#8755](https://github.com/matrix-org/synapse/issues/8755)) + + +Improved Documentation +---------------------- + +- Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk. ([\#8582](https://github.com/matrix-org/synapse/issues/8582)) +- Improve the sample configuration for single sign-on providers. ([\#8635](https://github.com/matrix-org/synapse/issues/8635)) +- Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs. ([\#8657](https://github.com/matrix-org/synapse/issues/8657)) +- Note support for Python 3.9. ([\#8665](https://github.com/matrix-org/synapse/issues/8665)) +- Minor updates to docs on running tests. ([\#8666](https://github.com/matrix-org/synapse/issues/8666)) +- Interlink prometheus/grafana documentation. ([\#8667](https://github.com/matrix-org/synapse/issues/8667)) +- Notes on SSO logins and media_repository worker. ([\#8701](https://github.com/matrix-org/synapse/issues/8701)) +- Document experimental support for running multiple event persisters. ([\#8706](https://github.com/matrix-org/synapse/issues/8706)) +- Add information regarding the various sources of, and expected contributions to, Synapse's documentation to `CONTRIBUTING.md`. ([\#8714](https://github.com/matrix-org/synapse/issues/8714)) +- Migrate documentation `docs/admin_api/event_reports` to markdown. ([\#8742](https://github.com/matrix-org/synapse/issues/8742)) +- Add some helpful hints to the README for new Synapse developers. Contributed by @chagai95. ([\#8746](https://github.com/matrix-org/synapse/issues/8746)) + + +Internal Changes +---------------- + +- Optimise `/createRoom` with multiple invited users. ([\#8559](https://github.com/matrix-org/synapse/issues/8559)) +- Implement and use an @lru_cache decorator. ([\#8595](https://github.com/matrix-org/synapse/issues/8595)) +- Don't instansiate Requester directly. ([\#8614](https://github.com/matrix-org/synapse/issues/8614)) +- Type hints for `RegistrationStore`. ([\#8615](https://github.com/matrix-org/synapse/issues/8615)) +- Change schema to support access tokens belonging to one user but granting access to another. ([\#8616](https://github.com/matrix-org/synapse/issues/8616)) +- Remove unused OPTIONS handlers. ([\#8621](https://github.com/matrix-org/synapse/issues/8621)) +- Run `mypy` as part of the lint.sh script. ([\#8633](https://github.com/matrix-org/synapse/issues/8633)) +- Correct Synapse's PyPI package name in the OpenID Connect installation instructions. ([\#8634](https://github.com/matrix-org/synapse/issues/8634)) +- Catch exceptions during initialization of `password_providers`. Contributed by Nicolai Søborg. ([\#8636](https://github.com/matrix-org/synapse/issues/8636)) +- Fix typos and spelling errors in the code. ([\#8639](https://github.com/matrix-org/synapse/issues/8639)) +- Reduce number of OpenTracing spans started. ([\#8640](https://github.com/matrix-org/synapse/issues/8640), [\#8668](https://github.com/matrix-org/synapse/issues/8668), [\#8670](https://github.com/matrix-org/synapse/issues/8670)) +- Add field `total` to device list in admin API. ([\#8644](https://github.com/matrix-org/synapse/issues/8644)) +- Add more type hints to the application services code. ([\#8655](https://github.com/matrix-org/synapse/issues/8655), [\#8693](https://github.com/matrix-org/synapse/issues/8693)) +- Tell Black to format code for Python 3.5. ([\#8664](https://github.com/matrix-org/synapse/issues/8664)) +- Don't pull event from DB when handling replication traffic. ([\#8669](https://github.com/matrix-org/synapse/issues/8669)) +- Abstract some invite-related code in preparation for landing knocking. ([\#8671](https://github.com/matrix-org/synapse/issues/8671), [\#8688](https://github.com/matrix-org/synapse/issues/8688)) +- Clarify representation of events in logfiles. ([\#8679](https://github.com/matrix-org/synapse/issues/8679)) +- Don't require `hiredis` package to be installed to run unit tests. ([\#8680](https://github.com/matrix-org/synapse/issues/8680)) +- Fix typing info on cache call signature to accept `on_invalidate`. ([\#8684](https://github.com/matrix-org/synapse/issues/8684)) +- Fail tests if they do not await coroutines. ([\#8690](https://github.com/matrix-org/synapse/issues/8690)) +- Improve start time by adding an index to `e2e_cross_signing_keys.stream_id`. ([\#8694](https://github.com/matrix-org/synapse/issues/8694)) +- Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. ([\#8697](https://github.com/matrix-org/synapse/issues/8697)) +- Use Python 3.8 in Docker images by default. ([\#8698](https://github.com/matrix-org/synapse/issues/8698)) +- Remove the "draft" status of the Room Details Admin API. ([\#8702](https://github.com/matrix-org/synapse/issues/8702)) +- Improve the error returned when a non-string displayname or avatar_url is used when updating a user's profile. ([\#8705](https://github.com/matrix-org/synapse/issues/8705)) +- Block attempts by clients to send server ACLs, or redactions of server ACLs, that would result in the local server being blocked from the room. ([\#8708](https://github.com/matrix-org/synapse/issues/8708)) +- Add metrics the allow the local sysadmin to track 3PID `/requestToken` requests. ([\#8712](https://github.com/matrix-org/synapse/issues/8712)) +- Consolidate duplicated lists of purged tables that are checked in tests. ([\#8713](https://github.com/matrix-org/synapse/issues/8713)) +- Add some `mdui:UIInfo` element examples for `saml2_config` in the homeserver config. ([\#8718](https://github.com/matrix-org/synapse/issues/8718)) +- Improve the error message returned when a remote server incorrectly sets the `Content-Type` header in response to a JSON request. ([\#8719](https://github.com/matrix-org/synapse/issues/8719)) +- Speed up repeated state resolutions on the same room by caching event ID to auth event ID lookups. ([\#8752](https://github.com/matrix-org/synapse/issues/8752)) + + Synapse 1.22.1 (2020-10-30) =========================== diff --git a/changelog.d/8286.feature b/changelog.d/8286.feature deleted file mode 100644 index 2c371419af..0000000000 --- a/changelog.d/8286.feature +++ /dev/null @@ -1 +0,0 @@ -Add a push rule that highlights when a jitsi conference is created in a room. diff --git a/changelog.d/8455.bugfix b/changelog.d/8455.bugfix deleted file mode 100644 index 561e73f5e0..0000000000 --- a/changelog.d/8455.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already. diff --git a/changelog.d/8519.feature b/changelog.d/8519.feature deleted file mode 100644 index e2ab548681..0000000000 --- a/changelog.d/8519.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8539.feature b/changelog.d/8539.feature deleted file mode 100644 index 15ce02fb86..0000000000 --- a/changelog.d/8539.feature +++ /dev/null @@ -1 +0,0 @@ -Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8559.misc b/changelog.d/8559.misc deleted file mode 100644 index d7bd00964e..0000000000 --- a/changelog.d/8559.misc +++ /dev/null @@ -1 +0,0 @@ -Optimise `/createRoom` with multiple invited users. diff --git a/changelog.d/8580.bugfix b/changelog.d/8580.bugfix deleted file mode 100644 index 31734fd97d..0000000000 --- a/changelog.d/8580.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information. diff --git a/changelog.d/8582.doc b/changelog.d/8582.doc deleted file mode 100644 index 041f168717..0000000000 --- a/changelog.d/8582.doc +++ /dev/null @@ -1 +0,0 @@ -Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk. diff --git a/changelog.d/8595.misc b/changelog.d/8595.misc deleted file mode 100644 index 24fab65cda..0000000000 --- a/changelog.d/8595.misc +++ /dev/null @@ -1 +0,0 @@ -Implement and use an @lru_cache decorator. diff --git a/changelog.d/8607.feature b/changelog.d/8607.feature deleted file mode 100644 index fef1eccb92..0000000000 --- a/changelog.d/8607.feature +++ /dev/null @@ -1 +0,0 @@ -Support generating structured logs via the standard logging configuration. diff --git a/changelog.d/8610.feature b/changelog.d/8610.feature deleted file mode 100644 index ed8d926964..0000000000 --- a/changelog.d/8610.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8614.misc b/changelog.d/8614.misc deleted file mode 100644 index 1bf9ea08f0..0000000000 --- a/changelog.d/8614.misc +++ /dev/null @@ -1 +0,0 @@ -Don't instansiate Requester directly. diff --git a/changelog.d/8615.misc b/changelog.d/8615.misc deleted file mode 100644 index 79fa7b7ff8..0000000000 --- a/changelog.d/8615.misc +++ /dev/null @@ -1 +0,0 @@ -Type hints for `RegistrationStore`. diff --git a/changelog.d/8616.misc b/changelog.d/8616.misc deleted file mode 100644 index 385b14063e..0000000000 --- a/changelog.d/8616.misc +++ /dev/null @@ -1 +0,0 @@ -Change schema to support access tokens belonging to one user but granting access to another. diff --git a/changelog.d/8620.bugfix b/changelog.d/8620.bugfix deleted file mode 100644 index c1078a3fb5..0000000000 --- a/changelog.d/8620.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error. diff --git a/changelog.d/8621.misc b/changelog.d/8621.misc deleted file mode 100644 index 5720b665fe..0000000000 --- a/changelog.d/8621.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unused OPTIONS handlers. diff --git a/changelog.d/8627.bugfix b/changelog.d/8627.bugfix deleted file mode 100644 index 143cf95f92..0000000000 --- a/changelog.d/8627.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix email notifications for invites without local state. diff --git a/changelog.d/8628.bugfix b/changelog.d/8628.bugfix deleted file mode 100644 index 1316136ca2..0000000000 --- a/changelog.d/8628.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix handling of invalid group IDs to return a 400 rather than log an exception and return a 500. diff --git a/changelog.d/8632.bugfix b/changelog.d/8632.bugfix deleted file mode 100644 index 7d834aa2e2..0000000000 --- a/changelog.d/8632.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded. diff --git a/changelog.d/8633.misc b/changelog.d/8633.misc deleted file mode 100644 index 8e1d006b36..0000000000 --- a/changelog.d/8633.misc +++ /dev/null @@ -1 +0,0 @@ -Run `mypy` as part of the lint.sh script. diff --git a/changelog.d/8634.misc b/changelog.d/8634.misc deleted file mode 100644 index c4f74ba7c9..0000000000 --- a/changelog.d/8634.misc +++ /dev/null @@ -1 +0,0 @@ -Correct Synapse's PyPI package name in the OpenID Connect installation instructions. \ No newline at end of file diff --git a/changelog.d/8635.doc b/changelog.d/8635.doc deleted file mode 100644 index 00fb1e61a7..0000000000 --- a/changelog.d/8635.doc +++ /dev/null @@ -1 +0,0 @@ -Improve the sample configuration for single sign-on providers. diff --git a/changelog.d/8636.misc b/changelog.d/8636.misc deleted file mode 100644 index df4dca42f8..0000000000 --- a/changelog.d/8636.misc +++ /dev/null @@ -1 +0,0 @@ -Catch exceptions during initialization of `password_providers`. Contributed by Nicolai Søborg. diff --git a/changelog.d/8639.misc b/changelog.d/8639.misc deleted file mode 100644 index 20a213df39..0000000000 --- a/changelog.d/8639.misc +++ /dev/null @@ -1 +0,0 @@ -Fix typos and spelling errors in the code. diff --git a/changelog.d/8640.misc b/changelog.d/8640.misc deleted file mode 100644 index cf6023f783..0000000000 --- a/changelog.d/8640.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of OpenTracing spans started. diff --git a/changelog.d/8643.bugfix b/changelog.d/8643.bugfix deleted file mode 100644 index fcda1ca871..0000000000 --- a/changelog.d/8643.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0. diff --git a/changelog.d/8644.misc b/changelog.d/8644.misc deleted file mode 100644 index 87f2b72924..0000000000 --- a/changelog.d/8644.misc +++ /dev/null @@ -1 +0,0 @@ -Add field `total` to device list in admin API. \ No newline at end of file diff --git a/changelog.d/8647.feature b/changelog.d/8647.feature deleted file mode 100644 index 79e98f6e90..0000000000 --- a/changelog.d/8647.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin API `GET /_synapse/admin/v1/users//media` to get information about uploaded media. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8655.misc b/changelog.d/8655.misc deleted file mode 100644 index b588bdd3e2..0000000000 --- a/changelog.d/8655.misc +++ /dev/null @@ -1 +0,0 @@ -Add more type hints to the application services code. diff --git a/changelog.d/8657.doc b/changelog.d/8657.doc deleted file mode 100644 index 3dcbb221af..0000000000 --- a/changelog.d/8657.doc +++ /dev/null @@ -1 +0,0 @@ -Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs. diff --git a/changelog.d/8664.misc b/changelog.d/8664.misc deleted file mode 100644 index 278cf53adc..0000000000 --- a/changelog.d/8664.misc +++ /dev/null @@ -1 +0,0 @@ -Tell Black to format code for Python 3.5. diff --git a/changelog.d/8665.doc b/changelog.d/8665.doc deleted file mode 100644 index 3b75307dc5..0000000000 --- a/changelog.d/8665.doc +++ /dev/null @@ -1 +0,0 @@ -Note support for Python 3.9. diff --git a/changelog.d/8666.doc b/changelog.d/8666.doc deleted file mode 100644 index dee86b4a26..0000000000 --- a/changelog.d/8666.doc +++ /dev/null @@ -1 +0,0 @@ -Minor updates to docs on running tests. diff --git a/changelog.d/8667.doc b/changelog.d/8667.doc deleted file mode 100644 index 422d697da6..0000000000 --- a/changelog.d/8667.doc +++ /dev/null @@ -1 +0,0 @@ -Interlink prometheus/grafana documentation. diff --git a/changelog.d/8668.misc b/changelog.d/8668.misc deleted file mode 100644 index cf6023f783..0000000000 --- a/changelog.d/8668.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of OpenTracing spans started. diff --git a/changelog.d/8669.misc b/changelog.d/8669.misc deleted file mode 100644 index 5228105cd3..0000000000 --- a/changelog.d/8669.misc +++ /dev/null @@ -1 +0,0 @@ -Don't pull event from DB when handling replication traffic. diff --git a/changelog.d/8670.misc b/changelog.d/8670.misc deleted file mode 100644 index cf6023f783..0000000000 --- a/changelog.d/8670.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of OpenTracing spans started. diff --git a/changelog.d/8671.misc b/changelog.d/8671.misc deleted file mode 100644 index bef8dc425a..0000000000 --- a/changelog.d/8671.misc +++ /dev/null @@ -1 +0,0 @@ -Abstract some invite-related code in preparation for landing knocking. \ No newline at end of file diff --git a/changelog.d/8679.misc b/changelog.d/8679.misc deleted file mode 100644 index 662eced4cf..0000000000 --- a/changelog.d/8679.misc +++ /dev/null @@ -1 +0,0 @@ -Clarify representation of events in logfiles. diff --git a/changelog.d/8680.misc b/changelog.d/8680.misc deleted file mode 100644 index 2ca2975464..0000000000 --- a/changelog.d/8680.misc +++ /dev/null @@ -1 +0,0 @@ -Don't require `hiredis` package to be installed to run unit tests. diff --git a/changelog.d/8682.bugfix b/changelog.d/8682.bugfix deleted file mode 100644 index e61276aa05..0000000000 --- a/changelog.d/8682.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories. diff --git a/changelog.d/8684.misc b/changelog.d/8684.misc deleted file mode 100644 index 1d23d42926..0000000000 --- a/changelog.d/8684.misc +++ /dev/null @@ -1 +0,0 @@ -Fix typing info on cache call signature to accept `on_invalidate`. diff --git a/changelog.d/8685.feature b/changelog.d/8685.feature deleted file mode 100644 index fef1eccb92..0000000000 --- a/changelog.d/8685.feature +++ /dev/null @@ -1 +0,0 @@ -Support generating structured logs via the standard logging configuration. diff --git a/changelog.d/8688.misc b/changelog.d/8688.misc deleted file mode 100644 index bef8dc425a..0000000000 --- a/changelog.d/8688.misc +++ /dev/null @@ -1 +0,0 @@ -Abstract some invite-related code in preparation for landing knocking. \ No newline at end of file diff --git a/changelog.d/8689.feature b/changelog.d/8689.feature deleted file mode 100644 index ed8d926964..0000000000 --- a/changelog.d/8689.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/8690.misc b/changelog.d/8690.misc deleted file mode 100644 index 0f38ba1f5d..0000000000 --- a/changelog.d/8690.misc +++ /dev/null @@ -1 +0,0 @@ -Fail tests if they do not await coroutines. diff --git a/changelog.d/8693.misc b/changelog.d/8693.misc deleted file mode 100644 index b588bdd3e2..0000000000 --- a/changelog.d/8693.misc +++ /dev/null @@ -1 +0,0 @@ -Add more type hints to the application services code. diff --git a/changelog.d/8694.misc b/changelog.d/8694.misc deleted file mode 100644 index c90a6375ad..0000000000 --- a/changelog.d/8694.misc +++ /dev/null @@ -1 +0,0 @@ -Improve start time by adding an index to `e2e_cross_signing_keys.stream_id`. diff --git a/changelog.d/8697.misc b/changelog.d/8697.misc deleted file mode 100644 index 7982a4e46d..0000000000 --- a/changelog.d/8697.misc +++ /dev/null @@ -1 +0,0 @@ - Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. diff --git a/changelog.d/8698.misc b/changelog.d/8698.misc deleted file mode 100644 index 6b777fb295..0000000000 --- a/changelog.d/8698.misc +++ /dev/null @@ -1 +0,0 @@ -Use Python 3.8 in Docker images by default. diff --git a/changelog.d/8700.feature b/changelog.d/8700.feature deleted file mode 100644 index 47d63dce02..0000000000 --- a/changelog.d/8700.feature +++ /dev/null @@ -1 +0,0 @@ -Add an admin API for local user media statistics. Contributed by @dklimpel. diff --git a/changelog.d/8701.doc b/changelog.d/8701.doc deleted file mode 100644 index e2e8b2f79a..0000000000 --- a/changelog.d/8701.doc +++ /dev/null @@ -1 +0,0 @@ -Notes on SSO logins and media_repository worker. \ No newline at end of file diff --git a/changelog.d/8702.misc b/changelog.d/8702.misc deleted file mode 100644 index f20085cbe4..0000000000 --- a/changelog.d/8702.misc +++ /dev/null @@ -1 +0,0 @@ -Remove the "draft" status of the Room Details Admin API. \ No newline at end of file diff --git a/changelog.d/8705.misc b/changelog.d/8705.misc deleted file mode 100644 index 1189464a02..0000000000 --- a/changelog.d/8705.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the error returned when a non-string displayname or avatar_url is used when updating a user's profile. \ No newline at end of file diff --git a/changelog.d/8706.doc b/changelog.d/8706.doc deleted file mode 100644 index 96a0427e73..0000000000 --- a/changelog.d/8706.doc +++ /dev/null @@ -1 +0,0 @@ -Document experimental support for running multiple event persisters. diff --git a/changelog.d/8708.misc b/changelog.d/8708.misc deleted file mode 100644 index be679fb0f8..0000000000 --- a/changelog.d/8708.misc +++ /dev/null @@ -1 +0,0 @@ -Block attempts by clients to send server ACLs, or redactions of server ACLs, that would result in the local server being blocked from the room. diff --git a/changelog.d/8712.misc b/changelog.d/8712.misc deleted file mode 100644 index 90d63a9a23..0000000000 --- a/changelog.d/8712.misc +++ /dev/null @@ -1 +0,0 @@ -Add metrics the allow the local sysadmin to track 3PID `/requestToken` requests. diff --git a/changelog.d/8713.misc b/changelog.d/8713.misc deleted file mode 100644 index c5d3f3216b..0000000000 --- a/changelog.d/8713.misc +++ /dev/null @@ -1 +0,0 @@ -Consolidate duplicated lists of purged tables that are checked in tests. \ No newline at end of file diff --git a/changelog.d/8714.doc b/changelog.d/8714.doc deleted file mode 100644 index bda22714e7..0000000000 --- a/changelog.d/8714.doc +++ /dev/null @@ -1 +0,0 @@ -Add information regarding the various sources of, and expected contributions to, Synapse's documentation to `CONTRIBUTING.md`. \ No newline at end of file diff --git a/changelog.d/8718.misc b/changelog.d/8718.misc deleted file mode 100644 index f1868ce709..0000000000 --- a/changelog.d/8718.misc +++ /dev/null @@ -1 +0,0 @@ -Add some `mdui:UIInfo` element examples for `saml2_config` in the homeserver config. diff --git a/changelog.d/8719.misc b/changelog.d/8719.misc deleted file mode 100644 index 9aabef8fc3..0000000000 --- a/changelog.d/8719.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the error message returned when a remote server incorrectly sets the `Content-Type` header in response to a JSON request. diff --git a/changelog.d/8722.feature b/changelog.d/8722.feature deleted file mode 100644 index 0413d8838b..0000000000 --- a/changelog.d/8722.feature +++ /dev/null @@ -1 +0,0 @@ -Add `displayname` to Shared-Secret Registration for admins. \ No newline at end of file diff --git a/changelog.d/8726.bugfix b/changelog.d/8726.bugfix deleted file mode 100644 index 831f773a25..0000000000 --- a/changelog.d/8726.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where Synapse would not recover after losing connection to the database. diff --git a/changelog.d/8728.bugfix b/changelog.d/8728.bugfix deleted file mode 100644 index 8064aad0ff..0000000000 --- a/changelog.d/8728.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where the `/_synapse/admin/v1/send_server_notice` API could send notices to non-notice rooms. diff --git a/changelog.d/8729.bugfix b/changelog.d/8729.bugfix deleted file mode 100644 index 7f59a3b9e2..0000000000 --- a/changelog.d/8729.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix port script fails when DB has no backfilled events. Broke in v1.21.0. diff --git a/changelog.d/8730.bugfix b/changelog.d/8730.bugfix deleted file mode 100644 index dcc42bc981..0000000000 --- a/changelog.d/8730.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix port script to correctly handle foreign key constraints. Broke in v1.21.0. diff --git a/changelog.d/8742.doc b/changelog.d/8742.doc deleted file mode 100644 index cbae6cfdc6..0000000000 --- a/changelog.d/8742.doc +++ /dev/null @@ -1 +0,0 @@ -Migrate documentation `docs/admin_api/event_reports` to markdown. \ No newline at end of file diff --git a/changelog.d/8746.doc b/changelog.d/8746.doc deleted file mode 100644 index 6baf58ba7a..0000000000 --- a/changelog.d/8746.doc +++ /dev/null @@ -1 +0,0 @@ -Add some helpful hints to the README for new Synapse developers. Contributed by @chagai95. \ No newline at end of file diff --git a/changelog.d/8752.misc b/changelog.d/8752.misc deleted file mode 100644 index eac92e9d1d..0000000000 --- a/changelog.d/8752.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up repeated state resolutions on the same room by caching event ID to auth event ID lookups. diff --git a/changelog.d/8755.bugfix b/changelog.d/8755.bugfix deleted file mode 100644 index 42bbed3ac2..0000000000 --- a/changelog.d/8755.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix port script so that it can be run again after a failure. Broke in v1.21.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index 3e1df2b035..537f2239e5 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -48,7 +48,7 @@ try: except ImportError: pass -__version__ = "1.22.1" +__version__ = "1.23.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 0a5185495bcb7c946214b4398d3a705f80c5354e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Nov 2020 14:06:52 +0000 Subject: [PATCH 05/57] Fix changelog --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dc7d245812..f05bbe7868 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,10 +5,10 @@ Features -------- - Add a push rule that highlights when a jitsi conference is created in a room. ([\#8286](https://github.com/matrix-org/synapse/issues/8286)) -- Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel. ([\#8519](https://github.com/matrix-org/synapse/issues/8519)) +- Add an admin api to delete a single file or files that were not used for a defined time from server. Contributed by @dklimpel. ([\#8519](https://github.com/matrix-org/synapse/issues/8519)) - Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel. ([\#8539](https://github.com/matrix-org/synapse/issues/8539)) - Support generating structured logs via the standard logging configuration. ([\#8607](https://github.com/matrix-org/synapse/issues/8607), [\#8685](https://github.com/matrix-org/synapse/issues/8685)) -- Add an admin APIs to allow server admins to list users' pushers. Contributed by @dklimpel. ([\#8610](https://github.com/matrix-org/synapse/issues/8610), [\#8689](https://github.com/matrix-org/synapse/issues/8689)) +- Add an admin API to allow server admins to list users' pushers. Contributed by @dklimpel. ([\#8610](https://github.com/matrix-org/synapse/issues/8610), [\#8689](https://github.com/matrix-org/synapse/issues/8689)) - Add an admin API `GET /_synapse/admin/v1/users//media` to get information about uploaded media. Contributed by @dklimpel. ([\#8647](https://github.com/matrix-org/synapse/issues/8647)) - Add an admin API for local user media statistics. Contributed by @dklimpel. ([\#8700](https://github.com/matrix-org/synapse/issues/8700)) - Add `displayname` to Shared-Secret Registration for admins. ([\#8722](https://github.com/matrix-org/synapse/issues/8722)) From 34226ec7614027046e98fc3447b2ed7f2d47226e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Nov 2020 14:14:09 +0000 Subject: [PATCH 06/57] Fix changelog --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f05bbe7868..75871979c2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,11 +25,11 @@ Bugfixes - Fix handling of User-Agent headers that are invalid UTF-8, which caused user agents of users to not get correctly recorded. ([\#8632](https://github.com/matrix-org/synapse/issues/8632)) - Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0. ([\#8643](https://github.com/matrix-org/synapse/issues/8643)) - Fix exception during handling multiple concurrent requests for remote media when using multiple media repositories. ([\#8682](https://github.com/matrix-org/synapse/issues/8682)) -- Fix bug where Synapse would not recover after losing connection to the database. ([\#8726](https://github.com/matrix-org/synapse/issues/8726)) +- Fix bug that prevented Synapse from recovering after losing connection to the database. ([\#8726](https://github.com/matrix-org/synapse/issues/8726)) - Fix bug where the `/_synapse/admin/v1/send_server_notice` API could send notices to non-notice rooms. ([\#8728](https://github.com/matrix-org/synapse/issues/8728)) -- Fix port script fails when DB has no backfilled events. Broke in v1.21.0. ([\#8729](https://github.com/matrix-org/synapse/issues/8729)) -- Fix port script to correctly handle foreign key constraints. Broke in v1.21.0. ([\#8730](https://github.com/matrix-org/synapse/issues/8730)) -- Fix port script so that it can be run again after a failure. Broke in v1.21.0. ([\#8755](https://github.com/matrix-org/synapse/issues/8755)) +- Fix PostgreSQL port script fails when DB has no backfilled events. Broke in v1.21.0. ([\#8729](https://github.com/matrix-org/synapse/issues/8729)) +- Fix PostgreSQL port script to correctly handle foreign key constraints. Broke in v1.21.0. ([\#8730](https://github.com/matrix-org/synapse/issues/8730)) +- Fix PostgreSQL port script so that it can be run again after a failure. Broke in v1.21.0. ([\#8755](https://github.com/matrix-org/synapse/issues/8755)) Improved Documentation From e8d08537394a49f3e66e9cbea3627e3c25818a7d Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 13 Nov 2020 16:24:04 +0000 Subject: [PATCH 07/57] Generalise _maybe_store_room_on_invite (#8754) There's a handy function called maybe_store_room_on_invite which allows us to create an entry in the rooms table for a room and its version for which we aren't joined to yet, but we can reference when ingesting events about. This is currently used for invites where we receive some stripped state about the room and pass it down via /sync to the client, without us being in the room yet. There is a similar requirement for knocking, where we will eventually do the same thing, and need an entry in the rooms table as well. Thus, reusing this function works, however its name needs to be generalised a bit. Separated out from #6739. --- changelog.d/8754.misc | 1 + synapse/handlers/federation.py | 10 ++++++---- synapse/replication/http/federation.py | 10 +++++----- synapse/storage/databases/main/room.py | 10 ++++++---- 4 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 changelog.d/8754.misc diff --git a/changelog.d/8754.misc b/changelog.d/8754.misc new file mode 100644 index 0000000000..0436bb1be7 --- /dev/null +++ b/changelog.d/8754.misc @@ -0,0 +1 @@ +Generalise `RoomStore.maybe_store_room_on_invite` to handle other, non-invite membership events. \ No newline at end of file diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c386957706..69bc5ba44d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -67,7 +67,7 @@ from synapse.replication.http.devices import ReplicationUserDevicesResyncRestSer from synapse.replication.http.federation import ( ReplicationCleanRoomRestServlet, ReplicationFederationSendEventsRestServlet, - ReplicationStoreRoomOnInviteRestServlet, + ReplicationStoreRoomOnOutlierMembershipRestServlet, ) from synapse.state import StateResolutionStore from synapse.storage.databases.main.events_worker import EventRedactBehaviour @@ -152,12 +152,14 @@ class FederationHandler(BaseHandler): self._user_device_resync = ReplicationUserDevicesResyncRestServlet.make_client( hs ) - self._maybe_store_room_on_invite = ReplicationStoreRoomOnInviteRestServlet.make_client( + self._maybe_store_room_on_outlier_membership = ReplicationStoreRoomOnOutlierMembershipRestServlet.make_client( hs ) else: self._device_list_updater = hs.get_device_handler().device_list_updater - self._maybe_store_room_on_invite = self.store.maybe_store_room_on_invite + self._maybe_store_room_on_outlier_membership = ( + self.store.maybe_store_room_on_outlier_membership + ) # When joining a room we need to queue any events for that room up. # For each room, a list of (pdu, origin) tuples. @@ -1617,7 +1619,7 @@ class FederationHandler(BaseHandler): # keep a record of the room version, if we don't yet know it. # (this may get overwritten if we later get a different room version in a # join dance). - await self._maybe_store_room_on_invite( + await self._maybe_store_room_on_outlier_membership( room_id=event.room_id, room_version=room_version ) diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py index b4f4a68b5c..7a0dbb5b1a 100644 --- a/synapse/replication/http/federation.py +++ b/synapse/replication/http/federation.py @@ -254,20 +254,20 @@ class ReplicationCleanRoomRestServlet(ReplicationEndpoint): return 200, {} -class ReplicationStoreRoomOnInviteRestServlet(ReplicationEndpoint): +class ReplicationStoreRoomOnOutlierMembershipRestServlet(ReplicationEndpoint): """Called to clean up any data in DB for a given room, ready for the server to join the room. Request format: - POST /_synapse/replication/store_room_on_invite/:room_id/:txn_id + POST /_synapse/replication/store_room_on_outlier_membership/:room_id/:txn_id { "room_version": "1", } """ - NAME = "store_room_on_invite" + NAME = "store_room_on_outlier_membership" PATH_ARGS = ("room_id",) def __init__(self, hs): @@ -282,7 +282,7 @@ class ReplicationStoreRoomOnInviteRestServlet(ReplicationEndpoint): async def _handle_request(self, request, room_id): content = parse_json_object_from_request(request) room_version = KNOWN_ROOM_VERSIONS[content["room_version"]] - await self.store.maybe_store_room_on_invite(room_id, room_version) + await self.store.maybe_store_room_on_outlier_membership(room_id, room_version) return 200, {} @@ -291,4 +291,4 @@ def register_servlets(hs, http_server): ReplicationFederationSendEduRestServlet(hs).register(http_server) ReplicationGetQueryRestServlet(hs).register(http_server) ReplicationCleanRoomRestServlet(hs).register(http_server) - ReplicationStoreRoomOnInviteRestServlet(hs).register(http_server) + ReplicationStoreRoomOnOutlierMembershipRestServlet(hs).register(http_server) diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index dc0c4b5499..6b89db15c9 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1240,13 +1240,15 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore): logger.error("store_room with room_id=%s failed: %s", room_id, e) raise StoreError(500, "Problem creating room.") - async def maybe_store_room_on_invite(self, room_id: str, room_version: RoomVersion): + async def maybe_store_room_on_outlier_membership( + self, room_id: str, room_version: RoomVersion + ): """ - When we receive an invite over federation, store the version of the room if we - don't already know the room version. + When we receive an invite or any other event over federation that may relate to a room + we are not in, store the version of the room if we don't already know the room version. """ await self.db_pool.simple_upsert( - desc="maybe_store_room_on_invite", + desc="maybe_store_room_on_outlier_membership", table="rooms", keyvalues={"room_id": room_id}, values={}, From f1de4bb58b5b68b0fbb0033bd42fa30df0944281 Mon Sep 17 00:00:00 2001 From: Adrian Wannenmacher Date: Sun, 15 Nov 2020 00:09:36 +0100 Subject: [PATCH 08/57] Clarify the usecase for an msisdn delegate (#8734) Signed-off-by: Adrian Wannenmacher --- changelog.d/8734.doc | 1 + docs/sample_config.yaml | 5 +++-- synapse/config/registration.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changelog.d/8734.doc diff --git a/changelog.d/8734.doc b/changelog.d/8734.doc new file mode 100644 index 0000000000..3bff9021c7 --- /dev/null +++ b/changelog.d/8734.doc @@ -0,0 +1 @@ +Clarify the usecase for an msisdn delegate. Contributed by Adrian Wannenmacher. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index c0cd009230..e9e77ca94e 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1230,8 +1230,9 @@ account_validity: # email will be globally disabled. # # Additionally, if `msisdn` is not set, registration and password resets via msisdn -# will be disabled regardless. This is due to Synapse currently not supporting any -# method of sending SMS messages on its own. +# will be disabled regardless, and users will not be able to associate an msisdn +# identifier to their account. This is due to Synapse currently not supporting +# any method of sending SMS messages on its own. # # To enable using an identity server for operations regarding a particular third-party # identifier type, set the value to the URL of that identity server as shown in the diff --git a/synapse/config/registration.py b/synapse/config/registration.py index b0a77a2e43..cc5f75123c 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -347,8 +347,9 @@ class RegistrationConfig(Config): # email will be globally disabled. # # Additionally, if `msisdn` is not set, registration and password resets via msisdn - # will be disabled regardless. This is due to Synapse currently not supporting any - # method of sending SMS messages on its own. + # will be disabled regardless, and users will not be able to associate an msisdn + # identifier to their account. This is due to Synapse currently not supporting + # any method of sending SMS messages on its own. # # To enable using an identity server for operations regarding a particular third-party # identifier type, set the value to the URL of that identity server as shown in the From d3523e3e9727f0a81f88e9aa58a8f0fc2b3ee260 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 13 Nov 2020 22:34:08 +0000 Subject: [PATCH 09/57] pass a Site into RestHelper --- tests/rest/client/v1/utils.py | 11 ++++++----- tests/unittest.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index afaf9f7b85..dc789fbdaa 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -23,6 +23,7 @@ from typing import Any, Dict, Optional import attr from twisted.web.resource import Resource +from twisted.web.server import Site from synapse.api.constants import Membership @@ -36,7 +37,7 @@ class RestHelper: """ hs = attr.ib() - resource = attr.ib() + site = attr.ib(type=Site) auth_user_id = attr.ib() def create_room_as( @@ -54,7 +55,7 @@ class RestHelper: request, channel = make_request( self.hs.get_reactor(), "POST", path, json.dumps(content).encode("utf8") ) - render(request, self.resource, self.hs.get_reactor()) + render(request, self.site.resource, self.hs.get_reactor()) assert channel.result["code"] == b"%d" % expect_code, channel.result self.auth_user_id = temp_id @@ -128,7 +129,7 @@ class RestHelper: self.hs.get_reactor(), "PUT", path, json.dumps(data).encode("utf8") ) - render(request, self.resource, self.hs.get_reactor()) + render(request, self.site.resource, self.hs.get_reactor()) assert int(channel.result["code"]) == expect_code, ( "Expected: %d, got: %d, resp: %r" @@ -160,7 +161,7 @@ class RestHelper: request, channel = make_request( self.hs.get_reactor(), "PUT", path, json.dumps(content).encode("utf8") ) - render(request, self.resource, self.hs.get_reactor()) + render(request, self.site.resource, self.hs.get_reactor()) assert int(channel.result["code"]) == expect_code, ( "Expected: %d, got: %d, resp: %r" @@ -212,7 +213,7 @@ class RestHelper: request, channel = make_request(self.hs.get_reactor(), method, path, content) - render(request, self.resource, self.hs.get_reactor()) + render(request, self.site.resource, self.hs.get_reactor()) assert int(channel.result["code"]) == expect_code, ( "Expected: %d, got: %d, resp: %r" diff --git a/tests/unittest.py b/tests/unittest.py index e36ac89196..0a24c2f6b2 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -253,7 +253,7 @@ class HomeserverTestCase(TestCase): from tests.rest.client.v1.utils import RestHelper - self.helper = RestHelper(self.hs, self.resource, getattr(self, "user_id", None)) + self.helper = RestHelper(self.hs, self.site, getattr(self, "user_id", None)) if hasattr(self, "user_id"): if self.hijack_auth: From 9debe657a39a234d574e949ae8faf3f5ed027c09 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 13 Nov 2020 22:39:09 +0000 Subject: [PATCH 10/57] pass a Site into make_request --- tests/rest/client/v1/utils.py | 31 +++++++++++++++++++++------ tests/server.py | 16 +++++++++++++- tests/test_server.py | 40 +++++++++++++++++++++++------------ tests/unittest.py | 1 + 4 files changed, 68 insertions(+), 20 deletions(-) diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index dc789fbdaa..60e4b9b846 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -27,7 +27,7 @@ from twisted.web.server import Site from synapse.api.constants import Membership -from tests.server import make_request, render +from tests.server import FakeSite, make_request, render @attr.s @@ -53,7 +53,11 @@ class RestHelper: path = path + "?access_token=%s" % tok request, channel = make_request( - self.hs.get_reactor(), "POST", path, json.dumps(content).encode("utf8") + self.hs.get_reactor(), + self.site, + "POST", + path, + json.dumps(content).encode("utf8"), ) render(request, self.site.resource, self.hs.get_reactor()) @@ -126,7 +130,11 @@ class RestHelper: data.update(extra_data) request, channel = make_request( - self.hs.get_reactor(), "PUT", path, json.dumps(data).encode("utf8") + self.hs.get_reactor(), + self.site, + "PUT", + path, + json.dumps(data).encode("utf8"), ) render(request, self.site.resource, self.hs.get_reactor()) @@ -159,7 +167,11 @@ class RestHelper: path = path + "?access_token=%s" % tok request, channel = make_request( - self.hs.get_reactor(), "PUT", path, json.dumps(content).encode("utf8") + self.hs.get_reactor(), + self.site, + "PUT", + path, + json.dumps(content).encode("utf8"), ) render(request, self.site.resource, self.hs.get_reactor()) @@ -211,7 +223,9 @@ class RestHelper: if body is not None: content = json.dumps(body).encode("utf8") - request, channel = make_request(self.hs.get_reactor(), method, path, content) + request, channel = make_request( + self.hs.get_reactor(), self.site, method, path, content + ) render(request, self.site.resource, self.hs.get_reactor()) @@ -297,7 +311,12 @@ class RestHelper: image_length = len(image_data) path = "/_matrix/media/r0/upload?filename=%s" % (filename,) request, channel = make_request( - self.hs.get_reactor(), "POST", path, content=image_data, access_token=tok + self.hs.get_reactor(), + FakeSite(resource), + "POST", + path, + content=image_data, + access_token=tok, ) request.requestHeaders.addRawHeader( b"Content-Length", str(image_length).encode("UTF-8") diff --git a/tests/server.py b/tests/server.py index 3dd2cfc072..b9ccde4962 100644 --- a/tests/server.py +++ b/tests/server.py @@ -21,6 +21,7 @@ from twisted.python.failure import Failure from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactorClock from twisted.web.http import unquote from twisted.web.http_headers import Headers +from twisted.web.resource import IResource from twisted.web.server import Site from synapse.http.site import SynapseRequest @@ -128,9 +129,21 @@ class FakeSite: site_tag = "test" access_logger = logging.getLogger("synapse.access.http.fake") + def __init__(self, resource: IResource): + """ + + Args: + resource: the resource to be used for rendering all requests + """ + self._resource = resource + + def getResourceFor(self, request): + return self._resource + def make_request( reactor, + site: Site, method, path, content=b"", @@ -145,6 +158,8 @@ def make_request( content, and return the Request and the Channel underneath. Args: + site: The twisted Site to associate with the Channel + method (bytes/unicode): The HTTP request method ("verb"). path (bytes/unicode): The HTTP path, suitably URL encoded (e.g. escaped UTF-8 & spaces and such). @@ -181,7 +196,6 @@ def make_request( if isinstance(content, str): content = content.encode("utf8") - site = FakeSite() channel = FakeChannel(site, reactor) req = request(channel) diff --git a/tests/test_server.py b/tests/test_server.py index 655c918a15..300d13ac95 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -26,6 +26,7 @@ from synapse.util import Clock from tests import unittest from tests.server import ( + FakeSite, ThreadedMemoryReactorClock, make_request, render, @@ -62,7 +63,7 @@ class JsonResourceTests(unittest.TestCase): ) request, channel = make_request( - self.reactor, b"GET", b"/_matrix/foo/%E2%98%83?a=%E2%98%83" + self.reactor, FakeSite(res), b"GET", b"/_matrix/foo/%E2%98%83?a=%E2%98%83" ) render(request, res, self.reactor) @@ -83,7 +84,9 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" ) - request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo") + request, channel = make_request( + self.reactor, FakeSite(res), b"GET", b"/_matrix/foo" + ) render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"500") @@ -108,7 +111,9 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" ) - request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo") + request, channel = make_request( + self.reactor, FakeSite(res), b"GET", b"/_matrix/foo" + ) render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"500") @@ -127,7 +132,9 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" ) - request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo") + request, channel = make_request( + self.reactor, FakeSite(res), b"GET", b"/_matrix/foo" + ) render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"403") @@ -150,7 +157,9 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" ) - request, channel = make_request(self.reactor, b"GET", b"/_matrix/foobar") + request, channel = make_request( + self.reactor, FakeSite(res), b"GET", b"/_matrix/foobar" + ) render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"400") @@ -173,7 +182,9 @@ class JsonResourceTests(unittest.TestCase): ) # The path was registered as GET, but this is a HEAD request. - request, channel = make_request(self.reactor, b"HEAD", b"/_matrix/foo") + request, channel = make_request( + self.reactor, FakeSite(res), b"HEAD", b"/_matrix/foo" + ) render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"200") @@ -196,9 +207,6 @@ class OptionsResourceTests(unittest.TestCase): def _make_request(self, method, path): """Create a request from the method/path and return a channel with the response.""" - request, channel = make_request(self.reactor, method, path, shorthand=False) - request.prepath = [] # This doesn't get set properly by make_request. - # Create a site and query for the resource. site = SynapseSite( "test", @@ -207,6 +215,12 @@ class OptionsResourceTests(unittest.TestCase): self.resource, "1.0", ) + + request, channel = make_request( + self.reactor, site, method, path, shorthand=False + ) + request.prepath = [] # This doesn't get set properly by make_request. + request.site = site resource = site.getResourceFor(request) @@ -284,7 +298,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): res = WrapHtmlRequestHandlerTests.TestResource() res.callback = callback - request, channel = make_request(self.reactor, b"GET", b"/path") + request, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"200") @@ -303,7 +317,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): res = WrapHtmlRequestHandlerTests.TestResource() res.callback = callback - request, channel = make_request(self.reactor, b"GET", b"/path") + request, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"301") @@ -325,7 +339,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): res = WrapHtmlRequestHandlerTests.TestResource() res.callback = callback - request, channel = make_request(self.reactor, b"GET", b"/path") + request, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"304") @@ -345,7 +359,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): res = WrapHtmlRequestHandlerTests.TestResource() res.callback = callback - request, channel = make_request(self.reactor, b"HEAD", b"/path") + request, channel = make_request(self.reactor, FakeSite(res), b"HEAD", b"/path") render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"200") diff --git a/tests/unittest.py b/tests/unittest.py index 0a24c2f6b2..8c7979a7c0 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -434,6 +434,7 @@ class HomeserverTestCase(TestCase): return make_request( self.reactor, + self.site, method, path, content, From 70c0d47989b7794766ea957369c77d99664429c5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 13 Nov 2020 23:48:25 +0000 Subject: [PATCH 11/57] fix dict handling for make_request() --- tests/server.py | 2 ++ tests/unittest.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/server.py b/tests/server.py index b9ccde4962..a74fb3fc67 100644 --- a/tests/server.py +++ b/tests/server.py @@ -193,6 +193,8 @@ def make_request( if not path.startswith(b"/"): path = b"/" + path + if isinstance(content, dict): + content = json.dumps(content).encode("utf8") if isinstance(content, str): content = content.encode("utf8") diff --git a/tests/unittest.py b/tests/unittest.py index 8c7979a7c0..3e656b7b12 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -429,9 +429,6 @@ class HomeserverTestCase(TestCase): Returns: Tuple[synapse.http.site.SynapseRequest, channel] """ - if isinstance(content, dict): - content = json.dumps(content).encode("utf8") - return make_request( self.reactor, self.site, From cfd895a22e7563b6b22141d14d9c8aebf177df44 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 13 Nov 2020 23:11:43 +0000 Subject: [PATCH 12/57] use global make_request() directly where we have a custom Resource Where we want to render a request against a specific Resource, call the global make_request() function rather than the one in HomeserverTestCase, allowing us to pass in an appropriate `Site`. --- tests/app/test_frontend_proxy.py | 13 +++--- tests/app/test_openid_listener.py | 17 ++++---- tests/http/test_additional_resource.py | 13 +++--- tests/replication/test_client_reader_shard.py | 29 +++++++++---- tests/replication/test_multi_media_repo.py | 10 +++-- .../test_sharded_event_persister.py | 42 ++++++++++++++----- tests/rest/admin/test_admin.py | 18 ++++++-- tests/rest/admin/test_media.py | 13 ++++-- tests/rest/client/test_consent.py | 28 +++++++++---- tests/rest/client/v2_alpha/test_account.py | 14 ++++++- tests/rest/media/v1/test_media_storage.py | 17 ++++++-- 11 files changed, 154 insertions(+), 60 deletions(-) diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py index 4a301b84e1..0bac7995e8 100644 --- a/tests/app/test_frontend_proxy.py +++ b/tests/app/test_frontend_proxy.py @@ -15,6 +15,7 @@ from synapse.app.generic_worker import GenericWorkerServer +from tests.server import make_request, render from tests.unittest import HomeserverTestCase @@ -55,10 +56,10 @@ class FrontendProxyTests(HomeserverTestCase): # Grab the resource from the site that was told to listen self.assertEqual(len(self.reactor.tcpServers), 1) site = self.reactor.tcpServers[0][1] - self.resource = site.resource.children[b"_matrix"].children[b"client"] + resource = site.resource.children[b"_matrix"].children[b"client"] - request, channel = self.make_request("PUT", "presence/a/status") - self.render(request) + request, channel = make_request(self.reactor, site, "PUT", "presence/a/status") + render(request, resource, self.reactor) # 400 + unrecognised, because nothing is registered self.assertEqual(channel.code, 400) @@ -77,10 +78,10 @@ class FrontendProxyTests(HomeserverTestCase): # Grab the resource from the site that was told to listen self.assertEqual(len(self.reactor.tcpServers), 1) site = self.reactor.tcpServers[0][1] - self.resource = site.resource.children[b"_matrix"].children[b"client"] + resource = site.resource.children[b"_matrix"].children[b"client"] - request, channel = self.make_request("PUT", "presence/a/status") - self.render(request) + request, channel = make_request(self.reactor, site, "PUT", "presence/a/status") + render(request, resource, self.reactor) # 401, because the stub servlet still checks authentication self.assertEqual(channel.code, 401) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index c2b10d2c70..1292145890 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -20,6 +20,7 @@ from synapse.app.generic_worker import GenericWorkerServer from synapse.app.homeserver import SynapseHomeServer from synapse.config.server import parse_listener_def +from tests.server import make_request, render from tests.unittest import HomeserverTestCase @@ -66,16 +67,16 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): # Grab the resource from the site that was told to listen site = self.reactor.tcpServers[0][1] try: - self.resource = site.resource.children[b"_matrix"].children[b"federation"] + resource = site.resource.children[b"_matrix"].children[b"federation"] except KeyError: if expectation == "no_resource": return raise - request, channel = self.make_request( - "GET", "/_matrix/federation/v1/openid/userinfo" + request, channel = make_request( + self.reactor, site, "GET", "/_matrix/federation/v1/openid/userinfo" ) - self.render(request) + render(request, resource, self.reactor) self.assertEqual(channel.code, 401) @@ -115,15 +116,15 @@ class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): # Grab the resource from the site that was told to listen site = self.reactor.tcpServers[0][1] try: - self.resource = site.resource.children[b"_matrix"].children[b"federation"] + resource = site.resource.children[b"_matrix"].children[b"federation"] except KeyError: if expectation == "no_resource": return raise - request, channel = self.make_request( - "GET", "/_matrix/federation/v1/openid/userinfo" + request, channel = make_request( + self.reactor, site, "GET", "/_matrix/federation/v1/openid/userinfo" ) - self.render(request) + render(request, resource, self.reactor) self.assertEqual(channel.code, 401) diff --git a/tests/http/test_additional_resource.py b/tests/http/test_additional_resource.py index 62d36c2906..e835512a41 100644 --- a/tests/http/test_additional_resource.py +++ b/tests/http/test_additional_resource.py @@ -17,6 +17,7 @@ from synapse.http.additional_resource import AdditionalResource from synapse.http.server import respond_with_json +from tests.server import FakeSite, make_request, render from tests.unittest import HomeserverTestCase @@ -43,20 +44,20 @@ class AdditionalResourceTests(HomeserverTestCase): def test_async(self): handler = _AsyncTestCustomEndpoint({}, None).handle_request - self.resource = AdditionalResource(self.hs, handler) + resource = AdditionalResource(self.hs, handler) - request, channel = self.make_request("GET", "/") - self.render(request) + request, channel = make_request(self.reactor, FakeSite(resource), "GET", "/") + render(request, resource, self.reactor) self.assertEqual(request.code, 200) self.assertEqual(channel.json_body, {"some_key": "some_value_async"}) def test_sync(self): handler = _SyncTestCustomEndpoint({}, None).handle_request - self.resource = AdditionalResource(self.hs, handler) + resource = AdditionalResource(self.hs, handler) - request, channel = self.make_request("GET", "/") - self.render(request) + request, channel = make_request(self.reactor, FakeSite(resource), "GET", "/") + render(request, resource, self.reactor) self.assertEqual(request.code, 200) self.assertEqual(channel.json_body, {"some_key": "some_value_sync"}) diff --git a/tests/replication/test_client_reader_shard.py b/tests/replication/test_client_reader_shard.py index 86c03fd89c..90172bd377 100644 --- a/tests/replication/test_client_reader_shard.py +++ b/tests/replication/test_client_reader_shard.py @@ -20,7 +20,7 @@ from synapse.rest.client.v2_alpha import register from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.rest.client.v2_alpha.test_auth import DummyRecaptchaChecker -from tests.server import FakeChannel +from tests.server import FakeChannel, make_request logger = logging.getLogger(__name__) @@ -46,8 +46,11 @@ class ClientReaderTestCase(BaseMultiWorkerStreamTestCase): """Test that registration works when using a single client reader worker. """ worker_hs = self.make_worker_hs("synapse.app.client_reader") + site = self._hs_to_site[worker_hs] - request_1, channel_1 = self.make_request( + request_1, channel_1 = make_request( + self.reactor, + site, "POST", "register", {"username": "user", "type": "m.login.password", "password": "bar"}, @@ -59,8 +62,12 @@ class ClientReaderTestCase(BaseMultiWorkerStreamTestCase): session = channel_1.json_body["session"] # also complete the dummy auth - request_2, channel_2 = self.make_request( - "POST", "register", {"auth": {"session": session, "type": "m.login.dummy"}} + request_2, channel_2 = make_request( + self.reactor, + site, + "POST", + "register", + {"auth": {"session": session, "type": "m.login.dummy"}}, ) # type: SynapseRequest, FakeChannel self.render_on_worker(worker_hs, request_2) self.assertEqual(request_2.code, 200) @@ -74,7 +81,10 @@ class ClientReaderTestCase(BaseMultiWorkerStreamTestCase): worker_hs_1 = self.make_worker_hs("synapse.app.client_reader") worker_hs_2 = self.make_worker_hs("synapse.app.client_reader") - request_1, channel_1 = self.make_request( + site_1 = self._hs_to_site[worker_hs_1] + request_1, channel_1 = make_request( + self.reactor, + site_1, "POST", "register", {"username": "user", "type": "m.login.password", "password": "bar"}, @@ -86,8 +96,13 @@ class ClientReaderTestCase(BaseMultiWorkerStreamTestCase): session = channel_1.json_body["session"] # also complete the dummy auth - request_2, channel_2 = self.make_request( - "POST", "register", {"auth": {"session": session, "type": "m.login.dummy"}} + site_2 = self._hs_to_site[worker_hs_2] + request_2, channel_2 = make_request( + self.reactor, + site_2, + "POST", + "register", + {"auth": {"session": session, "type": "m.login.dummy"}}, ) # type: SynapseRequest, FakeChannel self.render_on_worker(worker_hs_2, request_2) self.assertEqual(request_2.code, 200) diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py index 77c261dbf7..a9ac4aeec1 100644 --- a/tests/replication/test_multi_media_repo.py +++ b/tests/replication/test_multi_media_repo.py @@ -28,7 +28,7 @@ from synapse.server import HomeServer from tests.http import TestServerTLSConnectionFactory, get_test_ca_cert_file from tests.replication._base import BaseMultiWorkerStreamTestCase -from tests.server import FakeChannel, FakeTransport +from tests.server import FakeChannel, FakeSite, FakeTransport, make_request logger = logging.getLogger(__name__) @@ -67,14 +67,16 @@ class MediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): The channel for the *client* request and the *outbound* request for the media which the caller should respond to. """ - - request, channel = self.make_request( + resource = hs.get_media_repository_resource().children[b"download"] + request, channel = make_request( + self.reactor, + FakeSite(resource), "GET", "/{}/{}".format(target, media_id), shorthand=False, access_token=self.access_token, ) - request.render(hs.get_media_repository_resource().children[b"download"]) + request.render(resource) self.pump() clients = self.reactor.tcpClients diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py index 82cf033d4e..2820dd622f 100644 --- a/tests/replication/test_sharded_event_persister.py +++ b/tests/replication/test_sharded_event_persister.py @@ -22,6 +22,7 @@ from synapse.rest.client.v1 import login, room from synapse.rest.client.v2_alpha import sync from tests.replication._base import BaseMultiWorkerStreamTestCase +from tests.server import make_request from tests.utils import USE_POSTGRES_FOR_TESTS logger = logging.getLogger(__name__) @@ -148,6 +149,7 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): sync_hs = self.make_worker_hs( "synapse.app.generic_worker", {"worker_name": "sync"}, ) + sync_hs_site = self._hs_to_site[sync_hs] # Specially selected room IDs that get persisted on different workers. room_id1 = "!foo:test" @@ -178,7 +180,9 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): ) # Do an initial sync so that we're up to date. - request, channel = self.make_request("GET", "/sync", access_token=access_token) + request, channel = make_request( + self.reactor, sync_hs_site, "GET", "/sync", access_token=access_token + ) self.render_on_worker(sync_hs, request) next_batch = channel.json_body["next_batch"] @@ -203,8 +207,12 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): # Check that syncing still gets the new event, despite the gap in the # stream IDs. - request, channel = self.make_request( - "GET", "/sync?since={}".format(next_batch), access_token=access_token + request, channel = make_request( + self.reactor, + sync_hs_site, + "GET", + "/sync?since={}".format(next_batch), + access_token=access_token, ) self.render_on_worker(sync_hs, request) @@ -230,7 +238,9 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): response = self.helper.send(room_id2, body="Hi!", tok=self.other_access_token) first_event_in_room2 = response["event_id"] - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + sync_hs_site, "GET", "/sync?since={}".format(vector_clock_token), access_token=access_token, @@ -254,8 +264,12 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): self.helper.send(room_id1, body="Hi again!", tok=self.other_access_token) self.helper.send(room_id2, body="Hi again!", tok=self.other_access_token) - request, channel = self.make_request( - "GET", "/sync?since={}".format(next_batch), access_token=access_token + request, channel = make_request( + self.reactor, + sync_hs_site, + "GET", + "/sync?since={}".format(next_batch), + access_token=access_token, ) self.render_on_worker(sync_hs, request) @@ -269,7 +283,9 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): # Paginating back in the first room should not produce any results, as # no events have happened in it. This tests that we are correctly # filtering results based on the vector clock portion. - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + sync_hs_site, "GET", "/rooms/{}/messages?from={}&to={}&dir=b".format( room_id1, prev_batch1, vector_clock_token @@ -281,7 +297,9 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): # Paginating back on the second room should produce the first event # again. This tests that pagination isn't completely broken. - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + sync_hs_site, "GET", "/rooms/{}/messages?from={}&to={}&dir=b".format( room_id2, prev_batch2, vector_clock_token @@ -295,7 +313,9 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): ) # Paginating forwards should give the same results - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + sync_hs_site, "GET", "/rooms/{}/messages?from={}&to={}&dir=f".format( room_id1, vector_clock_token, prev_batch1 @@ -305,7 +325,9 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): self.render_on_worker(sync_hs, request) self.assertListEqual([], channel.json_body["chunk"]) - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + sync_hs_site, "GET", "/rooms/{}/messages?from={}&to={}&dir=f".format( room_id2, vector_clock_token, prev_batch2, diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 0f1144fe1e..64b6016729 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -30,6 +30,7 @@ from synapse.rest.client.v1 import login, room from synapse.rest.client.v2_alpha import groups from tests import unittest +from tests.server import FakeSite, make_request class VersionTestCase(unittest.HomeserverTestCase): @@ -222,8 +223,13 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): def _ensure_quarantined(self, admin_user_tok, server_and_media_id): """Ensure a piece of media is quarantined when trying to access it.""" - request, channel = self.make_request( - "GET", server_and_media_id, shorthand=False, access_token=admin_user_tok, + request, channel = make_request( + self.reactor, + FakeSite(self.download_resource), + "GET", + server_and_media_id, + shorthand=False, + access_token=admin_user_tok, ) request.render(self.download_resource) self.pump(1.0) @@ -287,7 +293,9 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): server_name, media_id = server_name_and_media_id.split("/") # Attempt to access the media - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + FakeSite(self.download_resource), "GET", server_name_and_media_id, shorthand=False, @@ -462,7 +470,9 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): self._ensure_quarantined(admin_user_tok, server_and_media_id_1) # Attempt to access each piece of media - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + FakeSite(self.download_resource), "GET", server_and_media_id_2, shorthand=False, diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 721fa1ed51..36e07f1b36 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -23,6 +23,7 @@ from synapse.rest.client.v1 import login, profile, room from synapse.rest.media.v1.filepath import MediaFilePaths from tests import unittest +from tests.server import FakeSite, make_request class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): @@ -124,7 +125,9 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): self.assertEqual(server_name, self.server_name) # Attempt to access media - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + FakeSite(download_resource), "GET", server_and_media_id, shorthand=False, @@ -161,7 +164,9 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): ) # Attempt to access media - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + FakeSite(download_resource), "GET", server_and_media_id, shorthand=False, @@ -535,7 +540,9 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): media_id = server_and_media_id.split("/")[1] local_path = self.filepaths.local_media_filepath(media_id) - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + FakeSite(download_resource), "GET", server_and_media_id, shorthand=False, diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py index 6803b372ac..2931859f25 100644 --- a/tests/rest/client/test_consent.py +++ b/tests/rest/client/test_consent.py @@ -21,7 +21,7 @@ from synapse.rest.client.v1 import login, room from synapse.rest.consent import consent_resource from tests import unittest -from tests.server import render +from tests.server import FakeSite, make_request, render class ConsentResourceTestCase(unittest.HomeserverTestCase): @@ -61,7 +61,9 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): def test_render_public_consent(self): """You can observe the terms form without specifying a user""" resource = consent_resource.ConsentResource(self.hs) - request, channel = self.make_request("GET", "/consent?v=1", shorthand=False) + request, channel = make_request( + self.reactor, FakeSite(resource), "GET", "/consent?v=1", shorthand=False + ) render(request, resource, self.reactor) self.assertEqual(channel.code, 200) @@ -81,8 +83,13 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): uri_builder.build_user_consent_uri(user_id).replace("_matrix/", "") + "&u=user" ) - request, channel = self.make_request( - "GET", consent_uri, access_token=access_token, shorthand=False + request, channel = make_request( + self.reactor, + FakeSite(resource), + "GET", + consent_uri, + access_token=access_token, + shorthand=False, ) render(request, resource, self.reactor) self.assertEqual(channel.code, 200) @@ -92,7 +99,9 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): self.assertEqual(consented, "False") # POST to the consent page, saying we've agreed - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + FakeSite(resource), "POST", consent_uri + "&v=" + version, access_token=access_token, @@ -103,8 +112,13 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): # Fetch the consent page, to get the consent version -- it should have # changed - request, channel = self.make_request( - "GET", consent_uri, access_token=access_token, shorthand=False + request, channel = make_request( + self.reactor, + FakeSite(resource), + "GET", + consent_uri, + access_token=access_token, + shorthand=False, ) render(request, resource, self.reactor) self.assertEqual(channel.code, 200) diff --git a/tests/rest/client/v2_alpha/test_account.py b/tests/rest/client/v2_alpha/test_account.py index 66ac4dbe85..94a627b0a6 100644 --- a/tests/rest/client/v2_alpha/test_account.py +++ b/tests/rest/client/v2_alpha/test_account.py @@ -31,6 +31,7 @@ from synapse.rest.client.v2_alpha import account, register from synapse.rest.synapse.client.password_reset import PasswordResetSubmitTokenResource from tests import unittest +from tests.server import FakeSite, make_request from tests.unittest import override_config @@ -255,9 +256,16 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): path = link.replace("https://example.com", "") # Load the password reset confirmation page - request, channel = self.make_request("GET", path, shorthand=False) + request, channel = make_request( + self.reactor, + FakeSite(self.submit_token_resource), + "GET", + path, + shorthand=False, + ) request.render(self.submit_token_resource) self.pump() + self.assertEquals(200, channel.code, channel.result) # Now POST to the same endpoint, mimicking the same behaviour as clicking the @@ -271,7 +279,9 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): form_args.append(arg) # Confirm the password reset - request, channel = self.make_request( + request, channel = make_request( + self.reactor, + FakeSite(self.submit_token_resource), "POST", path, content=urlencode(form_args).encode("utf8"), diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py index 5f897d49cf..0fd31a0096 100644 --- a/tests/rest/media/v1/test_media_storage.py +++ b/tests/rest/media/v1/test_media_storage.py @@ -36,6 +36,7 @@ from synapse.rest.media.v1.media_storage import MediaStorage from synapse.rest.media.v1.storage_provider import FileStorageProviderBackend from tests import unittest +from tests.server import FakeSite, make_request class MediaStorageTests(unittest.HomeserverTestCase): @@ -227,7 +228,13 @@ class MediaRepoTests(unittest.HomeserverTestCase): def _req(self, content_disposition): - request, channel = self.make_request("GET", self.media_id, shorthand=False) + request, channel = make_request( + self.reactor, + FakeSite(self.download_resource), + "GET", + self.media_id, + shorthand=False, + ) request.render(self.download_resource) self.pump() @@ -317,8 +324,12 @@ class MediaRepoTests(unittest.HomeserverTestCase): def _test_thumbnail(self, method, expected_body, expected_found): params = "?width=32&height=32&method=" + method - request, channel = self.make_request( - "GET", self.media_id + params, shorthand=False + request, channel = make_request( + self.reactor, + FakeSite(self.thumbnail_resource), + "GET", + self.media_id + params, + shorthand=False, ) request.render(self.thumbnail_resource) self.pump() From 0d33c535348b1b94fd700c40e0f219659cb9a1ea Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 14 Nov 2020 00:35:34 +0000 Subject: [PATCH 13/57] changelog --- changelog.d/8757.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8757.misc diff --git a/changelog.d/8757.misc b/changelog.d/8757.misc new file mode 100644 index 0000000000..54502e9b90 --- /dev/null +++ b/changelog.d/8757.misc @@ -0,0 +1 @@ +Refactor test utilities for injecting HTTP requests. From ebc405446e6615d6187a2e29cb33f27dd5bd0841 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 16 Nov 2020 14:45:22 +0000 Subject: [PATCH 14/57] Add a `custom_headers` param to `make_request` (#8760) Some tests want to set some custom HTTP request headers, so provide a way to do that before calling requestReceived(). --- changelog.d/8760.misc | 1 + tests/rest/client/v1/utils.py | 10 ++++++---- tests/server.py | 11 ++++++++++- tests/storage/test_client_ips.py | 13 +++++++------ 4 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 changelog.d/8760.misc diff --git a/changelog.d/8760.misc b/changelog.d/8760.misc new file mode 100644 index 0000000000..54502e9b90 --- /dev/null +++ b/changelog.d/8760.misc @@ -0,0 +1 @@ +Refactor test utilities for injecting HTTP requests. diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index afaf9f7b85..1b2d0497a6 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -296,10 +296,12 @@ class RestHelper: image_length = len(image_data) path = "/_matrix/media/r0/upload?filename=%s" % (filename,) request, channel = make_request( - self.hs.get_reactor(), "POST", path, content=image_data, access_token=tok - ) - request.requestHeaders.addRawHeader( - b"Content-Length", str(image_length).encode("UTF-8") + self.hs.get_reactor(), + "POST", + path, + content=image_data, + access_token=tok, + custom_headers=[(b"Content-Length", str(image_length))], ) request.render(resource) self.hs.get_reactor().pump([100]) diff --git a/tests/server.py b/tests/server.py index 3dd2cfc072..ef03109a6c 100644 --- a/tests/server.py +++ b/tests/server.py @@ -2,7 +2,7 @@ import json import logging from collections import deque from io import SEEK_END, BytesIO -from typing import Callable +from typing import Callable, Iterable, Optional, Tuple, Union import attr from typing_extensions import Deque @@ -139,6 +139,9 @@ def make_request( shorthand=True, federation_auth_origin=None, content_is_form=False, + custom_headers: Optional[ + Iterable[Tuple[Union[bytes, str], Union[bytes, str]]] + ] = None, ): """ Make a web request using the given method and path, feed it the @@ -157,6 +160,8 @@ def make_request( content_is_form: Whether the content is URL encoded form data. Adds the 'Content-Type': 'application/x-www-form-urlencoded' header. + custom_headers: (name, value) pairs to add as request headers + Returns: Tuple[synapse.http.site.SynapseRequest, channel] """ @@ -211,6 +216,10 @@ def make_request( # Assume the body is JSON req.requestHeaders.addRawHeader(b"Content-Type", b"application/json") + if custom_headers: + for k, v in custom_headers: + req.requestHeaders.addRawHeader(k, v) + req.requestReceived(method, path, b"1.1") return req, channel diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index e96ca1c8ca..efca43ec78 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -21,6 +21,7 @@ from synapse.http.site import XForwardedForRequest from synapse.rest.client.v1 import login from tests import unittest +from tests.server import make_request from tests.test_utils import make_awaitable from tests.unittest import override_config @@ -408,17 +409,17 @@ class ClientIpAuthTestCase(unittest.HomeserverTestCase): # Advance to a known time self.reactor.advance(123456 - self.reactor.seconds()) - request, channel = self.make_request( + headers1 = {b"User-Agent": b"Mozzila pizza"} + headers1.update(headers) + + request, channel = make_request( + self.reactor, "GET", "/_matrix/client/r0/admin/users/" + self.user_id, access_token=access_token, + custom_headers=headers1.items(), **make_request_args, ) - request.requestHeaders.addRawHeader(b"User-Agent", b"Mozzila pizza") - - # Add the optional headers - for h, v in headers.items(): - request.requestHeaders.addRawHeader(h, v) self.render(request) # Advance so the save loop occurs From 791d7cd6f065e166576538b9cba3d90febf83ea4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 16 Nov 2020 14:45:52 +0000 Subject: [PATCH 15/57] Rename `create_test_json_resource` to `create_test_resource` (#8759) The root resource isn't necessarily a JsonResource, so rename this method accordingly, and update a couple of test classes to use the method rather than directly manipulating self.resource. --- changelog.d/8759.misc | 1 + tests/replication/_base.py | 4 ++-- tests/rest/admin/test_admin.py | 2 +- tests/rest/key/v2/test_remote_key_resource.py | 2 +- tests/rest/test_health.py | 6 ++---- tests/rest/test_well_known.py | 6 ++---- tests/unittest.py | 18 +++++++----------- 7 files changed, 16 insertions(+), 23 deletions(-) create mode 100644 changelog.d/8759.misc diff --git a/changelog.d/8759.misc b/changelog.d/8759.misc new file mode 100644 index 0000000000..54502e9b90 --- /dev/null +++ b/changelog.d/8759.misc @@ -0,0 +1 @@ +Refactor test utilities for injecting HTTP requests. diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 5c633ac6df..bc56b13dcd 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -240,8 +240,8 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase): lambda: self._handle_http_replication_attempt(self.hs, 8765), ) - def create_test_json_resource(self): - """Overrides `HomeserverTestCase.create_test_json_resource`. + def create_test_resource(self): + """Overrides `HomeserverTestCase.create_test_resource`. """ # We override this so that it automatically registers all the HTTP # replication servlets, without having to explicitly do that in all diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 0f1144fe1e..6804f9337f 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -35,7 +35,7 @@ from tests import unittest class VersionTestCase(unittest.HomeserverTestCase): url = "/_synapse/admin/v1/server_version" - def create_test_json_resource(self): + def create_test_resource(self): resource = JsonResource(self.hs) VersionServlet(self.hs).register(resource) return resource diff --git a/tests/rest/key/v2/test_remote_key_resource.py b/tests/rest/key/v2/test_remote_key_resource.py index 6850c666be..6671cbd32d 100644 --- a/tests/rest/key/v2/test_remote_key_resource.py +++ b/tests/rest/key/v2/test_remote_key_resource.py @@ -41,7 +41,7 @@ class BaseRemoteKeyResourceTestCase(unittest.HomeserverTestCase): self.http_client = Mock() return self.setup_test_homeserver(http_client=self.http_client) - def create_test_json_resource(self): + def create_test_resource(self): return create_resource_tree( {"/_matrix/key/v2": KeyApiV2Resource(self.hs)}, root_resource=NoResource() ) diff --git a/tests/rest/test_health.py b/tests/rest/test_health.py index 2d021f6565..f4d06e2200 100644 --- a/tests/rest/test_health.py +++ b/tests/rest/test_health.py @@ -20,11 +20,9 @@ from tests import unittest class HealthCheckTests(unittest.HomeserverTestCase): - def setUp(self): - super().setUp() - + def create_test_resource(self): # replace the JsonResource with a HealthResource. - self.resource = HealthResource() + return HealthResource() def test_health(self): request, channel = self.make_request("GET", "/health", shorthand=False) diff --git a/tests/rest/test_well_known.py b/tests/rest/test_well_known.py index dcd65c2a50..a3746e7130 100644 --- a/tests/rest/test_well_known.py +++ b/tests/rest/test_well_known.py @@ -20,11 +20,9 @@ from tests import unittest class WellKnownTests(unittest.HomeserverTestCase): - def setUp(self): - super().setUp() - + def create_test_resource(self): # replace the JsonResource with a WellKnownResource - self.resource = WellKnownResource(self.hs) + return WellKnownResource(self.hs) def test_well_known(self): self.hs.config.public_baseurl = "https://tesths" diff --git a/tests/unittest.py b/tests/unittest.py index e36ac89196..c630760e51 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -30,6 +30,7 @@ from twisted.internet.defer import Deferred, ensureDeferred, succeed from twisted.python.failure import Failure from twisted.python.threadpool import ThreadPool from twisted.trial import unittest +from twisted.web.resource import Resource from synapse.api.constants import EventTypes, Membership from synapse.config.homeserver import HomeServerConfig @@ -239,10 +240,8 @@ class HomeserverTestCase(TestCase): if not isinstance(self.hs, HomeServer): raise Exception("A homeserver wasn't returned, but %r" % (self.hs,)) - # Register the resources - self.resource = self.create_test_json_resource() - - # create a site to wrap the resource. + # create the root resource, and a site to wrap it. + self.resource = self.create_test_resource() self.site = SynapseSite( logger_name="synapse.access.http.fake", site_tag=self.hs.config.server.server_name, @@ -323,15 +322,12 @@ class HomeserverTestCase(TestCase): hs = self.setup_test_homeserver() return hs - def create_test_json_resource(self): + def create_test_resource(self) -> Resource: """ - Create a test JsonResource, with the relevant servlets registerd to it + Create a the root resource for the test server. - The default implementation calls each function in `servlets` to do the - registration. - - Returns: - JsonResource: + The default implementation creates a JsonResource and calls each function in + `servlets` to register servletes against it """ resource = JsonResource(self.hs) From 4f76eef0e8190a1f5b9985853fe645275c9a797c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 16 Nov 2020 15:37:36 +0000 Subject: [PATCH 16/57] Generalise _locally_reject_invite (#8751) `_locally_reject_invite` generates an out-of-band membership event which can be passed to clients, but not other homeservers. This is used when we fail to reject an invite over federation. If this happens, we instead just generate a leave event locally and send it down /sync, allowing clients to reject invites even if we can't reach the remote homeserver. A similar flow needs to be put in place for rescinding knocks. If we're unable to contact any remote server from the room we've tried to knock on, we'd still like to generate and store the leave event locally. Hence the need to reuse, and thus generalise, this method. Separated from #6739. --- changelog.d/8751.misc | 1 + synapse/handlers/room_member.py | 36 +++++++++++++++++---------------- 2 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 changelog.d/8751.misc diff --git a/changelog.d/8751.misc b/changelog.d/8751.misc new file mode 100644 index 0000000000..204c280c0e --- /dev/null +++ b/changelog.d/8751.misc @@ -0,0 +1 @@ +Generalise `RoomMemberHandler._locally_reject_invite` to apply to more flows than just invite. \ No newline at end of file diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 7cd858b7db..fd85e08973 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1104,32 +1104,34 @@ class RoomMemberMasterHandler(RoomMemberHandler): # logger.warning("Failed to reject invite: %s", e) - return await self._locally_reject_invite( + return await self._generate_local_out_of_band_leave( invite_event, txn_id, requester, content ) - async def _locally_reject_invite( + async def _generate_local_out_of_band_leave( self, - invite_event: EventBase, + previous_membership_event: EventBase, txn_id: Optional[str], requester: Requester, content: JsonDict, ) -> Tuple[str, int]: - """Generate a local invite rejection + """Generate a local leave event for a room - This is called after we fail to reject an invite via a remote server. It - generates an out-of-band membership event locally. + This can be called after we e.g fail to reject an invite via a remote server. + It generates an out-of-band membership event locally. Args: - invite_event: the invite to be rejected + previous_membership_event: the previous membership event for this user txn_id: optional transaction ID supplied by the client - requester: user making the rejection request, according to the access token - content: additional content to include in the rejection event. + requester: user making the request, according to the access token + content: additional content to include in the leave event. Normally an empty dict. - """ - room_id = invite_event.room_id - target_user = invite_event.state_key + Returns: + A tuple containing (event_id, stream_id of the leave event) + """ + room_id = previous_membership_event.room_id + target_user = previous_membership_event.state_key content["membership"] = Membership.LEAVE @@ -1141,12 +1143,12 @@ class RoomMemberMasterHandler(RoomMemberHandler): "state_key": target_user, } - # the auth events for the new event are the same as that of the invite, plus - # the invite itself. + # the auth events for the new event are the same as that of the previous event, plus + # the event itself. # - # the prev_events are just the invite. - prev_event_ids = [invite_event.event_id] - auth_event_ids = invite_event.auth_event_ids() + prev_event_ids + # the prev_events consist solely of the previous membership event. + prev_event_ids = [previous_membership_event.event_id] + auth_event_ids = previous_membership_event.auth_event_ids() + prev_event_ids event, context = await self.event_creation_handler.create_event( requester, From c3e3552ec4f677ff51e784f5ac28823eea989ec4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 16 Nov 2020 15:51:47 +0000 Subject: [PATCH 17/57] fixup test --- tests/storage/test_client_ips.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index efca43ec78..583addb5b5 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -414,6 +414,7 @@ class ClientIpAuthTestCase(unittest.HomeserverTestCase): request, channel = make_request( self.reactor, + self.site, "GET", "/_matrix/client/r0/admin/users/" + self.user_id, access_token=access_token, From f125895475aeee9447f3988ecbd8bfd1836545bf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 16 Nov 2020 18:21:47 +0000 Subject: [PATCH 18/57] Move `wait_until_result` into `FakeChannel` (#8758) FakeChannel has everything we need, and this more accurately models the real flow. --- changelog.d/8758.misc | 1 + tests/rest/key/v2/test_remote_key_resource.py | 6 +-- tests/server.py | 42 +++++++++---------- 3 files changed, 24 insertions(+), 25 deletions(-) create mode 100644 changelog.d/8758.misc diff --git a/changelog.d/8758.misc b/changelog.d/8758.misc new file mode 100644 index 0000000000..54502e9b90 --- /dev/null +++ b/changelog.d/8758.misc @@ -0,0 +1 @@ +Refactor test utilities for injecting HTTP requests. diff --git a/tests/rest/key/v2/test_remote_key_resource.py b/tests/rest/key/v2/test_remote_key_resource.py index 6671cbd32d..fbcf8d5b86 100644 --- a/tests/rest/key/v2/test_remote_key_resource.py +++ b/tests/rest/key/v2/test_remote_key_resource.py @@ -32,7 +32,7 @@ from synapse.util.httpresourcetree import create_resource_tree from synapse.util.stringutils import random_string from tests import unittest -from tests.server import FakeChannel, wait_until_result +from tests.server import FakeChannel from tests.utils import default_config @@ -94,7 +94,7 @@ class RemoteKeyResourceTestCase(BaseRemoteKeyResourceTestCase): % (server_name.encode("utf-8"), key_id.encode("utf-8")), b"1.1", ) - wait_until_result(self.reactor, req) + channel.await_result() self.assertEqual(channel.code, 200) resp = channel.json_body return resp @@ -190,7 +190,7 @@ class EndToEndPerspectivesTests(BaseRemoteKeyResourceTestCase): req.requestReceived( b"POST", path.encode("utf-8"), b"1.1", ) - wait_until_result(self.reactor, req) + channel.await_result() self.assertEqual(channel.code, 200) resp = channel.json_body return resp diff --git a/tests/server.py b/tests/server.py index ef03109a6c..18cb8b2d72 100644 --- a/tests/server.py +++ b/tests/server.py @@ -117,6 +117,25 @@ class FakeChannel: def transport(self): return self + def await_result(self, timeout: int = 100) -> None: + """ + Wait until the request is finished. + """ + self._reactor.run() + x = 0 + + while not self.result.get("done"): + # If there's a producer, tell it to resume producing so we get content + if self._producer: + self._producer.resumeProducing() + + x += 1 + + if x > timeout: + raise TimedOutException("Timed out waiting for request to finish.") + + self._reactor.advance(0.1) + class FakeSite: """ @@ -225,30 +244,9 @@ def make_request( return req, channel -def wait_until_result(clock, request, timeout=100): - """ - Wait until the request is finished. - """ - clock.run() - x = 0 - - while not request.finished: - - # If there's a producer, tell it to resume producing so we get content - if request._channel._producer: - request._channel._producer.resumeProducing() - - x += 1 - - if x > timeout: - raise TimedOutException("Timed out waiting for request to finish.") - - clock.advance(0.1) - - def render(request, resource, clock): request.render(resource) - wait_until_result(clock, request) + request._channel.await_result() @implementer(IReactorPluggableNameResolver) From 1f41422c98967c969fe8fc093c04308164bde594 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 14 Nov 2020 01:29:16 +0000 Subject: [PATCH 19/57] Fix the URL in the URL preview tests the preview resource is mointed at preview_url, not url_preview --- tests/rest/media/v1/test_url_preview.py | 41 +++++++++++++------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py index c00a7b9114..e00ad61231 100644 --- a/tests/rest/media/v1/test_url_preview.py +++ b/tests/rest/media/v1/test_url_preview.py @@ -133,11 +133,14 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.reactor.nameResolver = Resolver() + def create_test_resource(self): + return self.hs.get_media_repository_resource() + def test_cache_returns_correct_type(self): self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")] request, channel = self.make_request( - "GET", "url_preview?url=http://matrix.org", shorthand=False + "GET", "preview_url?url=http://matrix.org", shorthand=False, ) request.render(self.preview_url) self.pump() @@ -160,7 +163,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Check the cache returns the correct response request, channel = self.make_request( - "GET", "url_preview?url=http://matrix.org", shorthand=False + "GET", "preview_url?url=http://matrix.org", shorthand=False ) request.render(self.preview_url) self.pump() @@ -178,7 +181,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Check the database cache returns the correct response request, channel = self.make_request( - "GET", "url_preview?url=http://matrix.org", shorthand=False + "GET", "preview_url?url=http://matrix.org", shorthand=False ) request.render(self.preview_url) self.pump() @@ -201,7 +204,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): ) request, channel = self.make_request( - "GET", "url_preview?url=http://matrix.org", shorthand=False + "GET", "preview_url?url=http://matrix.org", shorthand=False, ) request.render(self.preview_url) self.pump() @@ -234,7 +237,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): ) request, channel = self.make_request( - "GET", "url_preview?url=http://matrix.org", shorthand=False + "GET", "preview_url?url=http://matrix.org", shorthand=False ) request.render(self.preview_url) self.pump() @@ -267,7 +270,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): ) request, channel = self.make_request( - "GET", "url_preview?url=http://matrix.org", shorthand=False + "GET", "preview_url?url=http://matrix.org", shorthand=False ) request.render(self.preview_url) self.pump() @@ -298,7 +301,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.lookups["example.com"] = [(IPv4Address, "10.1.2.3")] request, channel = self.make_request( - "GET", "url_preview?url=http://example.com", shorthand=False + "GET", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -326,7 +329,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.lookups["example.com"] = [(IPv4Address, "192.168.1.1")] request, channel = self.make_request( - "GET", "url_preview?url=http://example.com", shorthand=False + "GET", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -349,7 +352,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.lookups["example.com"] = [(IPv4Address, "1.1.1.2")] request, channel = self.make_request( - "GET", "url_preview?url=http://example.com", shorthand=False + "GET", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -368,7 +371,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): Blacklisted IP addresses, accessed directly, are not spidered. """ request, channel = self.make_request( - "GET", "url_preview?url=http://192.168.1.1", shorthand=False + "GET", "preview_url?url=http://192.168.1.1", shorthand=False ) request.render(self.preview_url) self.pump() @@ -389,7 +392,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): Blacklisted IP ranges, accessed directly, are not spidered. """ request, channel = self.make_request( - "GET", "url_preview?url=http://1.1.1.2", shorthand=False + "GET", "preview_url?url=http://1.1.1.2", shorthand=False ) request.render(self.preview_url) self.pump() @@ -411,7 +414,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.lookups["example.com"] = [(IPv4Address, "1.1.1.1")] request, channel = self.make_request( - "GET", "url_preview?url=http://example.com", shorthand=False + "GET", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -446,7 +449,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): ] request, channel = self.make_request( - "GET", "url_preview?url=http://example.com", shorthand=False + "GET", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -468,7 +471,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): ] request, channel = self.make_request( - "GET", "url_preview?url=http://example.com", shorthand=False + "GET", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -491,7 +494,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.lookups["example.com"] = [(IPv6Address, "2001:800::1")] request, channel = self.make_request( - "GET", "url_preview?url=http://example.com", shorthand=False + "GET", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -510,7 +513,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): OPTIONS returns the OPTIONS. """ request, channel = self.make_request( - "OPTIONS", "url_preview?url=http://example.com", shorthand=False + "OPTIONS", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -525,7 +528,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Build and make a request to the server request, channel = self.make_request( - "GET", "url_preview?url=http://example.com", shorthand=False + "GET", "preview_url?url=http://example.com", shorthand=False ) request.render(self.preview_url) self.pump() @@ -598,7 +601,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", - "url_preview?url=http://twitter.com/matrixdotorg/status/12345", + "preview_url?url=http://twitter.com/matrixdotorg/status/12345", shorthand=False, ) request.render(self.preview_url) @@ -663,7 +666,7 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", - "url_preview?url=http://twitter.com/matrixdotorg/status/12345", + "preview_url?url=http://twitter.com/matrixdotorg/status/12345", shorthand=False, ) request.render(self.preview_url) From 129ae841e5aebb34a980dd7d118140d08b0ff81d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2020 22:47:54 +0000 Subject: [PATCH 20/57] Make `make_request` actually render the request remove the stubbing out of `request.process`, so that `requestReceived` also renders the request via the appropriate resource. Replace render() with a stub for now. --- tests/replication/test_multi_media_repo.py | 4 +- tests/rest/admin/test_admin.py | 6 -- tests/rest/admin/test_media.py | 6 -- tests/rest/client/v1/utils.py | 4 +- tests/rest/client/v2_alpha/test_account.py | 4 -- tests/rest/media/v1/test_media_storage.py | 7 +-- tests/rest/media/v1/test_url_preview.py | 66 ++++++++++------------ tests/server.py | 20 ++++--- tests/unittest.py | 8 +++ 9 files changed, 57 insertions(+), 68 deletions(-) diff --git a/tests/replication/test_multi_media_repo.py b/tests/replication/test_multi_media_repo.py index a9ac4aeec1..48b574ccbe 100644 --- a/tests/replication/test_multi_media_repo.py +++ b/tests/replication/test_multi_media_repo.py @@ -68,15 +68,15 @@ class MediaRepoShardTestCase(BaseMultiWorkerStreamTestCase): the media which the caller should respond to. """ resource = hs.get_media_repository_resource().children[b"download"] - request, channel = make_request( + _, channel = make_request( self.reactor, FakeSite(resource), "GET", "/{}/{}".format(target, media_id), shorthand=False, access_token=self.access_token, + await_result=False, ) - request.render(resource) self.pump() clients = self.reactor.tcpClients diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 9e4b0bca53..961a5732b3 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -231,8 +231,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): shorthand=False, access_token=admin_user_tok, ) - request.render(self.download_resource) - self.pump(1.0) # Should be quarantined self.assertEqual( @@ -301,8 +299,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): shorthand=False, access_token=non_admin_user_tok, ) - request.render(self.download_resource) - self.pump(1.0) # Should be successful self.assertEqual(200, int(channel.code), msg=channel.result["body"]) @@ -478,8 +474,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): shorthand=False, access_token=non_admin_user_tok, ) - request.render(self.download_resource) - self.pump(1.0) # Shouldn't be quarantined self.assertEqual( diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 36e07f1b36..64b7aa53ee 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -133,8 +133,6 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): shorthand=False, access_token=self.admin_user_tok, ) - request.render(download_resource) - self.pump(1.0) # Should be successful self.assertEqual( @@ -172,8 +170,6 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): shorthand=False, access_token=self.admin_user_tok, ) - request.render(download_resource) - self.pump(1.0) self.assertEqual( 404, channel.code, @@ -548,8 +544,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): shorthand=False, access_token=self.admin_user_tok, ) - request.render(download_resource) - self.pump(1.0) if expect_success: self.assertEqual( diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index 900852f85b..040a92d6f0 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -310,7 +310,7 @@ class RestHelper: """ image_length = len(image_data) path = "/_matrix/media/r0/upload?filename=%s" % (filename,) - request, channel = make_request( + _, channel = make_request( self.hs.get_reactor(), FakeSite(resource), "POST", @@ -319,8 +319,6 @@ class RestHelper: access_token=tok, custom_headers=[(b"Content-Length", str(image_length))], ) - request.render(resource) - self.hs.get_reactor().pump([100]) assert channel.code == expect_code, "Expected: %d, got: %d, resp: %r" % ( expect_code, diff --git a/tests/rest/client/v2_alpha/test_account.py b/tests/rest/client/v2_alpha/test_account.py index 94a627b0a6..b871200909 100644 --- a/tests/rest/client/v2_alpha/test_account.py +++ b/tests/rest/client/v2_alpha/test_account.py @@ -263,8 +263,6 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): path, shorthand=False, ) - request.render(self.submit_token_resource) - self.pump() self.assertEquals(200, channel.code, channel.result) @@ -288,8 +286,6 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): shorthand=False, content_is_form=True, ) - request.render(self.submit_token_resource) - self.pump() self.assertEquals(200, channel.code, channel.result) def _get_link_from_email(self): diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py index 0fd31a0096..2a3b2a8f27 100644 --- a/tests/rest/media/v1/test_media_storage.py +++ b/tests/rest/media/v1/test_media_storage.py @@ -234,8 +234,8 @@ class MediaRepoTests(unittest.HomeserverTestCase): "GET", self.media_id, shorthand=False, + await_result=False, ) - request.render(self.download_resource) self.pump() # We've made one fetch, to example.com, using the media URL, and asking @@ -330,8 +330,8 @@ class MediaRepoTests(unittest.HomeserverTestCase): "GET", self.media_id + params, shorthand=False, + await_result=False, ) - request.render(self.thumbnail_resource) self.pump() headers = { @@ -359,7 +359,6 @@ class MediaRepoTests(unittest.HomeserverTestCase): channel.json_body, { "errcode": "M_NOT_FOUND", - "error": "Not found [b'example.com', b'12345?width=32&height=32&method=%s']" - % method, + "error": "Not found [b'example.com', b'12345']", }, ) diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py index e00ad61231..ccdc8c2ecf 100644 --- a/tests/rest/media/v1/test_url_preview.py +++ b/tests/rest/media/v1/test_url_preview.py @@ -140,9 +140,11 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")] request, channel = self.make_request( - "GET", "preview_url?url=http://matrix.org", shorthand=False, + "GET", + "preview_url?url=http://matrix.org", + shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) @@ -165,8 +167,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://matrix.org", shorthand=False ) - request.render(self.preview_url) - self.pump() # Check the cache response has the same content self.assertEqual(channel.code, 200) @@ -183,8 +183,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://matrix.org", shorthand=False ) - request.render(self.preview_url) - self.pump() # Check the cache response has the same content self.assertEqual(channel.code, 200) @@ -204,9 +202,11 @@ class URLPreviewTests(unittest.HomeserverTestCase): ) request, channel = self.make_request( - "GET", "preview_url?url=http://matrix.org", shorthand=False, + "GET", + "preview_url?url=http://matrix.org", + shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) @@ -237,9 +237,11 @@ class URLPreviewTests(unittest.HomeserverTestCase): ) request, channel = self.make_request( - "GET", "preview_url?url=http://matrix.org", shorthand=False + "GET", + "preview_url?url=http://matrix.org", + shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) @@ -270,9 +272,11 @@ class URLPreviewTests(unittest.HomeserverTestCase): ) request, channel = self.make_request( - "GET", "preview_url?url=http://matrix.org", shorthand=False + "GET", + "preview_url?url=http://matrix.org", + shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) @@ -301,9 +305,11 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.lookups["example.com"] = [(IPv4Address, "10.1.2.3")] request, channel = self.make_request( - "GET", "preview_url?url=http://example.com", shorthand=False + "GET", + "preview_url?url=http://example.com", + shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) @@ -331,8 +337,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://example.com", shorthand=False ) - request.render(self.preview_url) - self.pump() # No requests made. self.assertEqual(len(self.reactor.tcpClients), 0) @@ -354,8 +358,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://example.com", shorthand=False ) - request.render(self.preview_url) - self.pump() self.assertEqual(channel.code, 502) self.assertEqual( @@ -373,8 +375,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://192.168.1.1", shorthand=False ) - request.render(self.preview_url) - self.pump() # No requests made. self.assertEqual(len(self.reactor.tcpClients), 0) @@ -394,8 +394,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://1.1.1.2", shorthand=False ) - request.render(self.preview_url) - self.pump() self.assertEqual(channel.code, 403) self.assertEqual( @@ -414,9 +412,11 @@ class URLPreviewTests(unittest.HomeserverTestCase): self.lookups["example.com"] = [(IPv4Address, "1.1.1.1")] request, channel = self.make_request( - "GET", "preview_url?url=http://example.com", shorthand=False + "GET", + "preview_url?url=http://example.com", + shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) @@ -451,8 +451,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://example.com", shorthand=False ) - request.render(self.preview_url) - self.pump() self.assertEqual(channel.code, 502) self.assertEqual( channel.json_body, @@ -473,8 +471,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://example.com", shorthand=False ) - request.render(self.preview_url) - self.pump() # No requests made. self.assertEqual(len(self.reactor.tcpClients), 0) @@ -496,8 +492,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "preview_url?url=http://example.com", shorthand=False ) - request.render(self.preview_url) - self.pump() self.assertEqual(channel.code, 502) self.assertEqual( @@ -515,8 +509,6 @@ class URLPreviewTests(unittest.HomeserverTestCase): request, channel = self.make_request( "OPTIONS", "preview_url?url=http://example.com", shorthand=False ) - request.render(self.preview_url) - self.pump() self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body, {}) @@ -528,9 +520,11 @@ class URLPreviewTests(unittest.HomeserverTestCase): # Build and make a request to the server request, channel = self.make_request( - "GET", "preview_url?url=http://example.com", shorthand=False + "GET", + "preview_url?url=http://example.com", + shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() # Extract Synapse's tcp client @@ -603,8 +597,8 @@ class URLPreviewTests(unittest.HomeserverTestCase): "GET", "preview_url?url=http://twitter.com/matrixdotorg/status/12345", shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) @@ -668,8 +662,8 @@ class URLPreviewTests(unittest.HomeserverTestCase): "GET", "preview_url?url=http://twitter.com/matrixdotorg/status/12345", shorthand=False, + await_result=False, ) - request.render(self.preview_url) self.pump() client = self.reactor.tcpClients[0][2].buildProtocol(None) diff --git a/tests/server.py b/tests/server.py index 5a1583a3e7..de7cb1d8b3 100644 --- a/tests/server.py +++ b/tests/server.py @@ -171,16 +171,18 @@ def make_request( shorthand=True, federation_auth_origin=None, content_is_form=False, + await_result: bool = True, custom_headers: Optional[ Iterable[Tuple[Union[bytes, str], Union[bytes, str]]] ] = None, ): """ - Make a web request using the given method and path, feed it the - content, and return the Request and the Channel underneath. + Make a web request using the given method, path and content, and render it + + Returns the Request and the Channel underneath. Args: - site: The twisted Site to associate with the Channel + site: The twisted Site to use to render the request method (bytes/unicode): The HTTP request method ("verb"). path (bytes/unicode): The HTTP path, suitably URL encoded (e.g. @@ -196,6 +198,10 @@ def make_request( custom_headers: (name, value) pairs to add as request headers + await_result: whether to wait for the request to complete rendering. If true, + will pump the reactor until the the renderer tells the channel the request + is finished. + Returns: Tuple[synapse.http.site.SynapseRequest, channel] """ @@ -225,11 +231,9 @@ def make_request( channel = FakeChannel(site, reactor) req = request(channel) - req.process = lambda: b"" req.content = BytesIO(content) # Twisted expects to be at the end of the content when parsing the request. req.content.seek(SEEK_END) - req.postpath = list(map(unquote, path[1:].split(b"/"))) if access_token: req.requestHeaders.addRawHeader( @@ -257,12 +261,14 @@ def make_request( req.requestReceived(method, path, b"1.1") + if await_result: + channel.await_result() + return req, channel def render(request, resource, clock): - request.render(resource) - request._channel.await_result() + pass @implementer(IReactorPluggableNameResolver) diff --git a/tests/unittest.py b/tests/unittest.py index e39cb8dec9..9c7eca3b6e 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -377,6 +377,7 @@ class HomeserverTestCase(TestCase): shorthand: bool = True, federation_auth_origin: str = None, content_is_form: bool = False, + await_result: bool = True, ) -> Tuple[SynapseRequest, FakeChannel]: ... @@ -391,6 +392,7 @@ class HomeserverTestCase(TestCase): shorthand: bool = True, federation_auth_origin: str = None, content_is_form: bool = False, + await_result: bool = True, ) -> Tuple[T, FakeChannel]: ... @@ -404,6 +406,7 @@ class HomeserverTestCase(TestCase): shorthand: bool = True, federation_auth_origin: str = None, content_is_form: bool = False, + await_result: bool = True, ) -> Tuple[T, FakeChannel]: """ Create a SynapseRequest at the path using the method and containing the @@ -422,6 +425,10 @@ class HomeserverTestCase(TestCase): content_is_form: Whether the content is URL encoded form data. Adds the 'Content-Type': 'application/x-www-form-urlencoded' header. + await_result: whether to wait for the request to complete rendering. If + true (the default), will pump the test reactor until the the renderer + tells the channel the request is finished. + Returns: Tuple[synapse.http.site.SynapseRequest, channel] """ @@ -436,6 +443,7 @@ class HomeserverTestCase(TestCase): shorthand, federation_auth_origin, content_is_form, + await_result, ) def render(self, request): From be8fa65d0baddcc0a64954e21d38a854e4ee00d7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2020 22:49:21 +0000 Subject: [PATCH 21/57] Remove redundant calls to `render()` --- tests/app/test_frontend_proxy.py | 10 ++-- tests/app/test_openid_listener.py | 12 ++--- tests/http/test_additional_resource.py | 4 +- tests/replication/_base.py | 5 +- tests/replication/test_client_reader_shard.py | 4 -- .../test_sharded_event_persister.py | 8 --- tests/rest/client/test_consent.py | 6 +-- tests/rest/client/v1/utils.py | 16 ++---- tests/rest/client/v2_alpha/test_sync.py | 6 +-- tests/server.py | 5 -- tests/storage/test_client_ips.py | 3 +- tests/test_server.py | 49 +++++-------------- tests/unittest.py | 10 +--- 13 files changed, 32 insertions(+), 106 deletions(-) diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py index 0bac7995e8..40abe9d72d 100644 --- a/tests/app/test_frontend_proxy.py +++ b/tests/app/test_frontend_proxy.py @@ -15,7 +15,7 @@ from synapse.app.generic_worker import GenericWorkerServer -from tests.server import make_request, render +from tests.server import make_request from tests.unittest import HomeserverTestCase @@ -56,10 +56,8 @@ class FrontendProxyTests(HomeserverTestCase): # Grab the resource from the site that was told to listen self.assertEqual(len(self.reactor.tcpServers), 1) site = self.reactor.tcpServers[0][1] - resource = site.resource.children[b"_matrix"].children[b"client"] - request, channel = make_request(self.reactor, site, "PUT", "presence/a/status") - render(request, resource, self.reactor) + _, channel = make_request(self.reactor, site, "PUT", "presence/a/status") # 400 + unrecognised, because nothing is registered self.assertEqual(channel.code, 400) @@ -78,10 +76,8 @@ class FrontendProxyTests(HomeserverTestCase): # Grab the resource from the site that was told to listen self.assertEqual(len(self.reactor.tcpServers), 1) site = self.reactor.tcpServers[0][1] - resource = site.resource.children[b"_matrix"].children[b"client"] - request, channel = make_request(self.reactor, site, "PUT", "presence/a/status") - render(request, resource, self.reactor) + _, channel = make_request(self.reactor, site, "PUT", "presence/a/status") # 401, because the stub servlet still checks authentication self.assertEqual(channel.code, 401) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 1292145890..ea3be95cf1 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -20,7 +20,7 @@ from synapse.app.generic_worker import GenericWorkerServer from synapse.app.homeserver import SynapseHomeServer from synapse.config.server import parse_listener_def -from tests.server import make_request, render +from tests.server import make_request from tests.unittest import HomeserverTestCase @@ -67,16 +67,15 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): # Grab the resource from the site that was told to listen site = self.reactor.tcpServers[0][1] try: - resource = site.resource.children[b"_matrix"].children[b"federation"] + site.resource.children[b"_matrix"].children[b"federation"] except KeyError: if expectation == "no_resource": return raise - request, channel = make_request( + _, channel = make_request( self.reactor, site, "GET", "/_matrix/federation/v1/openid/userinfo" ) - render(request, resource, self.reactor) self.assertEqual(channel.code, 401) @@ -116,15 +115,14 @@ class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): # Grab the resource from the site that was told to listen site = self.reactor.tcpServers[0][1] try: - resource = site.resource.children[b"_matrix"].children[b"federation"] + site.resource.children[b"_matrix"].children[b"federation"] except KeyError: if expectation == "no_resource": return raise - request, channel = make_request( + _, channel = make_request( self.reactor, site, "GET", "/_matrix/federation/v1/openid/userinfo" ) - render(request, resource, self.reactor) self.assertEqual(channel.code, 401) diff --git a/tests/http/test_additional_resource.py b/tests/http/test_additional_resource.py index e835512a41..05e9c449be 100644 --- a/tests/http/test_additional_resource.py +++ b/tests/http/test_additional_resource.py @@ -17,7 +17,7 @@ from synapse.http.additional_resource import AdditionalResource from synapse.http.server import respond_with_json -from tests.server import FakeSite, make_request, render +from tests.server import FakeSite, make_request from tests.unittest import HomeserverTestCase @@ -47,7 +47,6 @@ class AdditionalResourceTests(HomeserverTestCase): resource = AdditionalResource(self.hs, handler) request, channel = make_request(self.reactor, FakeSite(resource), "GET", "/") - render(request, resource, self.reactor) self.assertEqual(request.code, 200) self.assertEqual(channel.json_body, {"some_key": "some_value_async"}) @@ -57,7 +56,6 @@ class AdditionalResourceTests(HomeserverTestCase): resource = AdditionalResource(self.hs, handler) request, channel = make_request(self.reactor, FakeSite(resource), "GET", "/") - render(request, resource, self.reactor) self.assertEqual(request.code, 200) self.assertEqual(channel.json_body, {"some_key": "some_value_sync"}) diff --git a/tests/replication/_base.py b/tests/replication/_base.py index bc56b13dcd..516db4c30a 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -36,7 +36,7 @@ from synapse.server import HomeServer from synapse.util import Clock from tests import unittest -from tests.server import FakeTransport, render +from tests.server import FakeTransport try: import hiredis @@ -347,9 +347,6 @@ class BaseMultiWorkerStreamTestCase(unittest.HomeserverTestCase): config["worker_replication_http_port"] = "8765" return config - def render_on_worker(self, worker_hs: HomeServer, request: SynapseRequest): - render(request, self._hs_to_site[worker_hs].resource, self.reactor) - def replicate(self): """Tell the master side of replication that something has happened, and then wait for the replication to occur. diff --git a/tests/replication/test_client_reader_shard.py b/tests/replication/test_client_reader_shard.py index 90172bd377..96801db473 100644 --- a/tests/replication/test_client_reader_shard.py +++ b/tests/replication/test_client_reader_shard.py @@ -55,7 +55,6 @@ class ClientReaderTestCase(BaseMultiWorkerStreamTestCase): "register", {"username": "user", "type": "m.login.password", "password": "bar"}, ) # type: SynapseRequest, FakeChannel - self.render_on_worker(worker_hs, request_1) self.assertEqual(request_1.code, 401) # Grab the session @@ -69,7 +68,6 @@ class ClientReaderTestCase(BaseMultiWorkerStreamTestCase): "register", {"auth": {"session": session, "type": "m.login.dummy"}}, ) # type: SynapseRequest, FakeChannel - self.render_on_worker(worker_hs, request_2) self.assertEqual(request_2.code, 200) # We're given a registered user. @@ -89,7 +87,6 @@ class ClientReaderTestCase(BaseMultiWorkerStreamTestCase): "register", {"username": "user", "type": "m.login.password", "password": "bar"}, ) # type: SynapseRequest, FakeChannel - self.render_on_worker(worker_hs_1, request_1) self.assertEqual(request_1.code, 401) # Grab the session @@ -104,7 +101,6 @@ class ClientReaderTestCase(BaseMultiWorkerStreamTestCase): "register", {"auth": {"session": session, "type": "m.login.dummy"}}, ) # type: SynapseRequest, FakeChannel - self.render_on_worker(worker_hs_2, request_2) self.assertEqual(request_2.code, 200) # We're given a registered user. diff --git a/tests/replication/test_sharded_event_persister.py b/tests/replication/test_sharded_event_persister.py index 2820dd622f..77fc3856d5 100644 --- a/tests/replication/test_sharded_event_persister.py +++ b/tests/replication/test_sharded_event_persister.py @@ -183,7 +183,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): request, channel = make_request( self.reactor, sync_hs_site, "GET", "/sync", access_token=access_token ) - self.render_on_worker(sync_hs, request) next_batch = channel.json_body["next_batch"] # We now gut wrench into the events stream MultiWriterIdGenerator on @@ -214,7 +213,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): "/sync?since={}".format(next_batch), access_token=access_token, ) - self.render_on_worker(sync_hs, request) # We should only see the new event and nothing else self.assertIn(room_id1, channel.json_body["rooms"]["join"]) @@ -245,7 +243,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): "/sync?since={}".format(vector_clock_token), access_token=access_token, ) - self.render_on_worker(sync_hs, request) self.assertNotIn(room_id1, channel.json_body["rooms"]["join"]) self.assertIn(room_id2, channel.json_body["rooms"]["join"]) @@ -271,7 +268,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): "/sync?since={}".format(next_batch), access_token=access_token, ) - self.render_on_worker(sync_hs, request) prev_batch1 = channel.json_body["rooms"]["join"][room_id1]["timeline"][ "prev_batch" @@ -292,7 +288,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): ), access_token=access_token, ) - self.render_on_worker(sync_hs, request) self.assertListEqual([], channel.json_body["chunk"]) # Paginating back on the second room should produce the first event @@ -306,7 +301,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): ), access_token=access_token, ) - self.render_on_worker(sync_hs, request) self.assertEqual(len(channel.json_body["chunk"]), 1) self.assertEqual( channel.json_body["chunk"][0]["event_id"], first_event_in_room2 @@ -322,7 +316,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): ), access_token=access_token, ) - self.render_on_worker(sync_hs, request) self.assertListEqual([], channel.json_body["chunk"]) request, channel = make_request( @@ -334,7 +327,6 @@ class EventPersisterShardTestCase(BaseMultiWorkerStreamTestCase): ), access_token=access_token, ) - self.render_on_worker(sync_hs, request) self.assertEqual(len(channel.json_body["chunk"]), 1) self.assertEqual( channel.json_body["chunk"][0]["event_id"], first_event_in_room2 diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py index 2931859f25..e2e6a5e16d 100644 --- a/tests/rest/client/test_consent.py +++ b/tests/rest/client/test_consent.py @@ -21,7 +21,7 @@ from synapse.rest.client.v1 import login, room from synapse.rest.consent import consent_resource from tests import unittest -from tests.server import FakeSite, make_request, render +from tests.server import FakeSite, make_request class ConsentResourceTestCase(unittest.HomeserverTestCase): @@ -64,7 +64,6 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): request, channel = make_request( self.reactor, FakeSite(resource), "GET", "/consent?v=1", shorthand=False ) - render(request, resource, self.reactor) self.assertEqual(channel.code, 200) def test_accept_consent(self): @@ -91,7 +90,6 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): access_token=access_token, shorthand=False, ) - render(request, resource, self.reactor) self.assertEqual(channel.code, 200) # Get the version from the body, and whether we've consented @@ -107,7 +105,6 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): access_token=access_token, shorthand=False, ) - render(request, resource, self.reactor) self.assertEqual(channel.code, 200) # Fetch the consent page, to get the consent version -- it should have @@ -120,7 +117,6 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): access_token=access_token, shorthand=False, ) - render(request, resource, self.reactor) self.assertEqual(channel.code, 200) # Get the version from the body, and check that it's the version we diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index 040a92d6f0..b58768675b 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -27,7 +27,7 @@ from twisted.web.server import Site from synapse.api.constants import Membership -from tests.server import FakeSite, make_request, render +from tests.server import FakeSite, make_request @attr.s @@ -52,14 +52,13 @@ class RestHelper: if tok: path = path + "?access_token=%s" % tok - request, channel = make_request( + _, channel = make_request( self.hs.get_reactor(), self.site, "POST", path, json.dumps(content).encode("utf8"), ) - render(request, self.site.resource, self.hs.get_reactor()) assert channel.result["code"] == b"%d" % expect_code, channel.result self.auth_user_id = temp_id @@ -129,7 +128,7 @@ class RestHelper: data = {"membership": membership} data.update(extra_data) - request, channel = make_request( + _, channel = make_request( self.hs.get_reactor(), self.site, "PUT", @@ -137,8 +136,6 @@ class RestHelper: json.dumps(data).encode("utf8"), ) - render(request, self.site.resource, self.hs.get_reactor()) - assert int(channel.result["code"]) == expect_code, ( "Expected: %d, got: %d, resp: %r" % (expect_code, int(channel.result["code"]), channel.result["body"]) @@ -166,14 +163,13 @@ class RestHelper: if tok: path = path + "?access_token=%s" % tok - request, channel = make_request( + _, channel = make_request( self.hs.get_reactor(), self.site, "PUT", path, json.dumps(content).encode("utf8"), ) - render(request, self.site.resource, self.hs.get_reactor()) assert int(channel.result["code"]) == expect_code, ( "Expected: %d, got: %d, resp: %r" @@ -223,12 +219,10 @@ class RestHelper: if body is not None: content = json.dumps(body).encode("utf8") - request, channel = make_request( + _, channel = make_request( self.hs.get_reactor(), self.site, method, path, content ) - render(request, self.site.resource, self.hs.get_reactor()) - assert int(channel.result["code"]) == expect_code, ( "Expected: %d, got: %d, resp: %r" % (expect_code, int(channel.result["code"]), channel.result["body"]) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index a31e44c97e..f74d611943 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -320,10 +320,8 @@ class SyncTypingTests(unittest.HomeserverTestCase): typing._reset() # Now it SHOULD fail as it never completes! - request, channel = self.make_request( - "GET", sync_url % (access_token, next_batch) - ) - self.assertRaises(TimedOutException, self.render, request) + with self.assertRaises(TimedOutException): + self.make_request("GET", sync_url % (access_token, next_batch)) class UnreadMessagesTestCase(unittest.HomeserverTestCase): diff --git a/tests/server.py b/tests/server.py index de7cb1d8b3..a51ad0c14e 100644 --- a/tests/server.py +++ b/tests/server.py @@ -19,7 +19,6 @@ from twisted.internet.interfaces import ( ) from twisted.python.failure import Failure from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactorClock -from twisted.web.http import unquote from twisted.web.http_headers import Headers from twisted.web.resource import IResource from twisted.web.server import Site @@ -267,10 +266,6 @@ def make_request( return req, channel -def render(request, resource, clock): - pass - - @implementer(IReactorPluggableNameResolver) class ThreadedMemoryReactorClock(MemoryReactorClock): """ diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index 583addb5b5..6bdde1a2ba 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -412,7 +412,7 @@ class ClientIpAuthTestCase(unittest.HomeserverTestCase): headers1 = {b"User-Agent": b"Mozzila pizza"} headers1.update(headers) - request, channel = make_request( + make_request( self.reactor, self.site, "GET", @@ -421,7 +421,6 @@ class ClientIpAuthTestCase(unittest.HomeserverTestCase): custom_headers=headers1.items(), **make_request_args, ) - self.render(request) # Advance so the save loop occurs self.reactor.advance(100) diff --git a/tests/test_server.py b/tests/test_server.py index 300d13ac95..c387a85f2e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -29,7 +29,6 @@ from tests.server import ( FakeSite, ThreadedMemoryReactorClock, make_request, - render, setup_test_homeserver, ) @@ -65,7 +64,6 @@ class JsonResourceTests(unittest.TestCase): request, channel = make_request( self.reactor, FakeSite(res), b"GET", b"/_matrix/foo/%E2%98%83?a=%E2%98%83" ) - render(request, res, self.reactor) self.assertEqual(request.args, {b"a": ["\N{SNOWMAN}".encode("utf8")]}) self.assertEqual(got_kwargs, {"room_id": "\N{SNOWMAN}"}) @@ -84,10 +82,7 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" ) - request, channel = make_request( - self.reactor, FakeSite(res), b"GET", b"/_matrix/foo" - ) - render(request, res, self.reactor) + _, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/_matrix/foo") self.assertEqual(channel.result["code"], b"500") @@ -111,10 +106,7 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" ) - request, channel = make_request( - self.reactor, FakeSite(res), b"GET", b"/_matrix/foo" - ) - render(request, res, self.reactor) + _, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/_matrix/foo") self.assertEqual(channel.result["code"], b"500") @@ -132,10 +124,7 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" ) - request, channel = make_request( - self.reactor, FakeSite(res), b"GET", b"/_matrix/foo" - ) - render(request, res, self.reactor) + _, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/_matrix/foo") self.assertEqual(channel.result["code"], b"403") self.assertEqual(channel.json_body["error"], "Forbidden!!one!") @@ -157,10 +146,9 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo$")], _callback, "test_servlet" ) - request, channel = make_request( + _, channel = make_request( self.reactor, FakeSite(res), b"GET", b"/_matrix/foobar" ) - render(request, res, self.reactor) self.assertEqual(channel.result["code"], b"400") self.assertEqual(channel.json_body["error"], "Unrecognized request") @@ -182,10 +170,7 @@ class JsonResourceTests(unittest.TestCase): ) # The path was registered as GET, but this is a HEAD request. - request, channel = make_request( - self.reactor, FakeSite(res), b"HEAD", b"/_matrix/foo" - ) - render(request, res, self.reactor) + _, channel = make_request(self.reactor, FakeSite(res), b"HEAD", b"/_matrix/foo") self.assertEqual(channel.result["code"], b"200") self.assertNotIn("body", channel.result) @@ -216,16 +201,8 @@ class OptionsResourceTests(unittest.TestCase): "1.0", ) - request, channel = make_request( - self.reactor, site, method, path, shorthand=False - ) - request.prepath = [] # This doesn't get set properly by make_request. - - request.site = site - resource = site.getResourceFor(request) - - # Finally, render the resource and return the channel. - render(request, resource, self.reactor) + # render the request and return the channel + _, channel = make_request(self.reactor, site, method, path, shorthand=False) return channel def test_unknown_options_request(self): @@ -298,8 +275,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): res = WrapHtmlRequestHandlerTests.TestResource() res.callback = callback - request, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") - render(request, res, self.reactor) + _, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") self.assertEqual(channel.result["code"], b"200") body = channel.result["body"] @@ -317,8 +293,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): res = WrapHtmlRequestHandlerTests.TestResource() res.callback = callback - request, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") - render(request, res, self.reactor) + _, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") self.assertEqual(channel.result["code"], b"301") headers = channel.result["headers"] @@ -339,8 +314,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): res = WrapHtmlRequestHandlerTests.TestResource() res.callback = callback - request, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") - render(request, res, self.reactor) + _, channel = make_request(self.reactor, FakeSite(res), b"GET", b"/path") self.assertEqual(channel.result["code"], b"304") headers = channel.result["headers"] @@ -359,8 +333,7 @@ class WrapHtmlRequestHandlerTests(unittest.TestCase): res = WrapHtmlRequestHandlerTests.TestResource() res.callback = callback - request, channel = make_request(self.reactor, FakeSite(res), b"HEAD", b"/path") - render(request, res, self.reactor) + _, channel = make_request(self.reactor, FakeSite(res), b"HEAD", b"/path") self.assertEqual(channel.result["code"], b"200") self.assertNotIn("body", channel.result) diff --git a/tests/unittest.py b/tests/unittest.py index 9c7eca3b6e..8a49bb5262 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -48,13 +48,7 @@ from synapse.server import HomeServer from synapse.types import UserID, create_requester from synapse.util.ratelimitutils import FederationRateLimiter -from tests.server import ( - FakeChannel, - get_clock, - make_request, - render, - setup_test_homeserver, -) +from tests.server import FakeChannel, get_clock, make_request, setup_test_homeserver from tests.test_utils import event_injection, setup_awaitable_errors from tests.test_utils.logging_setup import setup_logging from tests.utils import default_config, setupdb @@ -454,7 +448,7 @@ class HomeserverTestCase(TestCase): Args: request (synapse.http.site.SynapseRequest): The request to render. """ - render(request, self.resource, self.reactor) + pass def setup_test_homeserver(self, *args, **kwargs): """ From acfe3b3065bf134e01439e24753bb8bb2ad9e1c2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2020 22:58:47 +0000 Subject: [PATCH 22/57] Remove redundant `HomeserverTestCase.render` --- tests/federation/test_complexity.py | 2 - tests/federation/test_federation_server.py | 3 - tests/federation/transport/test_server.py | 2 - tests/handlers/test_directory.py | 6 - tests/handlers/test_message.py | 1 - tests/handlers/test_typing.py | 1 - tests/handlers/test_user_directory.py | 2 - tests/rest/admin/test_admin.py | 13 --- tests/rest/admin/test_device.py | 35 ------ tests/rest/admin/test_event_reports.py | 27 ----- tests/rest/admin/test_media.py | 23 ---- tests/rest/admin/test_room.py | 46 -------- tests/rest/admin/test_statistics.py | 27 ----- tests/rest/admin/test_user.py | 95 ---------------- tests/rest/client/test_ephemeral_message.py | 1 - tests/rest/client/test_identity.py | 2 - tests/rest/client/test_redactions.py | 2 - tests/rest/client/test_retention.py | 1 - tests/rest/client/test_shadow_banned.py | 8 -- tests/rest/client/test_third_party_rules.py | 5 - tests/rest/client/v1/test_directory.py | 4 - tests/rest/client/v1/test_events.py | 4 - tests/rest/client/v1/test_login.py | 32 ------ tests/rest/client/v1/test_presence.py | 2 - tests/rest/client/v1/test_profile.py | 7 -- tests/rest/client/v1/test_push_rule_attrs.py | 33 ------ tests/rest/client/v1/test_rooms.py | 107 ------------------ tests/rest/client/v1/test_typing.py | 4 - tests/rest/client/v2_alpha/test_account.py | 19 ---- tests/rest/client/v2_alpha/test_auth.py | 6 - .../rest/client/v2_alpha/test_capabilities.py | 4 - tests/rest/client/v2_alpha/test_filter.py | 7 -- .../client/v2_alpha/test_password_policy.py | 8 -- tests/rest/client/v2_alpha/test_register.py | 31 ----- tests/rest/client/v2_alpha/test_relations.py | 18 --- .../rest/client/v2_alpha/test_shared_rooms.py | 1 - tests/rest/client/v2_alpha/test_sync.py | 12 -- tests/rest/test_health.py | 1 - tests/rest/test_well_known.py | 2 - tests/server_notices/test_consent.py | 3 - .../test_resource_limits_server_notices.py | 3 - tests/test_mau.py | 2 - tests/test_terms_auth.py | 3 - tests/unittest.py | 14 --- 44 files changed, 629 deletions(-) diff --git a/tests/federation/test_complexity.py b/tests/federation/test_complexity.py index 1471cc1a28..0187f56e21 100644 --- a/tests/federation/test_complexity.py +++ b/tests/federation/test_complexity.py @@ -51,7 +51,6 @@ class RoomComplexityTests(unittest.FederatingHomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/federation/unstable/rooms/%s/complexity" % (room_1,) ) - self.render(request) self.assertEquals(200, channel.code) complexity = channel.json_body["v1"] self.assertTrue(complexity > 0, complexity) @@ -64,7 +63,6 @@ class RoomComplexityTests(unittest.FederatingHomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/federation/unstable/rooms/%s/complexity" % (room_1,) ) - self.render(request) self.assertEquals(200, channel.code) complexity = channel.json_body["v1"] self.assertEqual(complexity, 1.23) diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index da933ecd75..3009fbb6c4 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -51,7 +51,6 @@ class FederationServerTests(unittest.FederatingHomeserverTestCase): "/_matrix/federation/v1/get_missing_events/%s" % (room_1,), query_content, ) - self.render(request) self.assertEquals(400, channel.code, channel.result) self.assertEqual(channel.json_body["errcode"], "M_NOT_JSON") @@ -99,7 +98,6 @@ class StateQueryTests(unittest.FederatingHomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/federation/v1/state/%s" % (room_1,) ) - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertEqual( @@ -132,7 +130,6 @@ class StateQueryTests(unittest.FederatingHomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/federation/v1/state/%s" % (room_1,) ) - self.render(request) self.assertEquals(403, channel.code, channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") diff --git a/tests/federation/transport/test_server.py b/tests/federation/transport/test_server.py index 72e22d655f..f9e3c7a51f 100644 --- a/tests/federation/transport/test_server.py +++ b/tests/federation/transport/test_server.py @@ -40,7 +40,6 @@ class RoomDirectoryFederationTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/federation/v1/publicRooms" ) - self.render(request) self.assertEquals(403, channel.code) @override_config({"allow_public_rooms_over_federation": True}) @@ -48,5 +47,4 @@ class RoomDirectoryFederationTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/federation/v1/publicRooms" ) - self.render(request) self.assertEquals(200, channel.code) diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 2ce6dc9528..ee6ef5e6fa 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -412,7 +412,6 @@ class TestCreateAliasACL(unittest.HomeserverTestCase): b"directory/room/%23test%3Atest", ('{"room_id":"%s"}' % (room_id,)).encode("ascii"), ) - self.render(request) self.assertEquals(403, channel.code, channel.result) def test_allowed(self): @@ -423,7 +422,6 @@ class TestCreateAliasACL(unittest.HomeserverTestCase): b"directory/room/%23unofficial_test%3Atest", ('{"room_id":"%s"}' % (room_id,)).encode("ascii"), ) - self.render(request) self.assertEquals(200, channel.code, channel.result) @@ -438,7 +436,6 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", b"directory/list/room/%s" % (room_id.encode("ascii"),), b"{}" ) - self.render(request) self.assertEquals(200, channel.code, channel.result) self.room_list_handler = hs.get_room_list_handler() @@ -452,7 +449,6 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase): # Room list is enabled so we should get some results request, channel = self.make_request("GET", b"publicRooms") - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertTrue(len(channel.json_body["chunk"]) > 0) @@ -461,7 +457,6 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase): # Room list disabled so we should get no results request, channel = self.make_request("GET", b"publicRooms") - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertTrue(len(channel.json_body["chunk"]) == 0) @@ -470,5 +465,4 @@ class TestRoomListSearchDisabled(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", b"directory/list/room/%s" % (room_id.encode("ascii"),), b"{}" ) - self.render(request) self.assertEquals(403, channel.code, channel.result) diff --git a/tests/handlers/test_message.py b/tests/handlers/test_message.py index 8b57081cbe..af42775815 100644 --- a/tests/handlers/test_message.py +++ b/tests/handlers/test_message.py @@ -209,5 +209,4 @@ class ServerAclValidationTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", path, content={}, access_token=self.access_token ) - self.render(request) self.assertEqual(int(channel.result["code"]), 403) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 16ff2e22d2..abbdf2d524 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -228,7 +228,6 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase): ), federation_auth_origin=b"farm", ) - self.render(request) self.assertEqual(channel.code, 200) self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])]) diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index 87be94111f..98e5af2072 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -537,7 +537,6 @@ class TestUserDirSearchDisabled(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", b"user_directory/search", b'{"search_term":"user2"}' ) - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertTrue(len(channel.json_body["results"]) > 0) @@ -546,6 +545,5 @@ class TestUserDirSearchDisabled(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", b"user_directory/search", b'{"search_term":"user2"}' ) - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertTrue(len(channel.json_body["results"]) == 0) diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 961a5732b3..898e43411e 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -43,7 +43,6 @@ class VersionTestCase(unittest.HomeserverTestCase): def test_version_string(self): request, channel = self.make_request("GET", self.url, shorthand=False) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( @@ -76,7 +75,6 @@ class DeleteGroupTestCase(unittest.HomeserverTestCase): content={"localpart": "test"}, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) group_id = channel.json_body["group_id"] @@ -89,14 +87,12 @@ class DeleteGroupTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url.encode("ascii"), access_token=self.admin_user_tok, content={} ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) url = "/groups/%s/self/accept_invite" % (group_id,) request, channel = self.make_request( "PUT", url.encode("ascii"), access_token=self.other_user_token, content={} ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Check other user knows they're in the group @@ -112,7 +108,6 @@ class DeleteGroupTestCase(unittest.HomeserverTestCase): content={"localpart": "test"}, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Check group returns 404 @@ -132,7 +127,6 @@ class DeleteGroupTestCase(unittest.HomeserverTestCase): "GET", url.encode("ascii"), access_token=self.admin_user_tok ) - self.render(request) self.assertEqual( expect_code, int(channel.result["code"]), msg=channel.result["body"] ) @@ -144,7 +138,6 @@ class DeleteGroupTestCase(unittest.HomeserverTestCase): "GET", "/joined_groups".encode("ascii"), access_token=access_token ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) return channel.json_body["groups"] @@ -251,7 +244,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url.encode("ascii"), access_token=non_admin_user_tok, ) - self.render(request) # Expect a forbidden error self.assertEqual( @@ -265,7 +257,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url.encode("ascii"), access_token=non_admin_user_tok, ) - self.render(request) # Expect a forbidden error self.assertEqual( @@ -309,7 +300,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): urllib.parse.quote(media_id), ) request, channel = self.make_request("POST", url, access_token=admin_user_tok,) - self.render(request) self.pump(1.0) self.assertEqual(200, int(channel.code), msg=channel.result["body"]) @@ -362,7 +352,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): room_id ) request, channel = self.make_request("POST", url, access_token=admin_user_tok,) - self.render(request) self.pump(1.0) self.assertEqual(200, int(channel.code), msg=channel.result["body"]) self.assertEqual( @@ -409,7 +398,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url.encode("ascii"), access_token=admin_user_tok, ) - self.render(request) self.pump(1.0) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( @@ -452,7 +440,6 @@ class QuarantineMediaTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url.encode("ascii"), access_token=admin_user_tok, ) - self.render(request) self.pump(1.0) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( diff --git a/tests/rest/admin/test_device.py b/tests/rest/admin/test_device.py index d89eb90cfe..cf3a007598 100644 --- a/tests/rest/admin/test_device.py +++ b/tests/rest/admin/test_device.py @@ -51,19 +51,16 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): Try to get a device of an user without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) request, channel = self.make_request("PUT", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) request, channel = self.make_request("DELETE", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -75,7 +72,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -83,7 +79,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", self.url, access_token=self.other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -91,7 +86,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", self.url, access_token=self.other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -108,7 +102,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -116,7 +109,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -124,7 +116,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -141,7 +132,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only lookup local users", channel.json_body["error"]) @@ -149,7 +139,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only lookup local users", channel.json_body["error"]) @@ -157,7 +146,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only lookup local users", channel.json_body["error"]) @@ -173,7 +161,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -181,14 +168,12 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) request, channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) - self.render(request) # Delete unknown device returns status 200 self.assertEqual(200, channel.code, msg=channel.json_body) @@ -218,7 +203,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual(Codes.TOO_LARGE, channel.json_body["errcode"]) @@ -227,7 +211,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual("new display", channel.json_body["display_name"]) @@ -247,7 +230,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) @@ -255,7 +237,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual("new display", channel.json_body["display_name"]) @@ -272,7 +253,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) @@ -280,7 +260,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual("new displayname", channel.json_body["display_name"]) @@ -292,7 +271,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(self.other_user, channel.json_body["user_id"]) @@ -316,7 +294,6 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) @@ -347,7 +324,6 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): Try to list devices of an user without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -361,7 +337,6 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -374,7 +349,6 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -388,7 +362,6 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only lookup local users", channel.json_body["error"]) @@ -403,7 +376,6 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -422,7 +394,6 @@ class DevicesRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(number_devices, channel.json_body["total"]) @@ -461,7 +432,6 @@ class DeleteDevicesRestTestCase(unittest.HomeserverTestCase): Try to delete devices of an user without authentication. """ request, channel = self.make_request("POST", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -475,7 +445,6 @@ class DeleteDevicesRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", self.url, access_token=other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -488,7 +457,6 @@ class DeleteDevicesRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -502,7 +470,6 @@ class DeleteDevicesRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only lookup local users", channel.json_body["error"]) @@ -518,7 +485,6 @@ class DeleteDevicesRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) # Delete unknown devices returns status 200 self.assertEqual(200, channel.code, msg=channel.json_body) @@ -550,7 +516,6 @@ class DeleteDevicesRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) diff --git a/tests/rest/admin/test_event_reports.py b/tests/rest/admin/test_event_reports.py index 303622217f..11b72c10f7 100644 --- a/tests/rest/admin/test_event_reports.py +++ b/tests/rest/admin/test_event_reports.py @@ -75,7 +75,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): Try to get an event report without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -88,7 +87,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.other_user_tok, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -101,7 +99,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -117,7 +114,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -133,7 +129,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -149,7 +144,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=5&limit=10", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -167,7 +161,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): self.url + "?room_id=%s" % self.room_id1, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 10) @@ -188,7 +181,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): self.url + "?user_id=%s" % self.other_user, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 10) @@ -209,7 +201,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): self.url + "?user_id=%s&room_id=%s" % (self.other_user, self.room_id1), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 5) @@ -230,7 +221,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?dir=b", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -247,7 +237,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?dir=f", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -268,7 +257,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?dir=bar", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -282,7 +270,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=-5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -295,7 +282,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=-5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -310,7 +296,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=20", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -322,7 +307,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=21", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -334,7 +318,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=19", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -347,7 +330,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=19", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -366,7 +348,6 @@ class EventReportsTestCase(unittest.HomeserverTestCase): json.dumps({"score": -100, "reason": "this makes me sad"}), access_token=user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) def _check_fields(self, content): @@ -419,7 +400,6 @@ class EventReportDetailTestCase(unittest.HomeserverTestCase): Try to get event report without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -432,7 +412,6 @@ class EventReportDetailTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.other_user_tok, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -445,7 +424,6 @@ class EventReportDetailTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self._check_fields(channel.json_body) @@ -461,7 +439,6 @@ class EventReportDetailTestCase(unittest.HomeserverTestCase): "/_synapse/admin/v1/event_reports/-123", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -476,7 +453,6 @@ class EventReportDetailTestCase(unittest.HomeserverTestCase): "/_synapse/admin/v1/event_reports/abcdef", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -491,7 +467,6 @@ class EventReportDetailTestCase(unittest.HomeserverTestCase): "/_synapse/admin/v1/event_reports/", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -510,7 +485,6 @@ class EventReportDetailTestCase(unittest.HomeserverTestCase): "/_synapse/admin/v1/event_reports/123", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -528,7 +502,6 @@ class EventReportDetailTestCase(unittest.HomeserverTestCase): json.dumps({"score": -100, "reason": "this makes me sad"}), access_token=user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) def _check_fields(self, content): diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 64b7aa53ee..2a65ab33bd 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -51,7 +51,6 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345") request, channel = self.make_request("DELETE", url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -68,7 +67,6 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", url, access_token=self.other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -82,7 +80,6 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -96,7 +93,6 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only delete local media", channel.json_body["error"]) @@ -153,7 +149,6 @@ class DeleteMediaByIDTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) @@ -211,7 +206,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): """ request, channel = self.make_request("POST", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -226,7 +220,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", self.url, access_token=self.other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -240,7 +233,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url + "?before_ts=1234", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only delete local media", channel.json_body["error"]) @@ -252,7 +244,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"]) @@ -267,7 +258,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", self.url + "?before_ts=-1234", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -281,7 +271,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=1234&size_gt=-1234", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -295,7 +284,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=1234&keep_profiles=not_bool", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"]) @@ -326,7 +314,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( @@ -351,7 +338,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -364,7 +350,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( @@ -388,7 +373,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms) + "&size_gt=67", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -400,7 +384,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms) + "&size_gt=66", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( @@ -425,7 +408,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): content=json.dumps({"avatar_url": "mxc://%s" % (server_and_media_id,)}), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) now_ms = self.clock.time_msec() @@ -434,7 +416,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -446,7 +427,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( @@ -472,7 +452,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): content=json.dumps({"url": "mxc://%s" % (server_and_media_id,)}), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) now_ms = self.clock.time_msec() @@ -481,7 +460,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -493,7 +471,6 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) self.assertEqual( diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 535d68f284..54824a5410 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -85,7 +85,6 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase): json.dumps({"new_room_user_id": self.admin_user}), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) @@ -110,7 +109,6 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase): json.dumps({"history_visibility": "world_readable"}), access_token=self.other_user_token, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Test that the admin can still send shutdown @@ -121,7 +119,6 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase): json.dumps({"new_room_user_id": self.admin_user}), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) @@ -136,7 +133,6 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok ) - self.render(request) self.assertEqual( expect_code, int(channel.result["code"]), msg=channel.result["body"] ) @@ -145,7 +141,6 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok ) - self.render(request) self.assertEqual( expect_code, int(channel.result["code"]), msg=channel.result["body"] ) @@ -192,7 +187,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", self.url, json.dumps({}), access_token=self.other_user_tok, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -206,7 +200,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url, json.dumps({}), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -220,7 +213,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url, json.dumps({}), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( @@ -239,7 +231,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertIn("new_room_id", channel.json_body) @@ -259,7 +250,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( @@ -278,7 +268,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.BAD_JSON, channel.json_body["errcode"]) @@ -295,7 +284,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.BAD_JSON, channel.json_body["errcode"]) @@ -322,7 +310,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(None, channel.json_body["new_room_id"]) @@ -356,7 +343,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(None, channel.json_body["new_room_id"]) @@ -391,7 +377,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(None, channel.json_body["new_room_id"]) @@ -439,7 +424,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): json.dumps({"new_room_user_id": self.admin_user}), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(self.other_user, channel.json_body["kicked_users"][0]) @@ -470,7 +454,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): json.dumps({"history_visibility": "world_readable"}), access_token=self.other_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Test that room is not purged @@ -488,7 +471,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): json.dumps({"new_room_user_id": self.admin_user}), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(self.other_user, channel.json_body["kicked_users"][0]) @@ -551,7 +533,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok ) - self.render(request) self.assertEqual( expect_code, int(channel.result["code"]), msg=channel.result["body"] ) @@ -560,7 +541,6 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok ) - self.render(request) self.assertEqual( expect_code, int(channel.result["code"]), msg=channel.result["body"] ) @@ -595,7 +575,6 @@ class PurgeRoomTestCase(unittest.HomeserverTestCase): {"room_id": room_id}, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) @@ -647,7 +626,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) # Check request completed successfully self.assertEqual(200, int(channel.code), msg=channel.json_body) @@ -729,7 +707,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual( 200, int(channel.result["code"]), msg=channel.result["body"] ) @@ -770,7 +747,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) def test_correct_room_attributes(self): @@ -794,7 +770,6 @@ class RoomTestCase(unittest.HomeserverTestCase): {"room_id": room_id}, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Set this new alias as the canonical alias for this room @@ -822,7 +797,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Check that rooms were returned @@ -867,7 +841,6 @@ class RoomTestCase(unittest.HomeserverTestCase): {"room_id": room_id}, access_token=admin_user_tok, ) - self.render(request) self.assertEqual( 200, int(channel.result["code"]), msg=channel.result["body"] ) @@ -905,7 +878,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) # Check that rooms were returned @@ -1042,7 +1014,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(expected_http_code, channel.code, msg=channel.json_body) if expected_http_code != 200: @@ -1104,7 +1075,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertIn("room_id", channel.json_body) @@ -1152,7 +1122,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertCountEqual( @@ -1164,7 +1133,6 @@ class RoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertCountEqual( @@ -1208,7 +1176,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.second_tok, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -1225,7 +1192,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"]) @@ -1242,7 +1208,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -1259,7 +1224,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( @@ -1280,7 +1244,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("No known servers", channel.json_body["error"]) @@ -1298,7 +1261,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( @@ -1318,7 +1280,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(self.public_room_id, channel.json_body["room_id"]) @@ -1328,7 +1289,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/joined_rooms", access_token=self.second_tok, ) - self.render(request) self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(self.public_room_id, channel.json_body["joined_rooms"][0]) @@ -1349,7 +1309,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -1377,7 +1336,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/joined_rooms", access_token=self.admin_user_tok, ) - self.render(request) self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(private_room_id, channel.json_body["joined_rooms"][0]) @@ -1392,7 +1350,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(private_room_id, channel.json_body["room_id"]) @@ -1401,7 +1358,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/joined_rooms", access_token=self.second_tok, ) - self.render(request) self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(private_room_id, channel.json_body["joined_rooms"][0]) @@ -1422,7 +1378,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): content=body.encode(encoding="utf_8"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(private_room_id, channel.json_body["room_id"]) @@ -1432,7 +1387,6 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/joined_rooms", access_token=self.second_tok, ) - self.render(request) self.assertEquals(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(private_room_id, channel.json_body["joined_rooms"][0]) diff --git a/tests/rest/admin/test_statistics.py b/tests/rest/admin/test_statistics.py index 816683a612..907b49f889 100644 --- a/tests/rest/admin/test_statistics.py +++ b/tests/rest/admin/test_statistics.py @@ -47,7 +47,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): Try to list users without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -59,7 +58,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, json.dumps({}), access_token=self.other_user_tok, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -72,7 +70,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?order_by=bar", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -81,7 +78,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=-5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -90,7 +86,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=-5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -99,7 +94,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from_ts=-1234", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -108,7 +102,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?until_ts=-1234", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -119,7 +112,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): self.url + "?from_ts=10&until_ts=5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -128,7 +120,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?search_term=", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -137,7 +128,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?dir=bar", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -151,7 +141,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 10) @@ -168,7 +157,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -185,7 +173,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=5&limit=10", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -206,7 +193,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=20", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_users) @@ -218,7 +204,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=21", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_users) @@ -230,7 +215,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=19", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_users) @@ -242,7 +226,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=19", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_users) @@ -258,7 +241,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -337,7 +319,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["users"][0]["media_count"], 3) @@ -346,7 +327,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from_ts=%s" % (ts1,), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 0) @@ -362,7 +342,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): self.url + "?from_ts=%s&until_ts=%s" % (ts1, ts2), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["users"][0]["media_count"], 3) @@ -370,7 +349,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?until_ts=%s" % (ts2,), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["users"][0]["media_count"], 6) @@ -381,7 +359,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 20) @@ -391,7 +368,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): self.url + "?search_term=foo_user_1", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 11) @@ -401,7 +377,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): self.url + "?search_term=bar_user_10", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["users"][0]["displayname"], "bar_user_10") self.assertEqual(channel.json_body["total"], 1) @@ -410,7 +385,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?search_term=foobar", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], 0) @@ -476,7 +450,6 @@ class UserMediaStatisticsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url.encode("ascii"), access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(channel.json_body["total"], len(expected_user_list)) diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index d74efede06..5699c41eb4 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -71,7 +71,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): self.hs.config.registration_shared_secret = None request, channel = self.make_request("POST", self.url, b"{}") - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( @@ -89,7 +88,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): self.hs.get_secrets = Mock(return_value=secrets) request, channel = self.make_request("GET", self.url) - self.render(request) self.assertEqual(channel.json_body, {"nonce": "abcd"}) @@ -99,7 +97,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): only last for SALT_TIMEOUT (60s). """ request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] # 59 seconds @@ -107,7 +104,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): body = json.dumps({"nonce": nonce}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("username must be specified", channel.json_body["error"]) @@ -116,7 +112,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): self.reactor.advance(2) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("unrecognised nonce", channel.json_body["error"]) @@ -126,7 +121,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): Only the provided nonce can be used, as it's checked in the MAC. """ request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -143,7 +137,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("HMAC incorrect", channel.json_body["error"]) @@ -154,7 +147,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): user is registered. """ request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -174,7 +166,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["user_id"]) @@ -184,7 +175,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): A valid unrecognised nonce. """ request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -201,14 +191,12 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["user_id"]) # Now, try and reuse it request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("unrecognised nonce", channel.json_body["error"]) @@ -222,7 +210,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): def nonce(): request, channel = self.make_request("GET", self.url) - self.render(request) return channel.json_body["nonce"] # @@ -232,7 +219,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Must be present body = json.dumps({}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("nonce must be specified", channel.json_body["error"]) @@ -244,7 +230,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Must be present body = json.dumps({"nonce": nonce()}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("username must be specified", channel.json_body["error"]) @@ -252,7 +237,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Must be a string body = json.dumps({"nonce": nonce(), "username": 1234}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("Invalid username", channel.json_body["error"]) @@ -260,7 +244,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Must not have null bytes body = json.dumps({"nonce": nonce(), "username": "abcd\u0000"}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("Invalid username", channel.json_body["error"]) @@ -268,7 +251,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Must not have null bytes body = json.dumps({"nonce": nonce(), "username": "a" * 1000}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("Invalid username", channel.json_body["error"]) @@ -280,7 +262,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Must be present body = json.dumps({"nonce": nonce(), "username": "a"}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("password must be specified", channel.json_body["error"]) @@ -288,7 +269,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Must be a string body = json.dumps({"nonce": nonce(), "username": "a", "password": 1234}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("Invalid password", channel.json_body["error"]) @@ -296,7 +276,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Must not have null bytes body = json.dumps({"nonce": nonce(), "username": "a", "password": "abcd\u0000"}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("Invalid password", channel.json_body["error"]) @@ -304,7 +283,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Super long body = json.dumps({"nonce": nonce(), "username": "a", "password": "A" * 1000}) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("Invalid password", channel.json_body["error"]) @@ -323,7 +301,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("Invalid user type", channel.json_body["error"]) @@ -335,7 +312,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # set no displayname request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -346,19 +322,16 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): {"nonce": nonce, "username": "bob1", "password": "abc123", "mac": want_mac} ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob1:test", channel.json_body["user_id"]) request, channel = self.make_request("GET", "/profile/@bob1:test/displayname") - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("bob1", channel.json_body["displayname"]) # displayname is None request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -375,19 +348,16 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob2:test", channel.json_body["user_id"]) request, channel = self.make_request("GET", "/profile/@bob2:test/displayname") - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("bob2", channel.json_body["displayname"]) # displayname is empty request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -404,18 +374,15 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob3:test", channel.json_body["user_id"]) request, channel = self.make_request("GET", "/profile/@bob3:test/displayname") - self.render(request) self.assertEqual(404, int(channel.result["code"]), msg=channel.result["body"]) # set displayname request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -432,13 +399,11 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob4:test", channel.json_body["user_id"]) request, channel = self.make_request("GET", "/profile/@bob4:test/displayname") - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("Bob's Name", channel.json_body["displayname"]) @@ -465,7 +430,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): # Register new user with admin API request, channel = self.make_request("GET", self.url) - self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -485,7 +449,6 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request("POST", self.url, body.encode("utf8")) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["user_id"]) @@ -511,7 +474,6 @@ class UsersListTestCase(unittest.HomeserverTestCase): Try to list users without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("M_MISSING_TOKEN", channel.json_body["errcode"]) @@ -526,7 +488,6 @@ class UsersListTestCase(unittest.HomeserverTestCase): b"{}", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(3, len(channel.json_body["users"])) @@ -562,7 +523,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("You are not a server admin", channel.json_body["error"]) @@ -570,7 +530,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url, access_token=self.other_user_token, content=b"{}", ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("You are not a server admin", channel.json_body["error"]) @@ -585,7 +544,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): "/_synapse/admin/v2/users/@unknown_person:test", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual("M_NOT_FOUND", channel.json_body["errcode"]) @@ -613,7 +571,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -626,7 +583,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -659,7 +615,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -672,7 +627,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -700,7 +654,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/sync", access_token=self.admin_user_tok ) - self.render(request) if channel.code != 200: raise HttpResponseException( @@ -729,7 +682,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -769,7 +721,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) # Admin user is not blocked by mau anymore self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"]) @@ -807,7 +758,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -852,7 +802,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -879,7 +828,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) @@ -897,7 +845,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -907,7 +854,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url_other_user, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -929,7 +875,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -940,7 +885,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url_other_user, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -961,7 +905,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -972,7 +915,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url_other_user, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -990,7 +932,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=json.dumps({"deactivated": True}).encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self._is_erased("@user:test", False) d = self.store.mark_user_erased("@user:test") @@ -1004,7 +945,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=json.dumps({"deactivated": False}).encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) # Reactivate the user. @@ -1016,14 +956,12 @@ class UserRestTestCase(unittest.HomeserverTestCase): encoding="utf_8" ), ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Get user request, channel = self.make_request( "GET", self.url_other_user, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -1044,7 +982,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -1054,7 +991,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url_other_user, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@user:test", channel.json_body["name"]) @@ -1076,7 +1012,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(201, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -1086,7 +1021,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -1102,7 +1036,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): access_token=self.admin_user_tok, content=body.encode(encoding="utf_8"), ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) @@ -1110,7 +1043,6 @@ class UserRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["name"]) @@ -1153,7 +1085,6 @@ class UserMembershipRestTestCase(unittest.HomeserverTestCase): Try to list rooms of an user without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -1167,7 +1098,6 @@ class UserMembershipRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -1180,7 +1110,6 @@ class UserMembershipRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -1194,7 +1123,6 @@ class UserMembershipRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only lookup local users", channel.json_body["error"]) @@ -1208,7 +1136,6 @@ class UserMembershipRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -1228,7 +1155,6 @@ class UserMembershipRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(number_rooms, channel.json_body["total"]) @@ -1258,7 +1184,6 @@ class PushersRestTestCase(unittest.HomeserverTestCase): Try to list pushers of an user without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -1272,7 +1197,6 @@ class PushersRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -1285,7 +1209,6 @@ class PushersRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -1299,7 +1222,6 @@ class PushersRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only lookup local users", channel.json_body["error"]) @@ -1313,7 +1235,6 @@ class PushersRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -1343,7 +1264,6 @@ class PushersRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(1, channel.json_body["total"]) @@ -1383,7 +1303,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): Try to list media of an user without authentication. """ request, channel = self.make_request("GET", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -1397,7 +1316,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=other_user_token, ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -1410,7 +1328,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(404, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) @@ -1424,7 +1341,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, channel.code, msg=channel.json_body) self.assertEqual("Can only lookup local users", channel.json_body["error"]) @@ -1441,7 +1357,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_media) @@ -1461,7 +1376,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_media) @@ -1481,7 +1395,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=5&limit=10", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_media) @@ -1497,7 +1410,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=-5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -1510,7 +1422,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=-5", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) @@ -1529,7 +1440,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=20", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_media) @@ -1541,7 +1451,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=21", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_media) @@ -1553,7 +1462,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?limit=19", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_media) @@ -1566,7 +1474,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url + "?from=19", access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(channel.json_body["total"], number_media) @@ -1582,7 +1489,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(0, channel.json_body["total"]) @@ -1600,7 +1506,6 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url, access_token=self.admin_user_tok, ) - self.render(request) self.assertEqual(200, channel.code, msg=channel.json_body) self.assertEqual(number_media, channel.json_body["total"]) diff --git a/tests/rest/client/test_ephemeral_message.py b/tests/rest/client/test_ephemeral_message.py index 5e9c07ebf3..a1ccc4ee9a 100644 --- a/tests/rest/client/test_ephemeral_message.py +++ b/tests/rest/client/test_ephemeral_message.py @@ -94,7 +94,6 @@ class EphemeralMessageTestCase(unittest.HomeserverTestCase): url = "/_matrix/client/r0/rooms/%s/event/%s" % (room_id, event_id) request, channel = self.make_request("GET", url) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index c973521907..259c6a1985 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -46,7 +46,6 @@ class IdentityTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", "/createRoom", b"{}", access_token=tok ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) room_id = channel.json_body["room_id"] @@ -60,5 +59,4 @@ class IdentityTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", request_url, request_data, access_token=tok ) - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) diff --git a/tests/rest/client/test_redactions.py b/tests/rest/client/test_redactions.py index d2bcf256fa..c1f516cc93 100644 --- a/tests/rest/client/test_redactions.py +++ b/tests/rest/client/test_redactions.py @@ -72,7 +72,6 @@ class RedactionsTestCase(HomeserverTestCase): request, channel = self.make_request( "POST", path, content={}, access_token=access_token ) - self.render(request) self.assertEqual(int(channel.result["code"]), expect_code) return channel.json_body @@ -80,7 +79,6 @@ class RedactionsTestCase(HomeserverTestCase): request, channel = self.make_request( "GET", "sync", access_token=self.mod_access_token ) - self.render(request) self.assertEqual(channel.result["code"], b"200") room_sync = channel.json_body["rooms"]["join"][room_id] return room_sync["timeline"]["events"] diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index 7d3773ff78..f56b5d9231 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -326,7 +326,6 @@ class RetentionNoDefaultPolicyTestCase(unittest.HomeserverTestCase): url = "/_matrix/client/r0/rooms/%s/event/%s" % (room_id, event_id) request, channel = self.make_request("GET", url, access_token=self.token) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) diff --git a/tests/rest/client/test_shadow_banned.py b/tests/rest/client/test_shadow_banned.py index 6bb02b9630..94dcfb9f7c 100644 --- a/tests/rest/client/test_shadow_banned.py +++ b/tests/rest/client/test_shadow_banned.py @@ -95,7 +95,6 @@ class RoomTestCase(_ShadowBannedBase): {"id_server": "test", "medium": "email", "address": "test@test.test"}, access_token=self.banned_access_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.result) # This should have raised an error earlier, but double check this wasn't called. @@ -110,7 +109,6 @@ class RoomTestCase(_ShadowBannedBase): {"visibility": "public", "invite": [self.other_user_id]}, access_token=self.banned_access_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.result) room_id = channel.json_body["room_id"] @@ -166,7 +164,6 @@ class RoomTestCase(_ShadowBannedBase): {"new_version": "6"}, access_token=self.banned_access_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.result) # A new room_id should be returned. self.assertIn("replacement_room", channel.json_body) @@ -192,7 +189,6 @@ class RoomTestCase(_ShadowBannedBase): {"typing": True, "timeout": 30000}, access_token=self.banned_access_token, ) - self.render(request) self.assertEquals(200, channel.code) # There should be no typing events. @@ -208,7 +204,6 @@ class RoomTestCase(_ShadowBannedBase): {"typing": True, "timeout": 30000}, access_token=self.other_access_token, ) - self.render(request) self.assertEquals(200, channel.code) # These appear in the room. @@ -255,7 +250,6 @@ class ProfileTestCase(_ShadowBannedBase): {"displayname": new_display_name}, access_token=self.banned_access_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertEqual(channel.json_body, {}) @@ -263,7 +257,6 @@ class ProfileTestCase(_ShadowBannedBase): request, channel = self.make_request( "GET", "/profile/%s/displayname" % (self.banned_user_id,) ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self.assertEqual(channel.json_body["displayname"], new_display_name) @@ -296,7 +289,6 @@ class ProfileTestCase(_ShadowBannedBase): {"membership": "join", "displayname": new_display_name}, access_token=self.banned_access_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertIn("event_id", channel.json_body) diff --git a/tests/rest/client/test_third_party_rules.py b/tests/rest/client/test_third_party_rules.py index 0048bea54a..0e96697f9b 100644 --- a/tests/rest/client/test_third_party_rules.py +++ b/tests/rest/client/test_third_party_rules.py @@ -92,7 +92,6 @@ class ThirdPartyRulesTestCase(unittest.HomeserverTestCase): {}, access_token=self.tok, ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) callback.assert_called_once() @@ -111,7 +110,6 @@ class ThirdPartyRulesTestCase(unittest.HomeserverTestCase): {}, access_token=self.tok, ) - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) def test_cannot_modify_event(self): @@ -131,7 +129,6 @@ class ThirdPartyRulesTestCase(unittest.HomeserverTestCase): {"x": "x"}, access_token=self.tok, ) - self.render(request) self.assertEqual(channel.result["code"], b"500", channel.result) def test_modify_event(self): @@ -151,7 +148,6 @@ class ThirdPartyRulesTestCase(unittest.HomeserverTestCase): {"x": "x"}, access_token=self.tok, ) - self.render(request) self.assertEqual(channel.result["code"], b"200", channel.result) event_id = channel.json_body["event_id"] @@ -161,7 +157,6 @@ class ThirdPartyRulesTestCase(unittest.HomeserverTestCase): "/_matrix/client/r0/rooms/%s/event/%s" % (self.room_id, event_id), access_token=self.tok, ) - self.render(request) self.assertEqual(channel.result["code"], b"200", channel.result) ev = channel.json_body self.assertEqual(ev["content"]["x"], "y") diff --git a/tests/rest/client/v1/test_directory.py b/tests/rest/client/v1/test_directory.py index ea5a7f3739..7a2c653df8 100644 --- a/tests/rest/client/v1/test_directory.py +++ b/tests/rest/client/v1/test_directory.py @@ -94,7 +94,6 @@ class DirectoryTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url, request_data, access_token=self.user_tok ) - self.render(request) self.assertEqual(channel.code, 400, channel.result) def test_room_creation(self): @@ -108,7 +107,6 @@ class DirectoryTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", url, request_data, access_token=self.user_tok ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) def set_alias_via_state_event(self, expected_code, alias_length=5): @@ -123,7 +121,6 @@ class DirectoryTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url, request_data, access_token=self.user_tok ) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) def set_alias_via_directory(self, expected_code, alias_length=5): @@ -134,7 +131,6 @@ class DirectoryTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url, request_data, access_token=self.user_tok ) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) def random_alias(self, length): diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index 3397ba5579..12a93f5687 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -66,14 +66,12 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/events?access_token=%s" % ("invalid" + self.token,) ) - self.render(request) self.assertEquals(channel.code, 401, msg=channel.result) # valid token, expect content request, channel = self.make_request( "GET", "/events?access_token=%s&timeout=0" % (self.token,) ) - self.render(request) self.assertEquals(channel.code, 200, msg=channel.result) self.assertTrue("chunk" in channel.json_body) self.assertTrue("start" in channel.json_body) @@ -92,7 +90,6 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/events?access_token=%s&timeout=0" % (self.token,) ) - self.render(request) self.assertEquals(channel.code, 200, msg=channel.result) # We may get a presence event for ourselves down @@ -155,5 +152,4 @@ class GetEventsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/events/" + event_id, access_token=self.token, ) - self.render(request) self.assertEquals(channel.code, 200, msg=channel.result) diff --git a/tests/rest/client/v1/test_login.py b/tests/rest/client/v1/test_login.py index 5d987a30c7..176ddf7ec9 100644 --- a/tests/rest/client/v1/test_login.py +++ b/tests/rest/client/v1/test_login.py @@ -64,7 +64,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): "password": "monkey", } request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) if i == 5: self.assertEquals(channel.result["code"], b"429", channel.result) @@ -84,7 +83,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): "password": "monkey", } request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) @@ -111,7 +109,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): "password": "monkey", } request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) if i == 5: self.assertEquals(channel.result["code"], b"429", channel.result) @@ -131,7 +128,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): "password": "monkey", } request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) @@ -158,7 +154,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): "password": "notamonkey", } request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) if i == 5: self.assertEquals(channel.result["code"], b"429", channel.result) @@ -178,7 +173,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): "password": "notamonkey", } request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) @@ -188,7 +182,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): # we shouldn't be able to make requests without an access token request, channel = self.make_request(b"GET", TEST_URL) - self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) self.assertEquals(channel.json_body["errcode"], "M_MISSING_TOKEN") @@ -199,7 +192,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): "password": "monkey", } request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) self.assertEquals(channel.code, 200, channel.result) access_token = channel.json_body["access_token"] @@ -209,7 +201,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"GET", TEST_URL, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 200, channel.result) # time passes @@ -219,7 +210,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"GET", TEST_URL, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 401, channel.result) self.assertEquals(channel.json_body["errcode"], "M_UNKNOWN_TOKEN") self.assertEquals(channel.json_body["soft_logout"], True) @@ -236,7 +226,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"GET", TEST_URL, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 401, channel.result) self.assertEquals(channel.json_body["errcode"], "M_UNKNOWN_TOKEN") self.assertEquals(channel.json_body["soft_logout"], True) @@ -247,7 +236,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"GET", TEST_URL, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 401, channel.result) self.assertEquals(channel.json_body["errcode"], "M_UNKNOWN_TOKEN") self.assertEquals(channel.json_body["soft_logout"], False) @@ -257,7 +245,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"DELETE", "devices/" + device_id, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 401, channel.result) # check it's a UI-Auth fail self.assertEqual( @@ -281,7 +268,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): access_token=access_token, content={"auth": auth}, ) - self.render(request) self.assertEquals(channel.code, 200, channel.result) @override_config({"session_lifetime": "24h"}) @@ -295,7 +281,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"GET", TEST_URL, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 200, channel.result) # time passes @@ -305,7 +290,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"GET", TEST_URL, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 401, channel.result) self.assertEquals(channel.json_body["errcode"], "M_UNKNOWN_TOKEN") self.assertEquals(channel.json_body["soft_logout"], True) @@ -314,7 +298,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", "/logout", access_token=access_token ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) @override_config({"session_lifetime": "24h"}) @@ -328,7 +311,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"GET", TEST_URL, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 200, channel.result) # time passes @@ -338,7 +320,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"GET", TEST_URL, access_token=access_token ) - self.render(request) self.assertEquals(channel.code, 401, channel.result) self.assertEquals(channel.json_body["errcode"], "M_UNKNOWN_TOKEN") self.assertEquals(channel.json_body["soft_logout"], True) @@ -347,7 +328,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", "/logout/all", access_token=access_token ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) @@ -423,7 +403,6 @@ class CASTestCase(unittest.HomeserverTestCase): # Get Synapse to call the fake CAS and serve the template. request, channel = self.make_request("GET", cas_ticket_url) - self.render(request) # Test that the response is HTML. self.assertEqual(channel.code, 200) @@ -468,7 +447,6 @@ class CASTestCase(unittest.HomeserverTestCase): # Get Synapse to call the fake CAS and serve the template. request, channel = self.make_request("GET", cas_ticket_url) - self.render(request) self.assertEqual(channel.code, 302) location_headers = channel.headers.getRawHeaders("Location") @@ -495,7 +473,6 @@ class CASTestCase(unittest.HomeserverTestCase): # Get Synapse to call the fake CAS and serve the template. request, channel = self.make_request("GET", cas_ticket_url) - self.render(request) # Because the user is deactivated they are served an error template. self.assertEqual(channel.code, 403) @@ -526,7 +503,6 @@ class JWTTestCase(unittest.HomeserverTestCase): {"type": "org.matrix.login.jwt", "token": self.jwt_encode(*args)} ) request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) return channel def test_login_jwt_valid_registered(self): @@ -659,7 +635,6 @@ class JWTTestCase(unittest.HomeserverTestCase): def test_login_no_token(self): params = json.dumps({"type": "org.matrix.login.jwt"}) request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) self.assertEqual(channel.result["code"], b"403", channel.result) self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") self.assertEqual(channel.json_body["error"], "Token field for JWT is missing") @@ -733,7 +708,6 @@ class JWTPubKeyTestCase(unittest.HomeserverTestCase): {"type": "org.matrix.login.jwt", "token": self.jwt_encode(*args)} ) request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) return channel def test_login_jwt_valid(self): @@ -766,7 +740,6 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): "/_matrix/client/r0/register?access_token=%s" % (self.service.token,), {"username": username}, ) - self.render(request) def make_homeserver(self, reactor, clock): self.hs = self.setup_test_homeserver() @@ -815,7 +788,6 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): b"POST", LOGIN_URL, params, access_token=self.service.token ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) def test_login_appservice_user_bot(self): @@ -831,7 +803,6 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): b"POST", LOGIN_URL, params, access_token=self.service.token ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) def test_login_appservice_wrong_user(self): @@ -847,7 +818,6 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): b"POST", LOGIN_URL, params, access_token=self.service.token ) - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) def test_login_appservice_wrong_as(self): @@ -863,7 +833,6 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): b"POST", LOGIN_URL, params, access_token=self.another_service.token ) - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) def test_login_appservice_no_token(self): @@ -878,5 +847,4 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): } request, channel = self.make_request(b"POST", LOGIN_URL, params) - self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 3c66255dac..b84f86d28c 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -53,7 +53,6 @@ class PresenceTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", "/presence/%s/status" % (self.user_id,), body ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual(self.hs.presence_handler.set_state.call_count, 1) @@ -69,7 +68,6 @@ class PresenceTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", "/presence/%s/status" % (self.user_id,), body ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual(self.hs.presence_handler.set_state.call_count, 0) diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index ace0a3c08d..383a9eafac 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -195,7 +195,6 @@ class ProfileTestCase(unittest.HomeserverTestCase): content=json.dumps({"displayname": "test"}), access_token=self.owner_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) res = self.get_displayname() @@ -209,7 +208,6 @@ class ProfileTestCase(unittest.HomeserverTestCase): content=json.dumps({"displayname": "test" * 100}), access_token=self.owner_tok, ) - self.render(request) self.assertEqual(channel.code, 400, channel.result) res = self.get_displayname() @@ -219,7 +217,6 @@ class ProfileTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/profile/%s/displayname" % (self.owner,) ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) return channel.json_body["displayname"] @@ -284,7 +281,6 @@ class ProfilesRestrictedTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.profile_url + url_suffix, access_token=access_token ) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) def ensure_requester_left_room(self): @@ -327,7 +323,6 @@ class OwnProfileUnrestrictedTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/profile/" + self.requester, access_token=self.requester_tok ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) request, channel = self.make_request( @@ -335,7 +330,6 @@ class OwnProfileUnrestrictedTestCase(unittest.HomeserverTestCase): "/profile/" + self.requester + "/displayname", access_token=self.requester_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) request, channel = self.make_request( @@ -343,5 +337,4 @@ class OwnProfileUnrestrictedTestCase(unittest.HomeserverTestCase): "/profile/" + self.requester + "/avatar_url", access_token=self.requester_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) diff --git a/tests/rest/client/v1/test_push_rule_attrs.py b/tests/rest/client/v1/test_push_rule_attrs.py index 081052f6a6..7add5523c8 100644 --- a/tests/rest/client/v1/test_push_rule_attrs.py +++ b/tests/rest/client/v1/test_push_rule_attrs.py @@ -48,14 +48,12 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "PUT", "/pushrules/global/override/best.friend", body, access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # GET enabled for that new rule request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["enabled"], True) @@ -79,7 +77,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "PUT", "/pushrules/global/override/best.friend", body, access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # disable the rule @@ -89,14 +86,12 @@ class PushRuleAttributesTestCase(HomeserverTestCase): {"enabled": False}, access_token=token, ) - self.render(request) self.assertEqual(channel.code, 200) # check rule disabled request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["enabled"], False) @@ -104,21 +99,18 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "DELETE", "/pushrules/global/override/best.friend", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # PUT a new rule request, channel = self.make_request( "PUT", "/pushrules/global/override/best.friend", body, access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # GET enabled for that new rule request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["enabled"], True) @@ -141,7 +133,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "PUT", "/pushrules/global/override/best.friend", body, access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # disable the rule @@ -151,14 +142,12 @@ class PushRuleAttributesTestCase(HomeserverTestCase): {"enabled": False}, access_token=token, ) - self.render(request) self.assertEqual(channel.code, 200) # check rule disabled request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["enabled"], False) @@ -169,14 +158,12 @@ class PushRuleAttributesTestCase(HomeserverTestCase): {"enabled": True}, access_token=token, ) - self.render(request) self.assertEqual(channel.code, 200) # check rule enabled request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["enabled"], True) @@ -198,7 +185,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -206,28 +192,24 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "PUT", "/pushrules/global/override/best.friend", body, access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # GET enabled for that new rule request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # DELETE the rule request, channel = self.make_request( "DELETE", "/pushrules/global/override/best.friend", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # check 404 for deleted rule request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -242,7 +224,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "GET", "/pushrules/global/override/.m.muahahaha/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -260,7 +241,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): {"enabled": True}, access_token=token, ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -278,7 +258,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): {"enabled": True}, access_token=token, ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -300,14 +279,12 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "PUT", "/pushrules/global/override/best.friend", body, access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # GET actions for that new rule request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/actions", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual( channel.json_body["actions"], ["notify", {"set_tweak": "highlight"}] @@ -331,7 +308,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "PUT", "/pushrules/global/override/best.friend", body, access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # change the rule actions @@ -341,14 +317,12 @@ class PushRuleAttributesTestCase(HomeserverTestCase): {"actions": ["dont_notify"]}, access_token=token, ) - self.render(request) self.assertEqual(channel.code, 200) # GET actions for that new rule request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/actions", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["actions"], ["dont_notify"]) @@ -370,7 +344,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -378,21 +351,18 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "PUT", "/pushrules/global/override/best.friend", body, access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # DELETE the rule request, channel = self.make_request( "DELETE", "/pushrules/global/override/best.friend", access_token=token ) - self.render(request) self.assertEqual(channel.code, 200) # check 404 for deleted rule request, channel = self.make_request( "GET", "/pushrules/global/override/best.friend/enabled", access_token=token ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -407,7 +377,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): request, channel = self.make_request( "GET", "/pushrules/global/override/.m.muahahaha/actions", access_token=token ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -425,7 +394,6 @@ class PushRuleAttributesTestCase(HomeserverTestCase): {"actions": ["dont_notify"]}, access_token=token, ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -443,6 +411,5 @@ class PushRuleAttributesTestCase(HomeserverTestCase): {"actions": ["dont_notify"]}, access_token=token, ) - self.render(request) self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 9ba5f9d943..49f1073c88 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -86,7 +86,6 @@ class RoomPermissionsTestCase(RoomBase): request, channel = self.make_request( "PUT", self.created_rmid_msg_path, b'{"msgtype":"m.text","body":"test msg"}' ) - self.render(request) self.assertEquals(200, channel.code, channel.result) # set topic for public room @@ -95,7 +94,6 @@ class RoomPermissionsTestCase(RoomBase): ("rooms/%s/state/m.room.topic" % self.created_public_rmid).encode("ascii"), b'{"topic":"Public Room Topic"}', ) - self.render(request) self.assertEquals(200, channel.code, channel.result) # auth as user_id now @@ -118,12 +116,10 @@ class RoomPermissionsTestCase(RoomBase): "/rooms/%s/send/m.room.message/mid2" % (self.uncreated_rmid,), msg_content, ) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) # send message in created room not joined (no state), expect 403 request, channel = self.make_request("PUT", send_msg_path(), msg_content) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) # send message in created room and invited, expect 403 @@ -131,19 +127,16 @@ class RoomPermissionsTestCase(RoomBase): room=self.created_rmid, src=self.rmcreator_id, targ=self.user_id ) request, channel = self.make_request("PUT", send_msg_path(), msg_content) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) # send message in created room and joined, expect 200 self.helper.join(room=self.created_rmid, user=self.user_id) request, channel = self.make_request("PUT", send_msg_path(), msg_content) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) # send message in created room and left, expect 403 self.helper.leave(room=self.created_rmid, user=self.user_id) request, channel = self.make_request("PUT", send_msg_path(), msg_content) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) def test_topic_perms(self): @@ -154,20 +147,16 @@ class RoomPermissionsTestCase(RoomBase): request, channel = self.make_request( "PUT", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid, topic_content ) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) request, channel = self.make_request( "GET", "/rooms/%s/state/m.room.topic" % self.uncreated_rmid ) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) # set/get topic in created PRIVATE room not joined, expect 403 request, channel = self.make_request("PUT", topic_path, topic_content) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) request, channel = self.make_request("GET", topic_path) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) # set topic in created PRIVATE room and invited, expect 403 @@ -175,12 +164,10 @@ class RoomPermissionsTestCase(RoomBase): room=self.created_rmid, src=self.rmcreator_id, targ=self.user_id ) request, channel = self.make_request("PUT", topic_path, topic_content) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) # get topic in created PRIVATE room and invited, expect 403 request, channel = self.make_request("GET", topic_path) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) # set/get topic in created PRIVATE room and joined, expect 200 @@ -189,29 +176,24 @@ class RoomPermissionsTestCase(RoomBase): # Only room ops can set topic by default self.helper.auth_user_id = self.rmcreator_id request, channel = self.make_request("PUT", topic_path, topic_content) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) self.helper.auth_user_id = self.user_id request, channel = self.make_request("GET", topic_path) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) self.assert_dict(json.loads(topic_content.decode("utf8")), channel.json_body) # set/get topic in created PRIVATE room and left, expect 403 self.helper.leave(room=self.created_rmid, user=self.user_id) request, channel = self.make_request("PUT", topic_path, topic_content) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) request, channel = self.make_request("GET", topic_path) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) # get topic in PUBLIC room, not joined, expect 403 request, channel = self.make_request( "GET", "/rooms/%s/state/m.room.topic" % self.created_public_rmid ) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) # set topic in PUBLIC room, not joined, expect 403 @@ -220,14 +202,12 @@ class RoomPermissionsTestCase(RoomBase): "/rooms/%s/state/m.room.topic" % self.created_public_rmid, topic_content, ) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) def _test_get_membership(self, room=None, members=[], expect_code=None): for member in members: path = "/rooms/%s/state/m.room.member/%s" % (room, member) request, channel = self.make_request("GET", path) - self.render(request) self.assertEquals(expect_code, channel.code) def test_membership_basic_room_perms(self): @@ -400,18 +380,15 @@ class RoomsMemberListTestCase(RoomBase): def test_get_member_list(self): room_id = self.helper.create_room_as(self.user_id) request, channel = self.make_request("GET", "/rooms/%s/members" % room_id) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) def test_get_member_list_no_room(self): request, channel = self.make_request("GET", "/rooms/roomdoesnotexist/members") - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) def test_get_member_list_no_permission(self): room_id = self.helper.create_room_as("@some_other_guy:red") request, channel = self.make_request("GET", "/rooms/%s/members" % room_id) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) def test_get_member_list_mixed_memberships(self): @@ -421,19 +398,16 @@ class RoomsMemberListTestCase(RoomBase): self.helper.invite(room=room_id, src=room_creator, targ=self.user_id) # can't see list if you're just invited. request, channel = self.make_request("GET", room_path) - self.render(request) self.assertEquals(403, channel.code, msg=channel.result["body"]) self.helper.join(room=room_id, user=self.user_id) # can see list now joined request, channel = self.make_request("GET", room_path) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) self.helper.leave(room=room_id, user=self.user_id) # can see old list once left request, channel = self.make_request("GET", room_path) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) @@ -446,7 +420,6 @@ class RoomsCreateTestCase(RoomBase): # POST with no config keys, expect new room id request, channel = self.make_request("POST", "/createRoom", "{}") - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertTrue("room_id" in channel.json_body) @@ -455,7 +428,6 @@ class RoomsCreateTestCase(RoomBase): request, channel = self.make_request( "POST", "/createRoom", b'{"visibility":"private"}' ) - self.render(request) self.assertEquals(200, channel.code) self.assertTrue("room_id" in channel.json_body) @@ -464,7 +436,6 @@ class RoomsCreateTestCase(RoomBase): request, channel = self.make_request( "POST", "/createRoom", b'{"custom":"stuff"}' ) - self.render(request) self.assertEquals(200, channel.code) self.assertTrue("room_id" in channel.json_body) @@ -473,18 +444,15 @@ class RoomsCreateTestCase(RoomBase): request, channel = self.make_request( "POST", "/createRoom", b'{"visibility":"private","custom":"things"}' ) - self.render(request) self.assertEquals(200, channel.code) self.assertTrue("room_id" in channel.json_body) def test_post_room_invalid_content(self): # POST with invalid content / paths, expect 400 request, channel = self.make_request("POST", "/createRoom", b'{"visibili') - self.render(request) self.assertEquals(400, channel.code) request, channel = self.make_request("POST", "/createRoom", b'["hello"]') - self.render(request) self.assertEquals(400, channel.code) def test_post_room_invitees_invalid_mxid(self): @@ -493,7 +461,6 @@ class RoomsCreateTestCase(RoomBase): request, channel = self.make_request( "POST", "/createRoom", b'{"invite":["@alice:example.com "]}' ) - self.render(request) self.assertEquals(400, channel.code) @@ -510,52 +477,42 @@ class RoomTopicTestCase(RoomBase): def test_invalid_puts(self): # missing keys or invalid json request, channel = self.make_request("PUT", self.path, "{}") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", self.path, '{"_name":"bo"}') - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", self.path, '{"nao') - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request( "PUT", self.path, '[{"_name":"bo"},{"_name":"jill"}]' ) - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", self.path, "text only") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", self.path, "") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) # valid key, wrong type content = '{"topic":["Topic name"]}' request, channel = self.make_request("PUT", self.path, content) - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) def test_rooms_topic(self): # nothing should be there request, channel = self.make_request("GET", self.path) - self.render(request) self.assertEquals(404, channel.code, msg=channel.result["body"]) # valid put content = '{"topic":"Topic name"}' request, channel = self.make_request("PUT", self.path, content) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) # valid get request, channel = self.make_request("GET", self.path) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) self.assert_dict(json.loads(content), channel.json_body) @@ -563,12 +520,10 @@ class RoomTopicTestCase(RoomBase): # valid put with extra keys content = '{"topic":"Seasons","subtopic":"Summer"}' request, channel = self.make_request("PUT", self.path, content) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) # valid get request, channel = self.make_request("GET", self.path) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) self.assert_dict(json.loads(content), channel.json_body) @@ -585,29 +540,23 @@ class RoomMemberStateTestCase(RoomBase): path = "/rooms/%s/state/m.room.member/%s" % (self.room_id, self.user_id) # missing keys or invalid json request, channel = self.make_request("PUT", path, "{}") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", path, '{"_name":"bo"}') - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", path, '{"nao') - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request( "PUT", path, b'[{"_name":"bo"},{"_name":"jill"}]' ) - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", path, "text only") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", path, "") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) # valid keys, wrong types @@ -617,7 +566,6 @@ class RoomMemberStateTestCase(RoomBase): Membership.LEAVE, ) request, channel = self.make_request("PUT", path, content.encode("ascii")) - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) def test_rooms_members_self(self): @@ -629,11 +577,9 @@ class RoomMemberStateTestCase(RoomBase): # valid join message (NOOP since we made the room) content = '{"membership":"%s"}' % Membership.JOIN request, channel = self.make_request("PUT", path, content.encode("ascii")) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) request, channel = self.make_request("GET", path, None) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) expected_response = {"membership": Membership.JOIN} @@ -649,11 +595,9 @@ class RoomMemberStateTestCase(RoomBase): # valid invite message content = '{"membership":"%s"}' % Membership.INVITE request, channel = self.make_request("PUT", path, content) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) request, channel = self.make_request("GET", path, None) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) self.assertEquals(json.loads(content), channel.json_body) @@ -670,11 +614,9 @@ class RoomMemberStateTestCase(RoomBase): "Join us!", ) request, channel = self.make_request("PUT", path, content) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) request, channel = self.make_request("GET", path, None) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) self.assertEquals(json.loads(content), channel.json_body) @@ -725,7 +667,6 @@ class RoomJoinRatelimitTestCase(RoomBase): # Update the display name for the user. path = "/_matrix/client/r0/profile/%s/displayname" % self.user_id request, channel = self.make_request("PUT", path, {"displayname": "John Doe"}) - self.render(request) self.assertEquals(channel.code, 200, channel.json_body) # Check that all the rooms have been sent a profile update into. @@ -736,7 +677,6 @@ class RoomJoinRatelimitTestCase(RoomBase): ) request, channel = self.make_request("GET", path) - self.render(request) self.assertEquals(channel.code, 200) self.assertIn("displayname", channel.json_body) @@ -761,7 +701,6 @@ class RoomJoinRatelimitTestCase(RoomBase): # if all of these requests ended up joining the user to a room. for i in range(4): request, channel = self.make_request("POST", path % room_id, {}) - self.render(request) self.assertEquals(channel.code, 200) @@ -777,29 +716,23 @@ class RoomMessagesTestCase(RoomBase): path = "/rooms/%s/send/m.room.message/mid1" % (urlparse.quote(self.room_id)) # missing keys or invalid json request, channel = self.make_request("PUT", path, b"{}") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", path, b'{"_name":"bo"}') - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", path, b'{"nao') - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request( "PUT", path, b'[{"_name":"bo"},{"_name":"jill"}]' ) - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", path, b"text only") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) request, channel = self.make_request("PUT", path, b"") - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) def test_rooms_messages_sent(self): @@ -807,20 +740,17 @@ class RoomMessagesTestCase(RoomBase): content = b'{"body":"test","msgtype":{"type":"a"}}' request, channel = self.make_request("PUT", path, content) - self.render(request) self.assertEquals(400, channel.code, msg=channel.result["body"]) # custom message types content = b'{"body":"test","msgtype":"test.custom.text"}' request, channel = self.make_request("PUT", path, content) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) # m.text message type path = "/rooms/%s/send/m.room.message/mid2" % (urlparse.quote(self.room_id)) content = b'{"body":"test2","msgtype":"m.text"}' request, channel = self.make_request("PUT", path, content) - self.render(request) self.assertEquals(200, channel.code, msg=channel.result["body"]) @@ -837,7 +767,6 @@ class RoomInitialSyncTestCase(RoomBase): request, channel = self.make_request( "GET", "/rooms/%s/initialSync" % self.room_id ) - self.render(request) self.assertEquals(200, channel.code) self.assertEquals(self.room_id, channel.json_body["room_id"]) @@ -881,7 +810,6 @@ class RoomMessageListTestCase(RoomBase): request, channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) - self.render(request) self.assertEquals(200, channel.code) self.assertTrue("start" in channel.json_body) self.assertEquals(token, channel.json_body["start"]) @@ -893,7 +821,6 @@ class RoomMessageListTestCase(RoomBase): request, channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) - self.render(request) self.assertEquals(200, channel.code) self.assertTrue("start" in channel.json_body) self.assertEquals(token, channel.json_body["start"]) @@ -933,7 +860,6 @@ class RoomMessageListTestCase(RoomBase): json.dumps({"types": [EventTypes.Message]}), ), ) - self.render(request) self.assertEqual(channel.code, 200, channel.json_body) chunk = channel.json_body["chunk"] @@ -962,7 +888,6 @@ class RoomMessageListTestCase(RoomBase): json.dumps({"types": [EventTypes.Message]}), ), ) - self.render(request) self.assertEqual(channel.code, 200, channel.json_body) chunk = channel.json_body["chunk"] @@ -980,7 +905,6 @@ class RoomMessageListTestCase(RoomBase): json.dumps({"types": [EventTypes.Message]}), ), ) - self.render(request) self.assertEqual(channel.code, 200, channel.json_body) chunk = channel.json_body["chunk"] @@ -1040,7 +964,6 @@ class RoomSearchTestCase(unittest.HomeserverTestCase): } }, ) - self.render(request) # Check we get the results we expect -- one search result, of the sent # messages @@ -1074,7 +997,6 @@ class RoomSearchTestCase(unittest.HomeserverTestCase): } }, ) - self.render(request) # Check we get the results we expect -- one search result, of the sent # messages @@ -1111,7 +1033,6 @@ class PublicRoomsRestrictedTestCase(unittest.HomeserverTestCase): def test_restricted_no_auth(self): request, channel = self.make_request("GET", self.url) - self.render(request) self.assertEqual(channel.code, 401, channel.result) def test_restricted_auth(self): @@ -1119,7 +1040,6 @@ class PublicRoomsRestrictedTestCase(unittest.HomeserverTestCase): tok = self.login("user", "pass") request, channel = self.make_request("GET", self.url, access_token=tok) - self.render(request) self.assertEqual(channel.code, 200, channel.result) @@ -1153,7 +1073,6 @@ class PerRoomProfilesForbiddenTestCase(unittest.HomeserverTestCase): request_data, access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok) @@ -1168,7 +1087,6 @@ class PerRoomProfilesForbiddenTestCase(unittest.HomeserverTestCase): request_data, access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) event_id = channel.json_body["event_id"] @@ -1177,7 +1095,6 @@ class PerRoomProfilesForbiddenTestCase(unittest.HomeserverTestCase): "/_matrix/client/r0/rooms/%s/event/%s" % (self.room_id, event_id), access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) res_displayname = channel.json_body["content"]["displayname"] @@ -1212,7 +1129,6 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase): content={"reason": reason}, access_token=self.second_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self._check_for_reason(reason) @@ -1227,7 +1143,6 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase): content={"reason": reason}, access_token=self.second_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self._check_for_reason(reason) @@ -1242,7 +1157,6 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase): content={"reason": reason, "user_id": self.second_user_id}, access_token=self.second_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self._check_for_reason(reason) @@ -1257,7 +1171,6 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase): content={"reason": reason, "user_id": self.second_user_id}, access_token=self.creator_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self._check_for_reason(reason) @@ -1270,7 +1183,6 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase): content={"reason": reason, "user_id": self.second_user_id}, access_token=self.creator_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self._check_for_reason(reason) @@ -1283,7 +1195,6 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase): content={"reason": reason, "user_id": self.second_user_id}, access_token=self.creator_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self._check_for_reason(reason) @@ -1303,7 +1214,6 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase): content={"reason": reason}, access_token=self.second_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self._check_for_reason(reason) @@ -1316,7 +1226,6 @@ class RoomMembershipReasonTestCase(unittest.HomeserverTestCase): ), access_token=self.creator_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) event_content = channel.json_body @@ -1365,7 +1274,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): % (self.room_id, event_id, json.dumps(self.FILTER_LABELS)), access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) events_before = channel.json_body["events_before"] @@ -1396,7 +1304,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): % (self.room_id, event_id, json.dumps(self.FILTER_NOT_LABELS)), access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) events_before = channel.json_body["events_before"] @@ -1432,7 +1339,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): % (self.room_id, event_id, json.dumps(self.FILTER_LABELS_NOT_LABELS)), access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) events_before = channel.json_body["events_before"] @@ -1460,7 +1366,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): "/rooms/%s/messages?access_token=%s&from=%s&filter=%s" % (self.room_id, self.tok, token, json.dumps(self.FILTER_LABELS)), ) - self.render(request) events = channel.json_body["chunk"] @@ -1478,7 +1383,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): "/rooms/%s/messages?access_token=%s&from=%s&filter=%s" % (self.room_id, self.tok, token, json.dumps(self.FILTER_NOT_LABELS)), ) - self.render(request) events = channel.json_body["chunk"] @@ -1507,7 +1411,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): json.dumps(self.FILTER_LABELS_NOT_LABELS), ), ) - self.render(request) events = channel.json_body["chunk"] @@ -1532,7 +1435,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", "/search?access_token=%s" % self.tok, request_data ) - self.render(request) results = channel.json_body["search_categories"]["room_events"]["results"] @@ -1568,7 +1470,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", "/search?access_token=%s" % self.tok, request_data ) - self.render(request) results = channel.json_body["search_categories"]["room_events"]["results"] @@ -1616,7 +1517,6 @@ class LabelsTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", "/search?access_token=%s" % self.tok, request_data ) - self.render(request) results = channel.json_body["search_categories"]["room_events"]["results"] @@ -1741,7 +1641,6 @@ class ContextTestCase(unittest.HomeserverTestCase): % (self.room_id, event_id), access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) events_before = channel.json_body["events_before"] @@ -1806,7 +1705,6 @@ class ContextTestCase(unittest.HomeserverTestCase): % (self.room_id, event_id), access_token=invited_tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) events_before = channel.json_body["events_before"] @@ -1897,7 +1795,6 @@ class RoomAliasListTestCase(unittest.HomeserverTestCase): % (self.room_id,), access_token=access_token, ) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) res = channel.json_body self.assertIsInstance(res, dict) @@ -1916,7 +1813,6 @@ class RoomAliasListTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url, request_data, access_token=self.room_owner_tok ) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) @@ -1947,7 +1843,6 @@ class RoomCanonicalAliasTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "PUT", url, request_data, access_token=self.room_owner_tok ) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) def _get_canonical_alias(self, expected_code: int = 200) -> JsonDict: @@ -1957,7 +1852,6 @@ class RoomCanonicalAliasTestCase(unittest.HomeserverTestCase): "rooms/%s/state/m.room.canonical_alias" % (self.room_id,), access_token=self.room_owner_tok, ) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) res = channel.json_body self.assertIsInstance(res, dict) @@ -1971,7 +1865,6 @@ class RoomCanonicalAliasTestCase(unittest.HomeserverTestCase): json.dumps(content), access_token=self.room_owner_tok, ) - self.render(request) self.assertEqual(channel.code, expected_code, channel.result) res = channel.json_body self.assertIsInstance(res, dict) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index cd58ee7792..bbd30f594b 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -99,7 +99,6 @@ class RoomTypingTestCase(unittest.HomeserverTestCase): "/rooms/%s/typing/%s" % (self.room_id, self.user_id), b'{"typing": true, "timeout": 30000}', ) - self.render(request) self.assertEquals(200, channel.code) self.assertEquals(self.event_source.get_current_key(), 1) @@ -123,7 +122,6 @@ class RoomTypingTestCase(unittest.HomeserverTestCase): "/rooms/%s/typing/%s" % (self.room_id, self.user_id), b'{"typing": false}', ) - self.render(request) self.assertEquals(200, channel.code) def test_typing_timeout(self): @@ -132,7 +130,6 @@ class RoomTypingTestCase(unittest.HomeserverTestCase): "/rooms/%s/typing/%s" % (self.room_id, self.user_id), b'{"typing": true, "timeout": 30000}', ) - self.render(request) self.assertEquals(200, channel.code) self.assertEquals(self.event_source.get_current_key(), 1) @@ -146,7 +143,6 @@ class RoomTypingTestCase(unittest.HomeserverTestCase): "/rooms/%s/typing/%s" % (self.room_id, self.user_id), b'{"typing": true, "timeout": 30000}', ) - self.render(request) self.assertEquals(200, channel.code) self.assertEquals(self.event_source.get_current_key(), 3) diff --git a/tests/rest/client/v2_alpha/test_account.py b/tests/rest/client/v2_alpha/test_account.py index b871200909..2ac1ecb7d3 100644 --- a/tests/rest/client/v2_alpha/test_account.py +++ b/tests/rest/client/v2_alpha/test_account.py @@ -246,7 +246,6 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): b"account/password/email/requestToken", {"client_secret": client_secret, "email": email, "send_attempt": 1}, ) - self.render(request) self.assertEquals(200, channel.code, channel.result) return channel.json_body["sid"] @@ -325,7 +324,6 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): }, }, ) - self.render(request) self.assertEquals(expected_code, channel.code, channel.result) @@ -355,7 +353,6 @@ class DeactivateTestCase(unittest.HomeserverTestCase): # Check that this access token has been invalidated. request, channel = self.make_request("GET", "account/whoami") - self.render(request) self.assertEqual(request.code, 401) def test_pending_invites(self): @@ -413,7 +410,6 @@ class DeactivateTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", "account/deactivate", request_data, access_token=tok ) - self.render(request) self.assertEqual(request.code, 200) @@ -548,7 +544,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): }, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -556,7 +551,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url_3pid, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertFalse(channel.json_body["threepids"]) @@ -581,14 +575,12 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): {"medium": "email", "address": self.email}, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Get user request, channel = self.make_request( "GET", self.url_3pid, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertFalse(channel.json_body["threepids"]) @@ -615,7 +607,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): {"medium": "email", "address": self.email}, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) @@ -624,7 +615,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url_3pid, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("email", channel.json_body["threepids"][0]["medium"]) @@ -653,7 +643,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): }, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.THREEPID_AUTH_FAILED, channel.json_body["errcode"]) @@ -661,7 +650,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url_3pid, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertFalse(channel.json_body["threepids"]) @@ -688,7 +676,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): }, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.THREEPID_AUTH_FAILED, channel.json_body["errcode"]) @@ -696,7 +683,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url_3pid, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertFalse(channel.json_body["threepids"]) @@ -801,7 +787,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", b"account/3pid/email/requestToken", body, ) - self.render(request) self.assertEquals(expect_code, channel.code, channel.result) return channel.json_body.get("sid") @@ -814,7 +799,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): b"account/3pid/email/requestToken", {"client_secret": client_secret, "email": email, "send_attempt": 1}, ) - self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(expected_errcode, channel.json_body["errcode"]) self.assertEqual(expected_error, channel.json_body["error"]) @@ -824,7 +808,6 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): path = link.replace("https://example.com", "") request, channel = self.make_request("GET", path, shorthand=False) - self.render(request) self.assertEquals(200, channel.code, channel.result) def _get_link_from_email(self): @@ -873,14 +856,12 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Get user request, channel = self.make_request( "GET", self.url_3pid, access_token=self.user_id_tok, ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("email", channel.json_body["threepids"][0]["medium"]) diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py index 86184f0d2e..f684c37db5 100644 --- a/tests/rest/client/v2_alpha/test_auth.py +++ b/tests/rest/client/v2_alpha/test_auth.py @@ -72,7 +72,6 @@ class FallbackAuthTests(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", "register", body ) # type: SynapseRequest, FakeChannel - self.render(request) self.assertEqual(request.code, expected_response) return channel @@ -87,7 +86,6 @@ class FallbackAuthTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "auth/m.login.recaptcha/fallback/web?session=" + session ) # type: SynapseRequest, FakeChannel - self.render(request) self.assertEqual(request.code, 200) request, channel = self.make_request( @@ -96,7 +94,6 @@ class FallbackAuthTests(unittest.HomeserverTestCase): + post_session + "&g-recaptcha-response=a", ) - self.render(request) self.assertEqual(request.code, expected_post_response) # The recaptcha handler is called with the response given @@ -177,7 +174,6 @@ class UIAuthTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "devices", access_token=self.user_tok, ) # type: SynapseRequest, FakeChannel - self.render(request) # Get the ID of the device. self.assertEqual(request.code, 200) @@ -190,7 +186,6 @@ class UIAuthTests(unittest.HomeserverTestCase): request, channel = self.make_request( "DELETE", "devices/" + device, body, access_token=self.user_tok ) # type: SynapseRequest, FakeChannel - self.render(request) # Ensure the response is sane. self.assertEqual(request.code, expected_response) @@ -204,7 +199,6 @@ class UIAuthTests(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", "delete_devices", body, access_token=self.user_tok, ) # type: SynapseRequest, FakeChannel - self.render(request) # Ensure the response is sane. self.assertEqual(request.code, expected_response) diff --git a/tests/rest/client/v2_alpha/test_capabilities.py b/tests/rest/client/v2_alpha/test_capabilities.py index b9e01c9418..767e126875 100644 --- a/tests/rest/client/v2_alpha/test_capabilities.py +++ b/tests/rest/client/v2_alpha/test_capabilities.py @@ -37,7 +37,6 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): def test_check_auth_required(self): request, channel = self.make_request("GET", self.url) - self.render(request) self.assertEqual(channel.code, 401) @@ -46,7 +45,6 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): access_token = self.login("user", "pass") request, channel = self.make_request("GET", self.url, access_token=access_token) - self.render(request) capabilities = channel.json_body["capabilities"] self.assertEqual(channel.code, 200) @@ -65,7 +63,6 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): access_token = self.login(user, password) request, channel = self.make_request("GET", self.url, access_token=access_token) - self.render(request) capabilities = channel.json_body["capabilities"] self.assertEqual(channel.code, 200) @@ -74,7 +71,6 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertTrue(capabilities["m.change_password"]["enabled"]) self.get_success(self.store.user_set_password_hash(user, None)) request, channel = self.make_request("GET", self.url, access_token=access_token) - self.render(request) capabilities = channel.json_body["capabilities"] self.assertEqual(channel.code, 200) diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index de00350580..231d5aefea 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -41,7 +41,6 @@ class FilterTestCase(unittest.HomeserverTestCase): "/_matrix/client/r0/user/%s/filter" % (self.user_id), self.EXAMPLE_FILTER_JSON, ) - self.render(request) self.assertEqual(channel.result["code"], b"200") self.assertEqual(channel.json_body, {"filter_id": "0"}) @@ -55,7 +54,6 @@ class FilterTestCase(unittest.HomeserverTestCase): "/_matrix/client/r0/user/%s/filter" % ("@watermelon:test"), self.EXAMPLE_FILTER_JSON, ) - self.render(request) self.assertEqual(channel.result["code"], b"403") self.assertEquals(channel.json_body["errcode"], Codes.FORBIDDEN) @@ -68,7 +66,6 @@ class FilterTestCase(unittest.HomeserverTestCase): "/_matrix/client/r0/user/%s/filter" % (self.user_id), self.EXAMPLE_FILTER_JSON, ) - self.render(request) self.hs.is_mine = _is_mine self.assertEqual(channel.result["code"], b"403") @@ -85,7 +82,6 @@ class FilterTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/user/%s/filter/%s" % (self.user_id, filter_id) ) - self.render(request) self.assertEqual(channel.result["code"], b"200") self.assertEquals(channel.json_body, self.EXAMPLE_FILTER) @@ -94,7 +90,6 @@ class FilterTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/user/%s/filter/12382148321" % (self.user_id) ) - self.render(request) self.assertEqual(channel.result["code"], b"404") self.assertEquals(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -105,7 +100,6 @@ class FilterTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/user/%s/filter/foobar" % (self.user_id) ) - self.render(request) self.assertEqual(channel.result["code"], b"400") @@ -114,6 +108,5 @@ class FilterTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/user/%s/filter/" % (self.user_id) ) - self.render(request) self.assertEqual(channel.result["code"], b"400") diff --git a/tests/rest/client/v2_alpha/test_password_policy.py b/tests/rest/client/v2_alpha/test_password_policy.py index c57072f50c..ee86b94917 100644 --- a/tests/rest/client/v2_alpha/test_password_policy.py +++ b/tests/rest/client/v2_alpha/test_password_policy.py @@ -73,7 +73,6 @@ class PasswordPolicyTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/password_policy" ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) self.assertEqual( @@ -91,7 +90,6 @@ class PasswordPolicyTestCase(unittest.HomeserverTestCase): def test_password_too_short(self): request_data = json.dumps({"username": "kermit", "password": "shorty"}) request, channel = self.make_request("POST", self.register_url, request_data) - self.render(request) self.assertEqual(channel.code, 400, channel.result) self.assertEqual( @@ -101,7 +99,6 @@ class PasswordPolicyTestCase(unittest.HomeserverTestCase): def test_password_no_digit(self): request_data = json.dumps({"username": "kermit", "password": "longerpassword"}) request, channel = self.make_request("POST", self.register_url, request_data) - self.render(request) self.assertEqual(channel.code, 400, channel.result) self.assertEqual( @@ -111,7 +108,6 @@ class PasswordPolicyTestCase(unittest.HomeserverTestCase): def test_password_no_symbol(self): request_data = json.dumps({"username": "kermit", "password": "l0ngerpassword"}) request, channel = self.make_request("POST", self.register_url, request_data) - self.render(request) self.assertEqual(channel.code, 400, channel.result) self.assertEqual( @@ -121,7 +117,6 @@ class PasswordPolicyTestCase(unittest.HomeserverTestCase): def test_password_no_uppercase(self): request_data = json.dumps({"username": "kermit", "password": "l0ngerpassword!"}) request, channel = self.make_request("POST", self.register_url, request_data) - self.render(request) self.assertEqual(channel.code, 400, channel.result) self.assertEqual( @@ -131,7 +126,6 @@ class PasswordPolicyTestCase(unittest.HomeserverTestCase): def test_password_no_lowercase(self): request_data = json.dumps({"username": "kermit", "password": "L0NGERPASSWORD!"}) request, channel = self.make_request("POST", self.register_url, request_data) - self.render(request) self.assertEqual(channel.code, 400, channel.result) self.assertEqual( @@ -141,7 +135,6 @@ class PasswordPolicyTestCase(unittest.HomeserverTestCase): def test_password_compliant(self): request_data = json.dumps({"username": "kermit", "password": "L0ngerpassword!"}) request, channel = self.make_request("POST", self.register_url, request_data) - self.render(request) # Getting a 401 here means the password has passed validation and the server has # responded with a list of registration flows. @@ -173,7 +166,6 @@ class PasswordPolicyTestCase(unittest.HomeserverTestCase): request_data, access_token=tok, ) - self.render(request) self.assertEqual(channel.code, 400, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.PASSWORD_NO_DIGIT) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 98c3887bbf..88923fcea4 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -64,7 +64,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", self.url + b"?access_token=i_am_an_app_service", request_data ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) det_data = {"user_id": user_id, "home_server": self.hs.hostname} @@ -76,14 +75,12 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", self.url + b"?access_token=i_am_an_app_service", request_data ) - self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) def test_POST_bad_password(self): request_data = json.dumps({"username": "kermit", "password": 666}) request, channel = self.make_request(b"POST", self.url, request_data) - self.render(request) self.assertEquals(channel.result["code"], b"400", channel.result) self.assertEquals(channel.json_body["error"], "Invalid password") @@ -91,7 +88,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): def test_POST_bad_username(self): request_data = json.dumps({"username": 777, "password": "monkey"}) request, channel = self.make_request(b"POST", self.url, request_data) - self.render(request) self.assertEquals(channel.result["code"], b"400", channel.result) self.assertEquals(channel.json_body["error"], "Invalid username") @@ -107,7 +103,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): } request_data = json.dumps(params) request, channel = self.make_request(b"POST", self.url, request_data) - self.render(request) det_data = { "user_id": user_id, @@ -123,7 +118,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.auth_result = (None, {"username": "kermit", "password": "monkey"}, None) request, channel = self.make_request(b"POST", self.url, request_data) - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals(channel.json_body["error"], "Registration has been disabled") @@ -133,7 +127,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.hs.config.allow_guest_access = True request, channel = self.make_request(b"POST", self.url + b"?kind=guest", b"{}") - self.render(request) det_data = {"home_server": self.hs.hostname, "device_id": "guest_device"} self.assertEquals(channel.result["code"], b"200", channel.result) @@ -143,7 +136,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.hs.config.allow_guest_access = False request, channel = self.make_request(b"POST", self.url + b"?kind=guest", b"{}") - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals(channel.json_body["error"], "Guest access is disabled") @@ -153,7 +145,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): for i in range(0, 6): url = self.url + b"?kind=guest" request, channel = self.make_request(b"POST", url, b"{}") - self.render(request) if i == 5: self.assertEquals(channel.result["code"], b"429", channel.result) @@ -164,7 +155,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.reactor.advance(retry_after_ms / 1000.0 + 1.0) request, channel = self.make_request(b"POST", self.url + b"?kind=guest", b"{}") - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) @@ -179,7 +169,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): } request_data = json.dumps(params) request, channel = self.make_request(b"POST", self.url, request_data) - self.render(request) if i == 5: self.assertEquals(channel.result["code"], b"429", channel.result) @@ -190,13 +179,11 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.reactor.advance(retry_after_ms / 1000.0 + 1.0) request, channel = self.make_request(b"POST", self.url + b"?kind=guest", b"{}") - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) def test_advertised_flows(self): request, channel = self.make_request(b"POST", self.url, b"{}") - self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) flows = channel.json_body["flows"] @@ -220,7 +207,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): ) def test_advertised_flows_captcha_and_terms_and_3pids(self): request, channel = self.make_request(b"POST", self.url, b"{}") - self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) flows = channel.json_body["flows"] @@ -253,7 +239,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): ) def test_advertised_flows_no_msisdn_email_required(self): request, channel = self.make_request(b"POST", self.url, b"{}") - self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) flows = channel.json_body["flows"] @@ -298,7 +283,6 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): b"register/email/requestToken", {"client_secret": "foobar", "email": email, "send_attempt": 1}, ) - self.render(request) self.assertEquals(200, channel.code, channel.result) self.assertIsNotNone(channel.json_body.get("sid")) @@ -334,14 +318,12 @@ class AccountValidityTestCase(unittest.HomeserverTestCase): # The specific endpoint doesn't matter, all we need is an authenticated # endpoint. request, channel = self.make_request(b"GET", "/sync", access_token=tok) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) self.reactor.advance(datetime.timedelta(weeks=1).total_seconds()) request, channel = self.make_request(b"GET", "/sync", access_token=tok) - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals( @@ -366,13 +348,11 @@ class AccountValidityTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", url, request_data, access_token=admin_tok ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) # The specific endpoint doesn't matter, all we need is an authenticated # endpoint. request, channel = self.make_request(b"GET", "/sync", access_token=tok) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) def test_manual_expire(self): @@ -392,13 +372,11 @@ class AccountValidityTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", url, request_data, access_token=admin_tok ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) # The specific endpoint doesn't matter, all we need is an authenticated # endpoint. request, channel = self.make_request(b"GET", "/sync", access_token=tok) - self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals( channel.json_body["errcode"], Codes.EXPIRED_ACCOUNT, channel.result @@ -421,12 +399,10 @@ class AccountValidityTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( b"POST", url, request_data, access_token=admin_tok ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) # Try to log the user out request, channel = self.make_request(b"POST", "/logout", access_token=tok) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) # Log the user in again (allowed for expired accounts) @@ -434,7 +410,6 @@ class AccountValidityTestCase(unittest.HomeserverTestCase): # Try to log out all of the user's sessions request, channel = self.make_request(b"POST", "/logout/all", access_token=tok) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) @@ -509,7 +484,6 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase): renewal_token = self.get_success(self.store.get_renewal_token_for_user(user_id)) url = "/_matrix/client/unstable/account_validity/renew?token=%s" % renewal_token request, channel = self.make_request(b"GET", url) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) # Check that we're getting HTML back. @@ -530,7 +504,6 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase): # succeed. self.reactor.advance(datetime.timedelta(days=3).total_seconds()) request, channel = self.make_request(b"GET", "/sync", access_token=tok) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) def test_renewal_invalid_token(self): @@ -538,7 +511,6 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase): # expected, i.e. that it responds with 404 Not Found and the correct HTML. url = "/_matrix/client/unstable/account_validity/renew?token=123" request, channel = self.make_request(b"GET", url) - self.render(request) self.assertEquals(channel.result["code"], b"404", channel.result) # Check that we're getting HTML back. @@ -564,7 +536,6 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase): "/_matrix/client/unstable/account_validity/send_mail", access_token=tok, ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) self.assertEqual(len(self.email_attempts), 1) @@ -587,7 +558,6 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", "account/deactivate", request_data, access_token=tok ) - self.render(request) self.assertEqual(request.code, 200) self.reactor.advance(datetime.timedelta(days=8).total_seconds()) @@ -641,7 +611,6 @@ class AccountValidityRenewalByEmailTestCase(unittest.HomeserverTestCase): "/_matrix/client/unstable/account_validity/send_mail", access_token=tok, ) - self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) self.assertEqual(len(self.email_attempts), 1) diff --git a/tests/rest/client/v2_alpha/test_relations.py b/tests/rest/client/v2_alpha/test_relations.py index 99c9f4e928..6cd4eb6624 100644 --- a/tests/rest/client/v2_alpha/test_relations.py +++ b/tests/rest/client/v2_alpha/test_relations.py @@ -65,7 +65,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): "/rooms/%s/event/%s" % (self.room, event_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assert_dict( @@ -114,7 +113,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, self.parent_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) # We expect to get back a single pagination result, which is the full @@ -160,7 +158,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, self.parent_id, from_token), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) found_event_ids.extend(e["event_id"] for e in channel.json_body["chunk"]) @@ -219,7 +216,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, self.parent_id, from_token), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertEqual(len(channel.json_body["chunk"]), 1, channel.json_body) @@ -296,7 +292,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): ), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertEqual(len(channel.json_body["chunk"]), 1, channel.json_body) @@ -336,7 +331,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, self.parent_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertEquals( @@ -369,7 +363,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): access_token=self.user_token, content={}, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) request, channel = self.make_request( @@ -378,7 +371,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, self.parent_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertEquals( @@ -396,7 +388,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, self.parent_id, RelationTypes.REPLACE), access_token=self.user_token, ) - self.render(request) self.assertEquals(400, channel.code, channel.json_body) def test_aggregation_get_event(self): @@ -428,7 +419,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): "/rooms/%s/event/%s" % (self.room, self.parent_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertEquals( @@ -465,7 +455,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): "/rooms/%s/event/%s" % (self.room, self.parent_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertEquals(channel.json_body["content"], new_body) @@ -523,7 +512,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): "/rooms/%s/event/%s" % (self.room, self.parent_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertEquals(channel.json_body["content"], new_body) @@ -567,7 +555,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, original_event_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertIn("chunk", channel.json_body) @@ -581,7 +568,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): access_token=self.user_token, content="{}", ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) # Try to check for remaining m.replace relations @@ -591,7 +577,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, original_event_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) # Check that no relations are returned @@ -624,7 +609,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): access_token=self.user_token, content="{}", ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) # Check that aggregations returns zero @@ -634,7 +618,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): % (self.room, original_event_id), access_token=self.user_token, ) - self.render(request) self.assertEquals(200, channel.code, channel.json_body) self.assertIn("chunk", channel.json_body) @@ -680,7 +663,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): json.dumps(content).encode("utf-8"), access_token=access_token, ) - self.render(request) return channel def _create_user(self, localpart): diff --git a/tests/rest/client/v2_alpha/test_shared_rooms.py b/tests/rest/client/v2_alpha/test_shared_rooms.py index 5ae72fd008..562a9c1ba4 100644 --- a/tests/rest/client/v2_alpha/test_shared_rooms.py +++ b/tests/rest/client/v2_alpha/test_shared_rooms.py @@ -47,7 +47,6 @@ class UserSharedRoomsTest(unittest.HomeserverTestCase): % other_user, access_token=token, ) - self.render(request) return request, channel def test_shared_room_list_public(self): diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index f74d611943..31ac0fccb8 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -36,7 +36,6 @@ class FilterTestCase(unittest.HomeserverTestCase): def test_sync_argless(self): request, channel = self.make_request("GET", "/sync") - self.render(request) self.assertEqual(channel.code, 200) self.assertTrue( @@ -57,7 +56,6 @@ class FilterTestCase(unittest.HomeserverTestCase): self.hs.config.use_presence = False request, channel = self.make_request("GET", "/sync") - self.render(request) self.assertEqual(channel.code, 200) self.assertTrue( @@ -199,7 +197,6 @@ class SyncFilterTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/sync?filter=%s" % sync_filter, access_token=tok ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) return channel.json_body["rooms"]["join"][room_id]["timeline"]["events"] @@ -253,13 +250,11 @@ class SyncTypingTests(unittest.HomeserverTestCase): typing_url % (room, other_user_id, other_access_token), b'{"typing": true, "timeout": 30000}', ) - self.render(request) self.assertEquals(200, channel.code) request, channel = self.make_request( "GET", "/sync?access_token=%s" % (access_token,) ) - self.render(request) self.assertEquals(200, channel.code) next_batch = channel.json_body["next_batch"] @@ -269,7 +264,6 @@ class SyncTypingTests(unittest.HomeserverTestCase): typing_url % (room, other_user_id, other_access_token), b'{"typing": false}', ) - self.render(request) self.assertEquals(200, channel.code) # Start typing. @@ -278,14 +272,12 @@ class SyncTypingTests(unittest.HomeserverTestCase): typing_url % (room, other_user_id, other_access_token), b'{"typing": true, "timeout": 30000}', ) - self.render(request) self.assertEquals(200, channel.code) # Should return immediately request, channel = self.make_request( "GET", sync_url % (access_token, next_batch) ) - self.render(request) self.assertEquals(200, channel.code) next_batch = channel.json_body["next_batch"] @@ -300,7 +292,6 @@ class SyncTypingTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", sync_url % (access_token, next_batch) ) - self.render(request) self.assertEquals(200, channel.code) next_batch = channel.json_body["next_batch"] @@ -311,7 +302,6 @@ class SyncTypingTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", sync_url % (access_token, next_batch) ) - self.render(request) self.assertEquals(200, channel.code) next_batch = channel.json_body["next_batch"] @@ -399,7 +389,6 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): body, access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.json_body) # Check that the unread counter is back to 0. @@ -464,7 +453,6 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", self.url % self.next_batch, access_token=self.tok, ) - self.render(request) self.assertEqual(channel.code, 200, channel.json_body) diff --git a/tests/rest/test_health.py b/tests/rest/test_health.py index f4d06e2200..02a46e5fda 100644 --- a/tests/rest/test_health.py +++ b/tests/rest/test_health.py @@ -26,7 +26,6 @@ class HealthCheckTests(unittest.HomeserverTestCase): def test_health(self): request, channel = self.make_request("GET", "/health", shorthand=False) - self.render(request) self.assertEqual(request.code, 200) self.assertEqual(channel.result["body"], b"OK") diff --git a/tests/rest/test_well_known.py b/tests/rest/test_well_known.py index a3746e7130..6a930f4148 100644 --- a/tests/rest/test_well_known.py +++ b/tests/rest/test_well_known.py @@ -31,7 +31,6 @@ class WellKnownTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/.well-known/matrix/client", shorthand=False ) - self.render(request) self.assertEqual(request.code, 200) self.assertEqual( @@ -48,6 +47,5 @@ class WellKnownTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/.well-known/matrix/client", shorthand=False ) - self.render(request) self.assertEqual(request.code, 404) diff --git a/tests/server_notices/test_consent.py b/tests/server_notices/test_consent.py index 872039c8f1..e0a9cd93ac 100644 --- a/tests/server_notices/test_consent.py +++ b/tests/server_notices/test_consent.py @@ -73,7 +73,6 @@ class ConsentNoticesTests(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/_matrix/client/r0/sync", access_token=self.access_token ) - self.render(request) self.assertEqual(channel.code, 200) # Get the Room ID to join @@ -85,14 +84,12 @@ class ConsentNoticesTests(unittest.HomeserverTestCase): "/_matrix/client/r0/rooms/" + room_id + "/join", access_token=self.access_token, ) - self.render(request) self.assertEqual(channel.code, 200) # Sync again, to get the message in the room request, channel = self.make_request( "GET", "/_matrix/client/r0/sync", access_token=self.access_token ) - self.render(request) self.assertEqual(channel.code, 200) # Get the message diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index 6382b19dc3..9c8027a5b2 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -306,7 +306,6 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): tok = self.login("user", "password") request, channel = self.make_request("GET", "/sync?timeout=0", access_token=tok) - self.render(request) invites = channel.json_body["rooms"]["invite"] self.assertEqual(len(invites), 0, invites) @@ -320,7 +319,6 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): # Sync again to retrieve the events in the room, so we can check whether this # room has a notice in it. request, channel = self.make_request("GET", "/sync?timeout=0", access_token=tok) - self.render(request) # Scan the events in the room to search for a message from the server notices # user. @@ -358,7 +356,6 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "/sync?timeout=0", access_token=tok, ) - self.render(request) # Also retrieves the list of invites for this user. We don't care about that # one except if we're processing the last user, which should have received an diff --git a/tests/test_mau.py b/tests/test_mau.py index 654a6fa42d..c5ec6396a7 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -202,7 +202,6 @@ class TestMauLimit(unittest.HomeserverTestCase): ) request, channel = self.make_request("POST", "/register", request_data) - self.render(request) if channel.code != 200: raise HttpResponseException( @@ -215,7 +214,6 @@ class TestMauLimit(unittest.HomeserverTestCase): def do_sync_for_user(self, token): request, channel = self.make_request("GET", "/sync", access_token=token) - self.render(request) if channel.code != 200: raise HttpResponseException( diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py index b89798336c..71580b454d 100644 --- a/tests/test_terms_auth.py +++ b/tests/test_terms_auth.py @@ -54,7 +54,6 @@ class TermsTestCase(unittest.HomeserverTestCase): # Do a UI auth request request_data = json.dumps({"username": "kermit", "password": "monkey"}) request, channel = self.make_request(b"POST", self.url, request_data) - self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) @@ -98,7 +97,6 @@ class TermsTestCase(unittest.HomeserverTestCase): self.registration_handler.check_username = Mock(return_value=True) request, channel = self.make_request(b"POST", self.url, request_data) - self.render(request) # We don't bother checking that the response is correct - we'll leave that to # other tests. We just want to make sure we're on the right path. @@ -116,7 +114,6 @@ class TermsTestCase(unittest.HomeserverTestCase): } ) request, channel = self.make_request(b"POST", self.url, request_data) - self.render(request) # We're interested in getting a response that looks like a successful # registration, not so much that the details are exactly what we want. diff --git a/tests/unittest.py b/tests/unittest.py index 8a49bb5262..c7c889c405 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -440,16 +440,6 @@ class HomeserverTestCase(TestCase): await_result, ) - def render(self, request): - """ - Render a request against the resources registered by the test class's - servlets. - - Args: - request (synapse.http.site.SynapseRequest): The request to render. - """ - pass - def setup_test_homeserver(self, *args, **kwargs): """ Set up the test homeserver, meant to be called by the overridable @@ -565,7 +555,6 @@ class HomeserverTestCase(TestCase): # Create the user request, channel = self.make_request("GET", "/_matrix/client/r0/admin/register") - self.render(request) self.assertEqual(channel.code, 200, msg=channel.result) nonce = channel.json_body["nonce"] @@ -593,7 +582,6 @@ class HomeserverTestCase(TestCase): request, channel = self.make_request( "POST", "/_matrix/client/r0/admin/register", body.encode("utf8") ) - self.render(request) self.assertEqual(channel.code, 200, channel.json_body) user_id = channel.json_body["user_id"] @@ -612,7 +600,6 @@ class HomeserverTestCase(TestCase): request, channel = self.make_request( "POST", "/_matrix/client/r0/login", json.dumps(body).encode("utf8") ) - self.render(request) self.assertEqual(channel.code, 200, channel.result) access_token = channel.json_body["access_token"] @@ -681,7 +668,6 @@ class HomeserverTestCase(TestCase): request, channel = self.make_request( "POST", "/_matrix/client/r0/login", json.dumps(body).encode("utf8") ) - self.render(request) self.assertEqual(channel.code, 403, channel.result) def inject_room_member(self, room: str, user: str, membership: Membership) -> None: From 0ce31ef61490a6e256e63683c38b1b86de58120c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 15 Nov 2020 23:00:36 +0000 Subject: [PATCH 23/57] changelog --- changelog.d/8761.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8761.misc diff --git a/changelog.d/8761.misc b/changelog.d/8761.misc new file mode 100644 index 0000000000..e6da7d038d --- /dev/null +++ b/changelog.d/8761.misc @@ -0,0 +1 @@ + Refactor test utilities for injecting HTTP requests. From f737368a26bb9eea401fcc3a5bdd7e0b59e91f09 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2020 10:51:25 +0000 Subject: [PATCH 24/57] Add admin API for logging in as a user (#8617) --- changelog.d/8617.feature | 1 + docs/admin_api/user_admin_api.rst | 35 +++ synapse/api/auth_blocking.py | 33 ++- synapse/handlers/_base.py | 4 +- synapse/handlers/auth.py | 24 +- synapse/handlers/deactivate_account.py | 5 +- synapse/handlers/message.py | 21 +- synapse/handlers/profile.py | 8 +- synapse/handlers/register.py | 24 +- synapse/handlers/room.py | 10 +- synapse/handlers/room_member.py | 5 +- synapse/handlers/sync.py | 4 +- synapse/module_api/__init__.py | 5 +- synapse/rest/admin/__init__.py | 2 + synapse/rest/admin/rooms.py | 4 +- synapse/rest/admin/users.py | 54 +++- synapse/rest/client/v2_alpha/sync.py | 1 + .../server_notices/server_notices_manager.py | 13 +- .../storage/databases/main/registration.py | 2 + tests/api/test_auth.py | 6 +- tests/handlers/test_sync.py | 14 +- tests/module_api/test_api.py | 11 +- tests/rest/admin/test_user.py | 245 +++++++++++++++++- tests/storage/test_cleanup_extrems.py | 30 --- tests/test_state.py | 1 + 25 files changed, 475 insertions(+), 87 deletions(-) create mode 100644 changelog.d/8617.feature diff --git a/changelog.d/8617.feature b/changelog.d/8617.feature new file mode 100644 index 0000000000..4f1e788506 --- /dev/null +++ b/changelog.d/8617.feature @@ -0,0 +1 @@ +Add admin API for logging in as a user. diff --git a/docs/admin_api/user_admin_api.rst b/docs/admin_api/user_admin_api.rst index d4051d0257..95b3da27c4 100644 --- a/docs/admin_api/user_admin_api.rst +++ b/docs/admin_api/user_admin_api.rst @@ -424,6 +424,41 @@ The following fields are returned in the JSON response body: - ``next_token``: integer - Indication for pagination. See above. - ``total`` - integer - Total number of media. +Login as a user +=============== + +Get an access token that can be used to authenticate as that user. Useful for +when admins wish to do actions on behalf of a user. + +The API is:: + + POST /_synapse/admin/v1/users//login + {} + +An optional ``valid_until_ms`` field can be specified in the request body as an +integer timestamp that specifies when the token should expire. By default tokens +do not expire. + +A response body like the following is returned: + +.. code:: json + + { + "access_token": "" + } + + +This API does *not* generate a new device for the user, and so will not appear +their ``/devices`` list, and in general the target user should not be able to +tell they have been logged in as. + +To expire the token call the standard ``/logout`` API with the token. + +Note: The token will expire if the *admin* user calls ``/logout/all`` from any +of their devices, but the token will *not* expire if the target user does the +same. + + User devices ============ diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index d8fafd7cb8..9c227218e0 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -14,10 +14,12 @@ # limitations under the License. import logging +from typing import Optional from synapse.api.constants import LimitBlockingTypes, UserTypes from synapse.api.errors import Codes, ResourceLimitError from synapse.config.server import is_threepid_reserved +from synapse.types import Requester logger = logging.getLogger(__name__) @@ -33,24 +35,47 @@ class AuthBlocking: self._max_mau_value = hs.config.max_mau_value self._limit_usage_by_mau = hs.config.limit_usage_by_mau self._mau_limits_reserved_threepids = hs.config.mau_limits_reserved_threepids + self._server_name = hs.hostname - async def check_auth_blocking(self, user_id=None, threepid=None, user_type=None): + async def check_auth_blocking( + self, + user_id: Optional[str] = None, + threepid: Optional[dict] = None, + user_type: Optional[str] = None, + requester: Optional[Requester] = None, + ): """Checks if the user should be rejected for some external reason, such as monthly active user limiting or global disable flag Args: - user_id(str|None): If present, checks for presence against existing + user_id: If present, checks for presence against existing MAU cohort - threepid(dict|None): If present, checks for presence against configured + threepid: If present, checks for presence against configured reserved threepid. Used in cases where the user is trying register with a MAU blocked server, normally they would be rejected but their threepid is on the reserved list. user_id and threepid should never be set at the same time. - user_type(str|None): If present, is used to decide whether to check against + user_type: If present, is used to decide whether to check against certain blocking reasons like MAU. + + requester: If present, and the authenticated entity is a user, checks for + presence against existing MAU cohort. Passing in both a `user_id` and + `requester` is an error. """ + if requester and user_id: + raise Exception( + "Passed in both 'user_id' and 'requester' to 'check_auth_blocking'" + ) + + if requester: + if requester.authenticated_entity.startswith("@"): + user_id = requester.authenticated_entity + elif requester.authenticated_entity == self._server_name: + # We never block the server from doing actions on behalf of + # users. + return # Never fail an auth check for the server notices users or support user # This can be a problem where event creation is prohibited due to blocking diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index bd8e71ae56..bb81c0e81d 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -169,7 +169,9 @@ class BaseHandler: # and having homeservers have their own users leave keeps more # of that decision-making and control local to the guest-having # homeserver. - requester = synapse.types.create_requester(target_user, is_guest=True) + requester = synapse.types.create_requester( + target_user, is_guest=True, authenticated_entity=self.server_name + ) handler = self.hs.get_room_member_handler() await handler.update_membership( requester, diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 213baea2e3..5163afd86c 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -698,8 +698,12 @@ class AuthHandler(BaseHandler): } async def get_access_token_for_user_id( - self, user_id: str, device_id: Optional[str], valid_until_ms: Optional[int] - ): + self, + user_id: str, + device_id: Optional[str], + valid_until_ms: Optional[int], + puppets_user_id: Optional[str] = None, + ) -> str: """ Creates a new access token for the user with the given user ID. @@ -725,13 +729,25 @@ class AuthHandler(BaseHandler): fmt_expiry = time.strftime( " until %Y-%m-%d %H:%M:%S", time.localtime(valid_until_ms / 1000.0) ) - logger.info("Logging in user %s on device %s%s", user_id, device_id, fmt_expiry) + + if puppets_user_id: + logger.info( + "Logging in user %s as %s%s", user_id, puppets_user_id, fmt_expiry + ) + else: + logger.info( + "Logging in user %s on device %s%s", user_id, device_id, fmt_expiry + ) await self.auth.check_auth_blocking(user_id) access_token = self.macaroon_gen.generate_access_token(user_id) await self.store.add_access_token_to_user( - user_id, access_token, device_id, valid_until_ms + user_id=user_id, + token=access_token, + device_id=device_id, + valid_until_ms=valid_until_ms, + puppets_user_id=puppets_user_id, ) # the device *should* have been registered before we got here; however, diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 4efe6c530a..e808142365 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -39,6 +39,7 @@ class DeactivateAccountHandler(BaseHandler): self._room_member_handler = hs.get_room_member_handler() self._identity_handler = hs.get_identity_handler() self.user_directory_handler = hs.get_user_directory_handler() + self._server_name = hs.hostname # Flag that indicates whether the process to part users from rooms is running self._user_parter_running = False @@ -152,7 +153,7 @@ class DeactivateAccountHandler(BaseHandler): for room in pending_invites: try: await self._room_member_handler.update_membership( - create_requester(user), + create_requester(user, authenticated_entity=self._server_name), user, room.room_id, "leave", @@ -208,7 +209,7 @@ class DeactivateAccountHandler(BaseHandler): logger.info("User parter parting %r from %r", user_id, room_id) try: await self._room_member_handler.update_membership( - create_requester(user), + create_requester(user, authenticated_entity=self._server_name), user, room_id, "leave", diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6791fb912..96843338ae 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -472,7 +472,7 @@ class EventCreationHandler: Returns: Tuple of created event, Context """ - await self.auth.check_auth_blocking(requester.user.to_string()) + await self.auth.check_auth_blocking(requester=requester) if event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "": room_version = event_dict["content"]["room_version"] @@ -619,7 +619,13 @@ class EventCreationHandler: if requester.app_service is not None: return - user_id = requester.user.to_string() + user_id = requester.authenticated_entity + if not user_id.startswith("@"): + # The authenticated entity might not be a user, e.g. if it's the + # server puppetting the user. + return + + user = UserID.from_string(user_id) # exempt the system notices user if ( @@ -639,9 +645,7 @@ class EventCreationHandler: if u["consent_version"] == self.config.user_consent_version: return - consent_uri = self._consent_uri_builder.build_user_consent_uri( - requester.user.localpart - ) + consent_uri = self._consent_uri_builder.build_user_consent_uri(user.localpart) msg = self._block_events_without_consent_error % {"consent_uri": consent_uri} raise ConsentNotGivenError(msg=msg, consent_uri=consent_uri) @@ -1252,7 +1256,7 @@ class EventCreationHandler: for user_id in members: if not self.hs.is_mine_id(user_id): continue - requester = create_requester(user_id) + requester = create_requester(user_id, authenticated_entity=self.server_name) try: event, context = await self.create_event( requester, @@ -1273,11 +1277,6 @@ class EventCreationHandler: requester, event, context, ratelimit=False, ignore_shadow_ban=True, ) return True - except ConsentNotGivenError: - logger.info( - "Failed to send dummy event into room %s for user %s due to " - "lack of consent. Will try another user" % (room_id, user_id) - ) except AuthError: logger.info( "Failed to send dummy event into room %s for user %s due to " diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 74a1ddd780..dee0ef45e7 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -206,7 +206,9 @@ class ProfileHandler(BaseHandler): # the join event to update the displayname in the rooms. # This must be done by the target user himself. if by_admin: - requester = create_requester(target_user) + requester = create_requester( + target_user, authenticated_entity=requester.authenticated_entity, + ) await self.store.set_profile_displayname( target_user.localpart, displayname_to_set @@ -286,7 +288,9 @@ class ProfileHandler(BaseHandler): # Same like set_displayname if by_admin: - requester = create_requester(target_user) + requester = create_requester( + target_user, authenticated_entity=requester.authenticated_entity + ) await self.store.set_profile_avatar_url(target_user.localpart, new_avatar_url) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index ed1ff62599..252f700786 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -52,6 +52,7 @@ class RegistrationHandler(BaseHandler): self.ratelimiter = hs.get_registration_ratelimiter() self.macaroon_gen = hs.get_macaroon_generator() self._server_notices_mxid = hs.config.server_notices_mxid + self._server_name = hs.hostname self.spam_checker = hs.get_spam_checker() @@ -317,7 +318,8 @@ class RegistrationHandler(BaseHandler): requires_join = False if self.hs.config.registration.auto_join_user_id: fake_requester = create_requester( - self.hs.config.registration.auto_join_user_id + self.hs.config.registration.auto_join_user_id, + authenticated_entity=self._server_name, ) # If the room requires an invite, add the user to the list of invites. @@ -329,7 +331,9 @@ class RegistrationHandler(BaseHandler): # being necessary this will occur after the invite was sent. requires_join = True else: - fake_requester = create_requester(user_id) + fake_requester = create_requester( + user_id, authenticated_entity=self._server_name + ) # Choose whether to federate the new room. if not self.hs.config.registration.autocreate_auto_join_rooms_federated: @@ -362,7 +366,9 @@ class RegistrationHandler(BaseHandler): # created it, then ensure the first user joins it. if requires_join: await room_member_handler.update_membership( - requester=create_requester(user_id), + requester=create_requester( + user_id, authenticated_entity=self._server_name + ), target=UserID.from_string(user_id), room_id=info["room_id"], # Since it was just created, there are no remote hosts. @@ -370,11 +376,6 @@ class RegistrationHandler(BaseHandler): action="join", ratelimit=False, ) - - except ConsentNotGivenError as e: - # Technically not necessary to pull out this error though - # moving away from bare excepts is a good thing to do. - logger.error("Failed to join new user to %r: %r", r, e) except Exception as e: logger.error("Failed to join new user to %r: %r", r, e) @@ -426,7 +427,8 @@ class RegistrationHandler(BaseHandler): if requires_invite: await room_member_handler.update_membership( requester=create_requester( - self.hs.config.registration.auto_join_user_id + self.hs.config.registration.auto_join_user_id, + authenticated_entity=self._server_name, ), target=UserID.from_string(user_id), room_id=room_id, @@ -437,7 +439,9 @@ class RegistrationHandler(BaseHandler): # Send the join. await room_member_handler.update_membership( - requester=create_requester(user_id), + requester=create_requester( + user_id, authenticated_entity=self._server_name + ), target=UserID.from_string(user_id), room_id=room_id, remote_room_hosts=remote_room_hosts, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index e73031475f..930047e730 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -587,7 +587,7 @@ class RoomCreationHandler(BaseHandler): """ user_id = requester.user.to_string() - await self.auth.check_auth_blocking(user_id) + await self.auth.check_auth_blocking(requester=requester) if ( self._server_notices_mxid is not None @@ -1257,7 +1257,9 @@ class RoomShutdownHandler: 400, "User must be our own: %s" % (new_room_user_id,) ) - room_creator_requester = create_requester(new_room_user_id) + room_creator_requester = create_requester( + new_room_user_id, authenticated_entity=requester_user_id + ) info, stream_id = await self._room_creation_handler.create_room( room_creator_requester, @@ -1297,7 +1299,9 @@ class RoomShutdownHandler: try: # Kick users from room - target_requester = create_requester(user_id) + target_requester = create_requester( + user_id, authenticated_entity=requester_user_id + ) _, stream_id = await self.room_member_handler.update_membership( requester=target_requester, target=target_requester.user, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index fd85e08973..70f8966267 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -965,6 +965,7 @@ class RoomMemberMasterHandler(RoomMemberHandler): self.distributor = hs.get_distributor() self.distributor.declare("user_left_room") + self._server_name = hs.hostname async def _is_remote_room_too_complex( self, room_id: str, remote_room_hosts: List[str] @@ -1059,7 +1060,9 @@ class RoomMemberMasterHandler(RoomMemberHandler): return event_id, stream_id # The room is too large. Leave. - requester = types.create_requester(user, None, False, False, None) + requester = types.create_requester( + user, authenticated_entity=self._server_name + ) await self.update_membership( requester=requester, target=user, room_id=room_id, action="leave" ) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 32e53c2d25..9827c7eb8d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -31,6 +31,7 @@ from synapse.types import ( Collection, JsonDict, MutableStateMap, + Requester, RoomStreamToken, StateMap, StreamToken, @@ -260,6 +261,7 @@ class SyncHandler: async def wait_for_sync_for_user( self, + requester: Requester, sync_config: SyncConfig, since_token: Optional[StreamToken] = None, timeout: int = 0, @@ -273,7 +275,7 @@ class SyncHandler: # not been exceeded (if not part of the group by this point, almost certain # auth_blocking will occur) user_id = sync_config.user.to_string() - await self.auth.check_auth_blocking(user_id) + await self.auth.check_auth_blocking(requester=requester) res = await self.response_cache.wrap( sync_config.request_key, diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 0142542852..72ab5750cc 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -49,6 +49,7 @@ class ModuleApi: self._store = hs.get_datastore() self._auth = hs.get_auth() self._auth_handler = auth_handler + self._server_name = hs.hostname # We expose these as properties below in order to attach a helpful docstring. self._http_client = hs.get_simple_http_client() # type: SimpleHttpClient @@ -336,7 +337,9 @@ class ModuleApi: SynapseError if the event was not allowed. """ # Create a requester object - requester = create_requester(event_dict["sender"]) + requester = create_requester( + event_dict["sender"], authenticated_entity=self._server_name + ) # Create and send the event ( diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 2a4f7a1740..7a3a5c46ca 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -61,6 +61,7 @@ from synapse.rest.admin.users import ( UserRestServletV2, UsersRestServlet, UsersRestServletV2, + UserTokenRestServlet, WhoisRestServlet, ) from synapse.types import RoomStreamToken @@ -223,6 +224,7 @@ def register_servlets(hs, http_server): UserAdminServlet(hs).register(http_server) UserMediaRestServlet(hs).register(http_server) UserMembershipRestServlet(hs).register(http_server) + UserTokenRestServlet(hs).register(http_server) UserRestServletV2(hs).register(http_server) UsersRestServletV2(hs).register(http_server) DeviceRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index f5304ff43d..ee345e12ce 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -309,7 +309,9 @@ class JoinRoomAliasServlet(RestServlet): 400, "%s was not legal room ID or room alias" % (room_identifier,) ) - fake_requester = create_requester(target_user) + fake_requester = create_requester( + target_user, authenticated_entity=requester.authenticated_entity + ) # send invite if room has "JoinRules.INVITE" room_state = await self.state_handler.get_current_state(room_id) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 3638e219f2..fa8d8e6d91 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -16,7 +16,7 @@ import hashlib import hmac import logging from http import HTTPStatus -from typing import Tuple +from typing import TYPE_CHECKING, Tuple from synapse.api.constants import UserTypes from synapse.api.errors import Codes, NotFoundError, SynapseError @@ -37,6 +37,9 @@ from synapse.rest.admin._base import ( ) from synapse.types import JsonDict, UserID +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) _GET_PUSHERS_ALLOWED_KEYS = { @@ -828,3 +831,52 @@ class UserMediaRestServlet(RestServlet): ret["next_token"] = start + len(media) return 200, ret + + +class UserTokenRestServlet(RestServlet): + """An admin API for logging in as a user. + + Example: + + POST /_synapse/admin/v1/users/@test:example.com/login + {} + + 200 OK + { + "access_token": "" + } + """ + + PATTERNS = admin_patterns("/users/(?P[^/]*)/login$") + + def __init__(self, hs: "HomeServer"): + self.hs = hs + self.store = hs.get_datastore() + self.auth = hs.get_auth() + self.auth_handler = hs.get_auth_handler() + + async def on_POST(self, request, user_id): + requester = await self.auth.get_user_by_req(request) + await assert_user_is_admin(self.auth, requester.user) + auth_user = requester.user + + if not self.hs.is_mine_id(user_id): + raise SynapseError(400, "Only local users can be logged in as") + + body = parse_json_object_from_request(request, allow_empty_body=True) + + valid_until_ms = body.get("valid_until_ms") + if valid_until_ms and not isinstance(valid_until_ms, int): + raise SynapseError(400, "'valid_until_ms' parameter must be an int") + + if auth_user.to_string() == user_id: + raise SynapseError(400, "Cannot use admin API to login as self") + + token = await self.auth_handler.get_access_token_for_user_id( + user_id=auth_user.to_string(), + device_id=None, + valid_until_ms=valid_until_ms, + puppets_user_id=user_id, + ) + + return 200, {"access_token": token} diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 2b84eb89c0..8e52e4cca4 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -171,6 +171,7 @@ class SyncRestServlet(RestServlet): ) with context: sync_result = await self.sync_handler.wait_for_sync_for_user( + requester, sync_config, since_token=since_token, timeout=timeout, diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index d464c75c03..100dbd5e2c 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -39,6 +39,7 @@ class ServerNoticesManager: self._room_member_handler = hs.get_room_member_handler() self._event_creation_handler = hs.get_event_creation_handler() self._is_mine_id = hs.is_mine_id + self._server_name = hs.hostname self._notifier = hs.get_notifier() self.server_notices_mxid = self._config.server_notices_mxid @@ -72,7 +73,9 @@ class ServerNoticesManager: await self.maybe_invite_user_to_room(user_id, room_id) system_mxid = self._config.server_notices_mxid - requester = create_requester(system_mxid) + requester = create_requester( + system_mxid, authenticated_entity=self._server_name + ) logger.info("Sending server notice to %s", user_id) @@ -145,7 +148,9 @@ class ServerNoticesManager: "avatar_url": self._config.server_notices_mxid_avatar_url, } - requester = create_requester(self.server_notices_mxid) + requester = create_requester( + self.server_notices_mxid, authenticated_entity=self._server_name + ) info, _ = await self._room_creation_handler.create_room( requester, config={ @@ -174,7 +179,9 @@ class ServerNoticesManager: user_id: The ID of the user to invite. room_id: The ID of the room to invite the user to. """ - requester = create_requester(self.server_notices_mxid) + requester = create_requester( + self.server_notices_mxid, authenticated_entity=self._server_name + ) # Check whether the user has already joined or been invited to this room. If # that's the case, there is no need to re-invite them. diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index e5d07ce72a..fedb8a6c26 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -1110,6 +1110,7 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): token: str, device_id: Optional[str], valid_until_ms: Optional[int], + puppets_user_id: Optional[str] = None, ) -> int: """Adds an access token for the given user. @@ -1133,6 +1134,7 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): "token": token, "device_id": device_id, "valid_until_ms": valid_until_ms, + "puppets_user_id": puppets_user_id, }, desc="add_access_token_to_user", ) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 0fd55f428a..ee5217b074 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -282,7 +282,11 @@ class AuthTestCase(unittest.TestCase): ) ) self.store.add_access_token_to_user.assert_called_with( - USER_ID, token, "DEVICE", None + user_id=USER_ID, + token=token, + device_id="DEVICE", + valid_until_ms=None, + puppets_user_id=None, ) def get_user(tok): diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index e178d7765b..e62586142e 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -16,7 +16,7 @@ from synapse.api.errors import Codes, ResourceLimitError from synapse.api.filtering import DEFAULT_FILTER_COLLECTION from synapse.handlers.sync import SyncConfig -from synapse.types import UserID +from synapse.types import UserID, create_requester import tests.unittest import tests.utils @@ -38,6 +38,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): user_id1 = "@user1:test" user_id2 = "@user2:test" sync_config = self._generate_sync_config(user_id1) + requester = create_requester(user_id1) self.reactor.advance(100) # So we get not 0 time self.auth_blocking._limit_usage_by_mau = True @@ -45,21 +46,26 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): # Check that the happy case does not throw errors self.get_success(self.store.upsert_monthly_active_user(user_id1)) - self.get_success(self.sync_handler.wait_for_sync_for_user(sync_config)) + self.get_success( + self.sync_handler.wait_for_sync_for_user(requester, sync_config) + ) # Test that global lock works self.auth_blocking._hs_disabled = True e = self.get_failure( - self.sync_handler.wait_for_sync_for_user(sync_config), ResourceLimitError + self.sync_handler.wait_for_sync_for_user(requester, sync_config), + ResourceLimitError, ) self.assertEquals(e.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED) self.auth_blocking._hs_disabled = False sync_config = self._generate_sync_config(user_id2) + requester = create_requester(user_id2) e = self.get_failure( - self.sync_handler.wait_for_sync_for_user(sync_config), ResourceLimitError + self.sync_handler.wait_for_sync_for_user(requester, sync_config), + ResourceLimitError, ) self.assertEquals(e.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED) diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 9b573ac24d..27206ca3db 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -94,12 +94,13 @@ class ModuleApiTestCase(HomeserverTestCase): self.assertFalse(hasattr(event, "state_key")) self.assertDictEqual(event.content, content) + expected_requester = create_requester( + user_id, authenticated_entity=self.hs.hostname + ) + # Check that the event was sent self.event_creation_handler.create_and_send_nonmember_event.assert_called_with( - create_requester(user_id), - event_dict, - ratelimit=False, - ignore_shadow_ban=True, + expected_requester, event_dict, ratelimit=False, ignore_shadow_ban=True, ) # Create and send a state event @@ -128,7 +129,7 @@ class ModuleApiTestCase(HomeserverTestCase): # Check that the event was sent self.event_creation_handler.create_and_send_nonmember_event.assert_called_with( - create_requester(user_id), + expected_requester, { "type": "m.room.power_levels", "content": content, diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index d74efede06..94cd45066c 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -24,8 +24,8 @@ from mock import Mock import synapse.rest.admin from synapse.api.constants import UserTypes from synapse.api.errors import Codes, HttpResponseException, ResourceLimitError -from synapse.rest.client.v1 import login, profile, room -from synapse.rest.client.v2_alpha import sync +from synapse.rest.client.v1 import login, logout, profile, room +from synapse.rest.client.v2_alpha import devices, sync from tests import unittest from tests.test_utils import make_awaitable @@ -1638,3 +1638,244 @@ class UserMediaRestTestCase(unittest.HomeserverTestCase): self.assertIn("last_access_ts", m) self.assertIn("quarantined_by", m) self.assertIn("safe_from_quarantine", m) + + +class UserTokenRestTestCase(unittest.HomeserverTestCase): + """Test for /_synapse/admin/v1/users//login + """ + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + sync.register_servlets, + room.register_servlets, + devices.register_servlets, + logout.register_servlets, + ] + + def prepare(self, reactor, clock, hs): + self.store = hs.get_datastore() + + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.other_user = self.register_user("user", "pass") + self.other_user_tok = self.login("user", "pass") + self.url = "/_synapse/admin/v1/users/%s/login" % urllib.parse.quote( + self.other_user + ) + + def _get_token(self) -> str: + request, channel = self.make_request( + "POST", self.url, b"{}", access_token=self.admin_user_tok + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + return channel.json_body["access_token"] + + def test_no_auth(self): + """Try to login as a user without authentication. + """ + request, channel = self.make_request("POST", self.url, b"{}") + self.render(request) + + self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_not_admin(self): + """Try to login as a user as a non-admin user. + """ + request, channel = self.make_request( + "POST", self.url, b"{}", access_token=self.other_user_tok + ) + self.render(request) + + self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) + + def test_send_event(self): + """Test that sending event as a user works. + """ + # Create a room. + room_id = self.helper.create_room_as(self.other_user, tok=self.other_user_tok) + + # Login in as the user + puppet_token = self._get_token() + + # Test that sending works, and generates the event as the right user. + resp = self.helper.send_event(room_id, "com.example.test", tok=puppet_token) + event_id = resp["event_id"] + event = self.get_success(self.store.get_event(event_id)) + self.assertEqual(event.sender, self.other_user) + + def test_devices(self): + """Tests that logging in as a user doesn't create a new device for them. + """ + # Login in as the user + self._get_token() + + # Check that we don't see a new device in our devices list + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=self.other_user_tok + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + # We should only see the one device (from the login in `prepare`) + self.assertEqual(len(channel.json_body["devices"]), 1) + + def test_logout(self): + """Test that calling `/logout` with the token works. + """ + # Login in as the user + puppet_token = self._get_token() + + # Test that we can successfully make a request + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=puppet_token + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + # Logout with the puppet token + request, channel = self.make_request( + "POST", "logout", b"{}", access_token=puppet_token + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + # The puppet token should no longer work + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=puppet_token + ) + self.render(request) + self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) + + # .. but the real user's tokens should still work + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=self.other_user_tok + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + def test_user_logout_all(self): + """Tests that the target user calling `/logout/all` does *not* expire + the token. + """ + # Login in as the user + puppet_token = self._get_token() + + # Test that we can successfully make a request + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=puppet_token + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + # Logout all with the real user token + request, channel = self.make_request( + "POST", "logout/all", b"{}", access_token=self.other_user_tok + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + # The puppet token should still work + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=puppet_token + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + # .. but the real user's tokens shouldn't + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=self.other_user_tok + ) + self.render(request) + self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) + + def test_admin_logout_all(self): + """Tests that the admin user calling `/logout/all` does expire the + token. + """ + # Login in as the user + puppet_token = self._get_token() + + # Test that we can successfully make a request + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=puppet_token + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + # Logout all with the admin user token + request, channel = self.make_request( + "POST", "logout/all", b"{}", access_token=self.admin_user_tok + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + # The puppet token should no longer work + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=puppet_token + ) + self.render(request) + self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) + + # .. but the real user's tokens should still work + request, channel = self.make_request( + "GET", "devices", b"{}", access_token=self.other_user_tok + ) + self.render(request) + self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) + + @unittest.override_config( + { + "public_baseurl": "https://example.org/", + "user_consent": { + "version": "1.0", + "policy_name": "My Cool Privacy Policy", + "template_dir": "/", + "require_at_registration": True, + "block_events_error": "You should accept the policy", + }, + "form_secret": "123secret", + } + ) + def test_consent(self): + """Test that sending a message is not subject to the privacy policies. + """ + # Have the admin user accept the terms. + self.get_success(self.store.user_set_consent_version(self.admin_user, "1.0")) + + # First, cheekily accept the terms and create a room + self.get_success(self.store.user_set_consent_version(self.other_user, "1.0")) + room_id = self.helper.create_room_as(self.other_user, tok=self.other_user_tok) + self.helper.send_event(room_id, "com.example.test", tok=self.other_user_tok) + + # Now unaccept it and check that we can't send an event + self.get_success(self.store.user_set_consent_version(self.other_user, "0.0")) + self.helper.send_event( + room_id, "com.example.test", tok=self.other_user_tok, expect_code=403 + ) + + # Login in as the user + puppet_token = self._get_token() + + # Sending an event on their behalf should work fine + self.helper.send_event(room_id, "com.example.test", tok=puppet_token) + + @override_config( + {"limit_usage_by_mau": True, "max_mau_value": 1, "mau_trial_days": 0} + ) + def test_mau_limit(self): + # Create a room as the admin user. This will bump the monthly active users to 1. + room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) + + # Trying to join as the other user should fail due to reaching MAU limit. + self.helper.join( + room_id, user=self.other_user, tok=self.other_user_tok, expect_code=403 + ) + + # Logging in as the other user and joining a room should work, even + # though the MAU limit would stop the user doing so. + puppet_token = self._get_token() + self.helper.join(room_id, user=self.other_user, tok=puppet_token) diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py index 5a1e5c4e66..c13a57dad1 100644 --- a/tests/storage/test_cleanup_extrems.py +++ b/tests/storage/test_cleanup_extrems.py @@ -309,36 +309,6 @@ class CleanupExtremDummyEventsTestCase(HomeserverTestCase): ) self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) - @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=0) - def test_send_dummy_event_without_consent(self): - self._create_extremity_rich_graph() - self._enable_consent_checking() - - # Pump the reactor repeatedly so that the background updates have a - # chance to run. Attempt to add dummy event with user that has not consented - # Check that dummy event send fails. - self.pump(10 * 60) - latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) - ) - self.assertTrue(len(latest_event_ids) == self.EXTREMITIES_COUNT) - - # Create new user, and add consent - user2 = self.register_user("user2", "password") - token2 = self.login("user2", "password") - self.get_success( - self.store.user_set_consent_version(user2, self.CONSENT_VERSION) - ) - self.helper.join(self.room_id, user2, tok=token2) - - # Background updates should now cause a dummy event to be added to the graph - self.pump(10 * 60) - - latest_event_ids = self.get_success( - self.store.get_latest_event_ids_in_room(self.room_id) - ) - self.assertTrue(len(latest_event_ids) < 10, len(latest_event_ids)) - @patch("synapse.handlers.message._DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY", new=250) def test_expiry_logic(self): """Simple test to ensure that _expire_rooms_to_exclude_from_dummy_event_insertion() diff --git a/tests/test_state.py b/tests/test_state.py index 80b0ccbc40..6227a3ba95 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -169,6 +169,7 @@ class StateTestCase(unittest.TestCase): "get_state_handler", "get_clock", "get_state_resolution_handler", + "hostname", ] ) hs.config = default_config("tesths", True) From 473dfec1e56cd4115886ad273dba5594ae02ea8a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 17 Nov 2020 09:09:40 -0500 Subject: [PATCH 25/57] Use TYPE_CHECKING instead of magic MYPY variable. (#8770) --- changelog.d/8770.misc | 1 + synapse/events/spamcheck.py | 5 ++--- synapse/handlers/presence.py | 5 ++--- synapse/rest/client/v1/room.py | 5 ++--- 4 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 changelog.d/8770.misc diff --git a/changelog.d/8770.misc b/changelog.d/8770.misc new file mode 100644 index 0000000000..b5876a82f9 --- /dev/null +++ b/changelog.d/8770.misc @@ -0,0 +1 @@ +Use `TYPE_CHECKING` instead of magic `MYPY` variable. diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py index bad18f7fdf..936896656a 100644 --- a/synapse/events/spamcheck.py +++ b/synapse/events/spamcheck.py @@ -15,13 +15,12 @@ # limitations under the License. import inspect -from typing import Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from synapse.spam_checker_api import RegistrationBehaviour from synapse.types import Collection -MYPY = False -if MYPY: +if TYPE_CHECKING: import synapse.events import synapse.server diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8e014c9bb5..22d1e9d35c 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -25,7 +25,7 @@ The methods that define policy are: import abc import logging from contextlib import contextmanager -from typing import Dict, Iterable, List, Set, Tuple +from typing import TYPE_CHECKING, Dict, Iterable, List, Set, Tuple from prometheus_client import Counter from typing_extensions import ContextManager @@ -46,8 +46,7 @@ from synapse.util.caches.descriptors import cached from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer -MYPY = False -if MYPY: +if TYPE_CHECKING: from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 25d3cc6148..93c06afe27 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -18,7 +18,7 @@ import logging import re -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from urllib import parse as urlparse from synapse.api.constants import EventTypes, Membership @@ -48,8 +48,7 @@ from synapse.types import RoomAlias, RoomID, StreamToken, ThirdPartyInstanceID, from synapse.util import json_decoder from synapse.util.stringutils import random_string -MYPY = False -if MYPY: +if TYPE_CHECKING: import synapse.server logger = logging.getLogger(__name__) From e487d9fabc24934743442a5b70ab371d756aa1dc Mon Sep 17 00:00:00 2001 From: chagai95 <31655082+chagai95@users.noreply.github.com> Date: Tue, 17 Nov 2020 15:13:56 +0100 Subject: [PATCH 26/57] a comma too much (#8771) Signed-off-by: Chagai Friedlander chagai95@gmail.com --- changelog.d/8771.doc | 1 + docs/admin_api/user_admin_api.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/8771.doc diff --git a/changelog.d/8771.doc b/changelog.d/8771.doc new file mode 100644 index 0000000000..297cf61e98 --- /dev/null +++ b/changelog.d/8771.doc @@ -0,0 +1 @@ +Remove extraneous comma from JSON example in User Admin API docs. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.rst b/docs/admin_api/user_admin_api.rst index 95b3da27c4..84863296e3 100644 --- a/docs/admin_api/user_admin_api.rst +++ b/docs/admin_api/user_admin_api.rst @@ -254,7 +254,7 @@ with a body of: { "new_password": "", - "logout_devices": true, + "logout_devices": true } To use it, you will need to authenticate by providing an ``access_token`` for a From ee382025b0c264701fc320133912e9fece40b021 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 17 Nov 2020 09:46:23 -0500 Subject: [PATCH 27/57] Abstract shared SSO code. (#8765) De-duplicates code between the SAML and OIDC implementations. --- changelog.d/8765.misc | 1 + synapse/handlers/oidc_handler.py | 92 ++++++++++++-------------------- synapse/handlers/saml_handler.py | 77 ++++++++------------------ synapse/handlers/sso.py | 90 +++++++++++++++++++++++++++++++ synapse/server.py | 5 ++ tests/handlers/test_oidc.py | 14 ++--- 6 files changed, 159 insertions(+), 120 deletions(-) create mode 100644 changelog.d/8765.misc create mode 100644 synapse/handlers/sso.py diff --git a/changelog.d/8765.misc b/changelog.d/8765.misc new file mode 100644 index 0000000000..053f9acc9c --- /dev/null +++ b/changelog.d/8765.misc @@ -0,0 +1 @@ +Consolidate logic between the OpenID Connect and SAML code. diff --git a/synapse/handlers/oidc_handler.py b/synapse/handlers/oidc_handler.py index 331d4e7e96..be8562d47b 100644 --- a/synapse/handlers/oidc_handler.py +++ b/synapse/handlers/oidc_handler.py @@ -34,7 +34,8 @@ from typing_extensions import TypedDict from twisted.web.client import readBody from synapse.config import ConfigError -from synapse.http.server import respond_with_html +from synapse.handlers._base import BaseHandler +from synapse.handlers.sso import MappingException from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable from synapse.types import JsonDict, UserID, map_username_to_mxid_localpart @@ -83,17 +84,12 @@ class OidcError(Exception): return self.error -class MappingException(Exception): - """Used to catch errors when mapping the UserInfo object - """ - - -class OidcHandler: +class OidcHandler(BaseHandler): """Handles requests related to the OpenID Connect login flow. """ def __init__(self, hs: "HomeServer"): - self.hs = hs + super().__init__(hs) self._callback_url = hs.config.oidc_callback_url # type: str self._scopes = hs.config.oidc_scopes # type: List[str] self._user_profile_method = hs.config.oidc_user_profile_method # type: str @@ -120,36 +116,13 @@ class OidcHandler: self._http_client = hs.get_proxied_http_client() self._auth_handler = hs.get_auth_handler() self._registration_handler = hs.get_registration_handler() - self._datastore = hs.get_datastore() - self._clock = hs.get_clock() - self._hostname = hs.hostname # type: str self._server_name = hs.config.server_name # type: str self._macaroon_secret_key = hs.config.macaroon_secret_key - self._error_template = hs.config.sso_error_template # identifier for the external_ids table self._auth_provider_id = "oidc" - def _render_error( - self, request, error: str, error_description: Optional[str] = None - ) -> None: - """Render the error template and respond to the request with it. - - This is used to show errors to the user. The template of this page can - be found under `synapse/res/templates/sso_error.html`. - - Args: - request: The incoming request from the browser. - We'll respond with an HTML page describing the error. - error: A technical identifier for this error. Those include - well-known OAuth2/OIDC error types like invalid_request or - access_denied. - error_description: A human-readable description of the error. - """ - html = self._error_template.render( - error=error, error_description=error_description - ) - respond_with_html(request, 400, html) + self._sso_handler = hs.get_sso_handler() def _validate_metadata(self): """Verifies the provider metadata. @@ -571,7 +544,7 @@ class OidcHandler: Since we might want to display OIDC-related errors in a user-friendly way, we don't raise SynapseError from here. Instead, we call - ``self._render_error`` which displays an HTML page for the error. + ``self._sso_handler.render_error`` which displays an HTML page for the error. Most of the OpenID Connect logic happens here: @@ -609,7 +582,7 @@ class OidcHandler: if error != "access_denied": logger.error("Error from the OIDC provider: %s %s", error, description) - self._render_error(request, error, description) + self._sso_handler.render_error(request, error, description) return # otherwise, it is presumably a successful response. see: @@ -619,7 +592,9 @@ class OidcHandler: session = request.getCookie(SESSION_COOKIE_NAME) # type: Optional[bytes] if session is None: logger.info("No session cookie found") - self._render_error(request, "missing_session", "No session cookie found") + self._sso_handler.render_error( + request, "missing_session", "No session cookie found" + ) return # Remove the cookie. There is a good chance that if the callback failed @@ -637,7 +612,9 @@ class OidcHandler: # Check for the state query parameter if b"state" not in request.args: logger.info("State parameter is missing") - self._render_error(request, "invalid_request", "State parameter is missing") + self._sso_handler.render_error( + request, "invalid_request", "State parameter is missing" + ) return state = request.args[b"state"][0].decode() @@ -651,17 +628,19 @@ class OidcHandler: ) = self._verify_oidc_session_token(session, state) except MacaroonDeserializationException as e: logger.exception("Invalid session") - self._render_error(request, "invalid_session", str(e)) + self._sso_handler.render_error(request, "invalid_session", str(e)) return except MacaroonInvalidSignatureException as e: logger.exception("Could not verify session") - self._render_error(request, "mismatching_session", str(e)) + self._sso_handler.render_error(request, "mismatching_session", str(e)) return # Exchange the code with the provider if b"code" not in request.args: logger.info("Code parameter is missing") - self._render_error(request, "invalid_request", "Code parameter is missing") + self._sso_handler.render_error( + request, "invalid_request", "Code parameter is missing" + ) return logger.debug("Exchanging code") @@ -670,7 +649,7 @@ class OidcHandler: token = await self._exchange_code(code) except OidcError as e: logger.exception("Could not exchange code") - self._render_error(request, e.error, e.error_description) + self._sso_handler.render_error(request, e.error, e.error_description) return logger.debug("Successfully obtained OAuth2 access token") @@ -683,7 +662,7 @@ class OidcHandler: userinfo = await self._fetch_userinfo(token) except Exception as e: logger.exception("Could not fetch userinfo") - self._render_error(request, "fetch_error", str(e)) + self._sso_handler.render_error(request, "fetch_error", str(e)) return else: logger.debug("Extracting userinfo from id_token") @@ -691,7 +670,7 @@ class OidcHandler: userinfo = await self._parse_id_token(token, nonce=nonce) except Exception as e: logger.exception("Invalid id_token") - self._render_error(request, "invalid_token", str(e)) + self._sso_handler.render_error(request, "invalid_token", str(e)) return # Pull out the user-agent and IP from the request. @@ -705,7 +684,7 @@ class OidcHandler: ) except MappingException as e: logger.exception("Could not map user") - self._render_error(request, "mapping_error", str(e)) + self._sso_handler.render_error(request, "mapping_error", str(e)) return # Mapping providers might not have get_extra_attributes: only call this @@ -770,7 +749,7 @@ class OidcHandler: macaroon.add_first_party_caveat( "ui_auth_session_id = %s" % (ui_auth_session_id,) ) - now = self._clock.time_msec() + now = self.clock.time_msec() expiry = now + duration_in_ms macaroon.add_first_party_caveat("time < %d" % (expiry,)) @@ -845,7 +824,7 @@ class OidcHandler: if not caveat.startswith(prefix): return False expiry = int(caveat[len(prefix) :]) - now = self._clock.time_msec() + now = self.clock.time_msec() return now < expiry async def _map_userinfo_to_user( @@ -885,20 +864,14 @@ class OidcHandler: # to be strings. remote_user_id = str(remote_user_id) - logger.info( - "Looking for existing mapping for user %s:%s", - self._auth_provider_id, - remote_user_id, - ) - - registered_user_id = await self._datastore.get_user_by_external_id( + # first of all, check if we already have a mapping for this user + previously_registered_user_id = await self._sso_handler.get_sso_user_by_remote_user_id( self._auth_provider_id, remote_user_id, ) + if previously_registered_user_id: + return previously_registered_user_id - if registered_user_id is not None: - logger.info("Found existing mapping %s", registered_user_id) - return registered_user_id - + # Otherwise, generate a new user. try: attributes = await self._user_mapping_provider.map_user_attributes( userinfo, token @@ -917,8 +890,8 @@ class OidcHandler: localpart = map_username_to_mxid_localpart(attributes["localpart"]) - user_id = UserID(localpart, self._hostname).to_string() - users = await self._datastore.get_users_by_id_case_insensitive(user_id) + user_id = UserID(localpart, self.server_name).to_string() + users = await self.store.get_users_by_id_case_insensitive(user_id) if users: if self._allow_existing_users: if len(users) == 1: @@ -942,7 +915,8 @@ class OidcHandler: default_display_name=attributes["display_name"], user_agent_ips=(user_agent, ip_address), ) - await self._datastore.record_user_external_id( + + await self.store.record_user_external_id( self._auth_provider_id, remote_user_id, registered_user_id, ) return registered_user_id diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index fd6c5e9ea8..aee772239a 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -24,7 +24,8 @@ from saml2.client import Saml2Client from synapse.api.errors import SynapseError from synapse.config import ConfigError from synapse.config.saml2_config import SamlAttributeRequirement -from synapse.http.server import respond_with_html +from synapse.handlers._base import BaseHandler +from synapse.handlers.sso import MappingException from synapse.http.servlet import parse_string from synapse.http.site import SynapseRequest from synapse.module_api import ModuleApi @@ -42,10 +43,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -class MappingException(Exception): - """Used to catch errors when mapping the SAML2 response to a user.""" - - @attr.s(slots=True) class Saml2SessionData: """Data we track about SAML2 sessions""" @@ -57,17 +54,13 @@ class Saml2SessionData: ui_auth_session_id = attr.ib(type=Optional[str], default=None) -class SamlHandler: +class SamlHandler(BaseHandler): def __init__(self, hs: "synapse.server.HomeServer"): - self.hs = hs + super().__init__(hs) self._saml_client = Saml2Client(hs.config.saml2_sp_config) - self._auth = hs.get_auth() self._auth_handler = hs.get_auth_handler() self._registration_handler = hs.get_registration_handler() - self._clock = hs.get_clock() - self._datastore = hs.get_datastore() - self._hostname = hs.hostname self._saml2_session_lifetime = hs.config.saml2_session_lifetime self._grandfathered_mxid_source_attribute = ( hs.config.saml2_grandfathered_mxid_source_attribute @@ -88,26 +81,9 @@ class SamlHandler: self._outstanding_requests_dict = {} # type: Dict[str, Saml2SessionData] # a lock on the mappings - self._mapping_lock = Linearizer(name="saml_mapping", clock=self._clock) + self._mapping_lock = Linearizer(name="saml_mapping", clock=self.clock) - def _render_error( - self, request, error: str, error_description: Optional[str] = None - ) -> None: - """Render the error template and respond to the request with it. - - This is used to show errors to the user. The template of this page can - be found under `synapse/res/templates/sso_error.html`. - - Args: - request: The incoming request from the browser. - We'll respond with an HTML page describing the error. - error: A technical identifier for this error. - error_description: A human-readable description of the error. - """ - html = self._error_template.render( - error=error, error_description=error_description - ) - respond_with_html(request, 400, html) + self._sso_handler = hs.get_sso_handler() def handle_redirect_request( self, client_redirect_url: bytes, ui_auth_session_id: Optional[str] = None @@ -130,7 +106,7 @@ class SamlHandler: # Since SAML sessions timeout it is useful to log when they were created. logger.info("Initiating a new SAML session: %s" % (reqid,)) - now = self._clock.time_msec() + now = self.clock.time_msec() self._outstanding_requests_dict[reqid] = Saml2SessionData( creation_time=now, ui_auth_session_id=ui_auth_session_id, ) @@ -171,12 +147,12 @@ class SamlHandler: # in the (user-visible) exception message, so let's log the exception here # so we can track down the session IDs later. logger.warning(str(e)) - self._render_error( + self._sso_handler.render_error( request, "unsolicited_response", "Unexpected SAML2 login." ) return except Exception as e: - self._render_error( + self._sso_handler.render_error( request, "invalid_response", "Unable to parse SAML2 response: %s." % (e,), @@ -184,7 +160,7 @@ class SamlHandler: return if saml2_auth.not_signed: - self._render_error( + self._sso_handler.render_error( request, "unsigned_respond", "SAML2 response was not signed." ) return @@ -210,7 +186,7 @@ class SamlHandler: # attributes. for requirement in self._saml2_attribute_requirements: if not _check_attribute_requirement(saml2_auth.ava, requirement): - self._render_error( + self._sso_handler.render_error( request, "unauthorised", "You are not authorised to log in here." ) return @@ -226,7 +202,7 @@ class SamlHandler: ) except MappingException as e: logger.exception("Could not map user") - self._render_error(request, "mapping_error", str(e)) + self._sso_handler.render_error(request, "mapping_error", str(e)) return # Complete the interactive auth session or the login. @@ -274,17 +250,11 @@ class SamlHandler: with (await self._mapping_lock.queue(self._auth_provider_id)): # first of all, check if we already have a mapping for this user - logger.info( - "Looking for existing mapping for user %s:%s", - self._auth_provider_id, - remote_user_id, + previously_registered_user_id = await self._sso_handler.get_sso_user_by_remote_user_id( + self._auth_provider_id, remote_user_id, ) - registered_user_id = await self._datastore.get_user_by_external_id( - self._auth_provider_id, remote_user_id - ) - if registered_user_id is not None: - logger.info("Found existing mapping %s", registered_user_id) - return registered_user_id + if previously_registered_user_id: + return previously_registered_user_id # backwards-compatibility hack: see if there is an existing user with a # suitable mapping from the uid @@ -294,7 +264,7 @@ class SamlHandler: ): attrval = saml2_auth.ava[self._grandfathered_mxid_source_attribute][0] user_id = UserID( - map_username_to_mxid_localpart(attrval), self._hostname + map_username_to_mxid_localpart(attrval), self.server_name ).to_string() logger.info( "Looking for existing account based on mapped %s %s", @@ -302,11 +272,11 @@ class SamlHandler: user_id, ) - users = await self._datastore.get_users_by_id_case_insensitive(user_id) + users = await self.store.get_users_by_id_case_insensitive(user_id) if users: registered_user_id = list(users.keys())[0] logger.info("Grandfathering mapping to %s", registered_user_id) - await self._datastore.record_user_external_id( + await self.store.record_user_external_id( self._auth_provider_id, remote_user_id, registered_user_id ) return registered_user_id @@ -335,8 +305,8 @@ class SamlHandler: emails = attribute_dict.get("emails", []) # Check if this mxid already exists - if not await self._datastore.get_users_by_id_case_insensitive( - UserID(localpart, self._hostname).to_string() + if not await self.store.get_users_by_id_case_insensitive( + UserID(localpart, self.server_name).to_string() ): # This mxid is free break @@ -348,7 +318,6 @@ class SamlHandler: ) logger.info("Mapped SAML user to local part %s", localpart) - registered_user_id = await self._registration_handler.register_user( localpart=localpart, default_display_name=displayname, @@ -356,13 +325,13 @@ class SamlHandler: user_agent_ips=(user_agent, ip_address), ) - await self._datastore.record_user_external_id( + await self.store.record_user_external_id( self._auth_provider_id, remote_user_id, registered_user_id ) return registered_user_id def expire_sessions(self): - expire_before = self._clock.time_msec() - self._saml2_session_lifetime + expire_before = self.clock.time_msec() - self._saml2_session_lifetime to_expire = set() for reqid, data in self._outstanding_requests_dict.items(): if data.creation_time < expire_before: diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py new file mode 100644 index 0000000000..9cb1866a71 --- /dev/null +++ b/synapse/handlers/sso.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from typing import TYPE_CHECKING, Optional + +from synapse.handlers._base import BaseHandler +from synapse.http.server import respond_with_html + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class MappingException(Exception): + """Used to catch errors when mapping the UserInfo object + """ + + +class SsoHandler(BaseHandler): + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + self._error_template = hs.config.sso_error_template + + def render_error( + self, request, error: str, error_description: Optional[str] = None + ) -> None: + """Renders the error template and responds with it. + + This is used to show errors to the user. The template of this page can + be found under `synapse/res/templates/sso_error.html`. + + Args: + request: The incoming request from the browser. + We'll respond with an HTML page describing the error. + error: A technical identifier for this error. + error_description: A human-readable description of the error. + """ + html = self._error_template.render( + error=error, error_description=error_description + ) + respond_with_html(request, 400, html) + + async def get_sso_user_by_remote_user_id( + self, auth_provider_id: str, remote_user_id: str + ) -> Optional[str]: + """ + Maps the user ID of a remote IdP to a mxid for a previously seen user. + + If the user has not been seen yet, this will return None. + + Args: + auth_provider_id: A unique identifier for this SSO provider, e.g. + "oidc" or "saml". + remote_user_id: The user ID according to the remote IdP. This might + be an e-mail address, a GUID, or some other form. It must be + unique and immutable. + + Returns: + The mxid of a previously seen user. + """ + # Check if we already have a mapping for this user. + logger.info( + "Looking for existing mapping for user %s:%s", + auth_provider_id, + remote_user_id, + ) + previously_registered_user_id = await self.store.get_user_by_external_id( + auth_provider_id, remote_user_id, + ) + + # A match was found, return the user ID. + if previously_registered_user_id is not None: + logger.info("Found existing mapping %s", previously_registered_user_id) + return previously_registered_user_id + + # No match. + return None diff --git a/synapse/server.py b/synapse/server.py index 21a232bbd9..12a783de17 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -89,6 +89,7 @@ from synapse.handlers.room_member import RoomMemberMasterHandler from synapse.handlers.room_member_worker import RoomMemberWorkerHandler from synapse.handlers.search import SearchHandler from synapse.handlers.set_password import SetPasswordHandler +from synapse.handlers.sso import SsoHandler from synapse.handlers.stats import StatsHandler from synapse.handlers.sync import SyncHandler from synapse.handlers.typing import FollowerTypingHandler, TypingWriterHandler @@ -390,6 +391,10 @@ class HomeServer(metaclass=abc.ABCMeta): else: return FollowerTypingHandler(self) + @cache_in_self + def get_sso_handler(self) -> SsoHandler: + return SsoHandler(self) + @cache_in_self def get_sync_handler(self) -> SyncHandler: return SyncHandler(self) diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index 0d51705849..630e6da808 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -154,6 +154,9 @@ class OidcHandlerTestCase(HomeserverTestCase): ) self.handler = OidcHandler(hs) + # Mock the render error method. + self.render_error = Mock(return_value=None) + self.handler._sso_handler.render_error = self.render_error return hs @@ -161,12 +164,12 @@ class OidcHandlerTestCase(HomeserverTestCase): return patch.dict(self.handler._provider_metadata, values) def assertRenderedError(self, error, error_description=None): - args = self.handler._render_error.call_args[0] + args = self.render_error.call_args[0] self.assertEqual(args[1], error) if error_description is not None: self.assertEqual(args[2], error_description) # Reset the render_error mock - self.handler._render_error.reset_mock() + self.render_error.reset_mock() def test_config(self): """Basic config correctly sets up the callback URL and client auth correctly.""" @@ -356,7 +359,6 @@ class OidcHandlerTestCase(HomeserverTestCase): def test_callback_error(self): """Errors from the provider returned in the callback are displayed.""" - self.handler._render_error = Mock() request = Mock(args={}) request.args[b"error"] = [b"invalid_client"] self.get_success(self.handler.handle_oidc_callback(request)) @@ -387,7 +389,6 @@ class OidcHandlerTestCase(HomeserverTestCase): "preferred_username": "bar", } user_id = "@foo:domain.org" - self.handler._render_error = Mock(return_value=None) self.handler._exchange_code = simple_async_mock(return_value=token) self.handler._parse_id_token = simple_async_mock(return_value=userinfo) self.handler._fetch_userinfo = simple_async_mock(return_value=userinfo) @@ -435,7 +436,7 @@ class OidcHandlerTestCase(HomeserverTestCase): userinfo, token, user_agent, ip_address ) self.handler._fetch_userinfo.assert_not_called() - self.handler._render_error.assert_not_called() + self.render_error.assert_not_called() # Handle mapping errors self.handler._map_userinfo_to_user = simple_async_mock( @@ -469,7 +470,7 @@ class OidcHandlerTestCase(HomeserverTestCase): userinfo, token, user_agent, ip_address ) self.handler._fetch_userinfo.assert_called_once_with(token) - self.handler._render_error.assert_not_called() + self.render_error.assert_not_called() # Handle userinfo fetching error self.handler._fetch_userinfo = simple_async_mock(raises=Exception()) @@ -485,7 +486,6 @@ class OidcHandlerTestCase(HomeserverTestCase): def test_callback_session(self): """The callback verifies the session presence and validity""" - self.handler._render_error = Mock(return_value=None) request = Mock(spec=["args", "getCookie", "addCookie"]) # Missing cookie From c087f680531b2c40a4dc731da64b351a133c27fe Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 17 Nov 2020 16:01:33 +0000 Subject: [PATCH 28/57] Cap the version of prometheus_client to =0.4.0,<0.9.0" \ psycopg2 \ pycparser \ pyrsistent \ diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 0ddead8a0f..aab77fc453 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -72,6 +72,10 @@ REQUIREMENTS = [ # prom-client has a history of breaking backwards compatibility between # minor versions (https://github.com/prometheus/client_python/issues/317), # so we also pin the minor version. + # + # Note that we replicate these constraints in the Synapse Dockerfile while + # pre-installing dependencies. If these constraints are updated here, the + # same change should be made in the Dockerfile. "prometheus_client>=0.4.0,<0.9.0", # we use attr.validators.deep_iterable, which arrived in 19.1.0 (Note: # Fedora 31 only has 19.1, so if we want to upgrade we should wait until 33 From ef366720d5745dac48a87802a40d5cb359b92e4b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2020 11:41:41 +0000 Subject: [PATCH 29/57] 1.23.0 --- CHANGES.md | 9 +++++++++ changelog.d/8767.bugfix | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/8767.bugfix diff --git a/CHANGES.md b/CHANGES.md index 75871979c2..15d7d0aeea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.23.0 (2020-11-18) +=========================== + +Bugfixes +-------- + +- Fix a dependency versioning bug in the Dockerfile that prevented Synapse from starting. ([\#8767](https://github.com/matrix-org/synapse/issues/8767)) + + Synapse 1.23.0rc1 (2020-11-13) ============================== diff --git a/changelog.d/8767.bugfix b/changelog.d/8767.bugfix deleted file mode 100644 index 9fe5107a5e..0000000000 --- a/changelog.d/8767.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a dependency versioning bug in the Dockerfile that prevented Synapse from starting. diff --git a/debian/changelog b/debian/changelog index ae8948650f..4ea4feddd5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.23.0) stable; urgency=medium + + * New synapse release 1.23.0. + + -- Synapse Packaging team Wed, 18 Nov 2020 11:41:28 +0000 + matrix-synapse-py3 (1.22.1) stable; urgency=medium * New synapse release 1.22.1. diff --git a/synapse/__init__.py b/synapse/__init__.py index 537f2239e5..65c1f5aa3f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -48,7 +48,7 @@ try: except ImportError: pass -__version__ = "1.23.0rc1" +__version__ = "1.23.0" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 59c8f4f0db904fa0560afcc20a70cbb76dfe752b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2020 11:57:19 +0000 Subject: [PATCH 30/57] Update changelog --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 15d7d0aeea..d444259861 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,10 @@ Synapse 1.23.0 (2020-11-18) =========================== +This release changes the way structured logging is configured. See the [upgrade notes](UPGRADE.rst#upgrading-to-v1230) for details. + +**Note**: We are aware of a trivially exploitable denial of service vulnerability in versions of Synapse prior to 1.20.0. Complete details will be disclosed on Monday, November 23rd. If you have not upgraded recently, please do so. + Bugfixes -------- From 0285885babdbff2930dbb617a0ddb4aaabad53fb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2020 12:00:13 +0000 Subject: [PATCH 31/57] Fix formatting in upgrades --- UPGRADE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index 960c2aeb2b..7c19cf2a70 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -87,7 +87,7 @@ then it should be modified based on the `structured logging documentation `_. The ``structured`` and ``drains`` logging options are now deprecated and should -be replaced by standard logging configuration of ``handlers`` and ``formatters`. +be replaced by standard logging configuration of ``handlers`` and ``formatters``. A future will release of Synapse will make using ``structured: true`` an error. From 244bff4edd620998abd033050922994e74294c16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2020 12:04:08 +0000 Subject: [PATCH 32/57] Update changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d444259861..52b2fd6f8f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -65,7 +65,7 @@ Internal Changes ---------------- - Optimise `/createRoom` with multiple invited users. ([\#8559](https://github.com/matrix-org/synapse/issues/8559)) -- Implement and use an @lru_cache decorator. ([\#8595](https://github.com/matrix-org/synapse/issues/8595)) +- Implement and use an `@lru_cache` decorator. ([\#8595](https://github.com/matrix-org/synapse/issues/8595)) - Don't instansiate Requester directly. ([\#8614](https://github.com/matrix-org/synapse/issues/8614)) - Type hints for `RegistrationStore`. ([\#8615](https://github.com/matrix-org/synapse/issues/8615)) - Change schema to support access tokens belonging to one user but granting access to another. ([\#8616](https://github.com/matrix-org/synapse/issues/8616)) From d356588339db627397e84e98d3468a4c996253ab Mon Sep 17 00:00:00 2001 From: Marcus Schopen Date: Wed, 18 Nov 2020 13:36:28 +0100 Subject: [PATCH 33/57] SAML: Document allowing a clock/time difference from IdP (#8731) Updates the sample configuration with the pysaml2 configuration for accepting clock skew/drift between the homeserver and IdP. --- changelog.d/8731.misc | 1 + docs/sample_config.yaml | 6 ++++++ synapse/config/saml2_config.py | 6 ++++++ 3 files changed, 13 insertions(+) create mode 100644 changelog.d/8731.misc diff --git a/changelog.d/8731.misc b/changelog.d/8731.misc new file mode 100644 index 0000000000..df5882e960 --- /dev/null +++ b/changelog.d/8731.misc @@ -0,0 +1 @@ +Add an example and documentation for clock skew to the SAML2 sample configuration to allow for clock/time difference between the homserver and IdP. Contributed by @localguru. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index e9e77ca94e..bedc147770 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1546,6 +1546,12 @@ saml2_config: # remote: # - url: https://our_idp/metadata.xml + # Allowed clock difference in seconds between the homeserver and IdP. + # + # Uncomment the below to increase the accepted time difference from 0 to 3 seconds. + # + #accepted_time_diff: 3 + # By default, the user has to go to our login page first. If you'd like # to allow IdP-initiated login, set 'allow_unsolicited: true' in a # 'service.sp' section: diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py index 2ff7dfb311..f233854941 100644 --- a/synapse/config/saml2_config.py +++ b/synapse/config/saml2_config.py @@ -256,6 +256,12 @@ class SAML2Config(Config): # remote: # - url: https://our_idp/metadata.xml + # Allowed clock difference in seconds between the homeserver and IdP. + # + # Uncomment the below to increase the accepted time difference from 0 to 3 seconds. + # + #accepted_time_diff: 3 + # By default, the user has to go to our login page first. If you'd like # to allow IdP-initiated login, set 'allow_unsolicited: true' in a # 'service.sp' section: From 03e392f787df3fb8d23aca0964695a5caa075940 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2020 15:43:11 +0000 Subject: [PATCH 34/57] Fix tests on develop (#8777) This was broken due to #8617 and #8761. --- changelog.d/8777.misc | 1 + tests/rest/admin/test_user.py | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) create mode 100644 changelog.d/8777.misc diff --git a/changelog.d/8777.misc b/changelog.d/8777.misc new file mode 100644 index 0000000000..e6da7d038d --- /dev/null +++ b/changelog.d/8777.misc @@ -0,0 +1 @@ + Refactor test utilities for injecting HTTP requests. diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index cc8a70be04..9661af7e79 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -1574,7 +1574,6 @@ class UserTokenRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", self.url, b"{}", access_token=self.admin_user_tok ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) return channel.json_body["access_token"] @@ -1582,7 +1581,6 @@ class UserTokenRestTestCase(unittest.HomeserverTestCase): """Try to login as a user without authentication. """ request, channel = self.make_request("POST", self.url, b"{}") - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) @@ -1593,7 +1591,6 @@ class UserTokenRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "POST", self.url, b"{}", access_token=self.other_user_tok ) - self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) @@ -1622,7 +1619,6 @@ class UserTokenRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "devices", b"{}", access_token=self.other_user_tok ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # We should only see the one device (from the login in `prepare`) @@ -1638,28 +1634,24 @@ class UserTokenRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "devices", b"{}", access_token=puppet_token ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Logout with the puppet token request, channel = self.make_request( "POST", "logout", b"{}", access_token=puppet_token ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # The puppet token should no longer work request, channel = self.make_request( "GET", "devices", b"{}", access_token=puppet_token ) - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) # .. but the real user's tokens should still work request, channel = self.make_request( "GET", "devices", b"{}", access_token=self.other_user_tok ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) def test_user_logout_all(self): @@ -1673,28 +1665,24 @@ class UserTokenRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "devices", b"{}", access_token=puppet_token ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Logout all with the real user token request, channel = self.make_request( "POST", "logout/all", b"{}", access_token=self.other_user_tok ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # The puppet token should still work request, channel = self.make_request( "GET", "devices", b"{}", access_token=puppet_token ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # .. but the real user's tokens shouldn't request, channel = self.make_request( "GET", "devices", b"{}", access_token=self.other_user_tok ) - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) def test_admin_logout_all(self): @@ -1708,28 +1696,24 @@ class UserTokenRestTestCase(unittest.HomeserverTestCase): request, channel = self.make_request( "GET", "devices", b"{}", access_token=puppet_token ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Logout all with the admin user token request, channel = self.make_request( "POST", "logout/all", b"{}", access_token=self.admin_user_tok ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # The puppet token should no longer work request, channel = self.make_request( "GET", "devices", b"{}", access_token=puppet_token ) - self.render(request) self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) # .. but the real user's tokens should still work request, channel = self.make_request( "GET", "devices", b"{}", access_token=self.other_user_tok ) - self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) @unittest.override_config( From 51338491c9bedcdfb5f9babad2a34cbfea6b57d2 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 18 Nov 2020 18:54:09 +0000 Subject: [PATCH 35/57] Improve appservice handler to send only the most recent read receipts when no stream_id is stored. (#8744) * Make this line debug (it's noisy) * Don't include from_key for presence if we are at 0 * Limit read receipts for all rooms to 100 * changelog.d/8744.bugfix * Allow from_key to be None * Update 8744.bugfix * The from_key is superflous * Update comment --- changelog.d/8744.bugfix | 1 + synapse/handlers/appservice.py | 2 +- synapse/handlers/receipts.py | 3 ++- synapse/storage/databases/main/receipts.py | 7 ++++++- 4 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 changelog.d/8744.bugfix diff --git a/changelog.d/8744.bugfix b/changelog.d/8744.bugfix new file mode 100644 index 0000000000..f8f9630bd6 --- /dev/null +++ b/changelog.d/8744.bugfix @@ -0,0 +1 @@ +Fix a bug where appservices may be sent an excessive amount of read receipts and presence. Broke in v1.22.0. diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9fc8444228..5c6458eb52 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -226,7 +226,7 @@ class ApplicationServicesHandler: new_token: Optional[int], users: Collection[Union[str, UserID]], ): - logger.info("Checking interested services for %s" % (stream_key)) + logger.debug("Checking interested services for %s" % (stream_key)) with Measure(self.clock, "notify_interested_services_ephemeral"): for service in services: # Only handle typing if we have the latest token diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index c242c409cf..153cbae7b9 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -158,7 +158,8 @@ class ReceiptEventSource: if from_key == to_key: return [], to_key - # We first need to fetch all new receipts + # Fetch all read receipts for all rooms, up to a limit of 100. This is ordered + # by most recent. rooms_to_events = await self.store.get_linearized_receipts_for_all_rooms( from_key=from_key, to_key=to_key ) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index ca7917c989..1e7949a323 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -278,7 +278,8 @@ class ReceiptsWorkerStore(SQLBaseStore, metaclass=abc.ABCMeta): async def get_linearized_receipts_for_all_rooms( self, to_key: int, from_key: Optional[int] = None ) -> Dict[str, JsonDict]: - """Get receipts for all rooms between two stream_ids. + """Get receipts for all rooms between two stream_ids, up + to a limit of the latest 100 read receipts. Args: to_key: Max stream id to fetch receipts upto. @@ -294,12 +295,16 @@ class ReceiptsWorkerStore(SQLBaseStore, metaclass=abc.ABCMeta): sql = """ SELECT * FROM receipts_linearized WHERE stream_id > ? AND stream_id <= ? + ORDER BY stream_id DESC + LIMIT 100 """ txn.execute(sql, [from_key, to_key]) else: sql = """ SELECT * FROM receipts_linearized WHERE stream_id <= ? + ORDER BY stream_id DESC + LIMIT 100 """ txn.execute(sql, [to_key]) From 950bb0305fc3c114f456eb4e2a806014150545d2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 19 Nov 2020 10:05:33 +0000 Subject: [PATCH 36/57] Consistently use room_id from federation request body (#8776) * Consistently use room_id from federation request body Some federation APIs have a redundant `room_id` path param (see https://github.com/matrix-org/matrix-doc/issues/2330). We should make sure we consistently use either the path param or the body param, and the body param is easier. * Kill off some references to "context" Once upon a time, "rooms" were known as "contexts". I think this kills of the last references to "contexts". --- changelog.d/8776.bugfix | 1 + synapse/federation/federation_server.py | 23 ++++----- synapse/federation/transport/server.py | 68 ++++++++++++------------- synapse/handlers/federation.py | 10 ++-- tests/handlers/test_federation.py | 1 - 5 files changed, 49 insertions(+), 54 deletions(-) create mode 100644 changelog.d/8776.bugfix diff --git a/changelog.d/8776.bugfix b/changelog.d/8776.bugfix new file mode 100644 index 0000000000..dd7ebbeb86 --- /dev/null +++ b/changelog.d/8776.bugfix @@ -0,0 +1 @@ +Fix a bug in some federation APIs which could lead to unexpected behaviour if different parameters were set in the URI and the request body. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 23278e36b7..4b6ab470d0 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -49,6 +49,7 @@ from synapse.federation.federation_base import FederationBase, event_from_pdu_js from synapse.federation.persistence import TransactionActions from synapse.federation.units import Edu, Transaction from synapse.http.endpoint import parse_server_name +from synapse.http.servlet import assert_params_in_dict from synapse.logging.context import ( make_deferred_yieldable, nested_logging_context, @@ -391,7 +392,7 @@ class FederationServer(FederationBase): TRANSACTION_CONCURRENCY_LIMIT, ) - async def on_context_state_request( + async def on_room_state_request( self, origin: str, room_id: str, event_id: str ) -> Tuple[int, Dict[str, Any]]: origin_host, _ = parse_server_name(origin) @@ -514,11 +515,12 @@ class FederationServer(FederationBase): return {"event": ret_pdu.get_pdu_json(time_now)} async def on_send_join_request( - self, origin: str, content: JsonDict, room_id: str + self, origin: str, content: JsonDict ) -> Dict[str, Any]: logger.debug("on_send_join_request: content: %s", content) - room_version = await self.store.get_room_version(room_id) + assert_params_in_dict(content, ["room_id"]) + room_version = await self.store.get_room_version(content["room_id"]) pdu = event_from_pdu_json(content, room_version) origin_host, _ = parse_server_name(origin) @@ -547,12 +549,11 @@ class FederationServer(FederationBase): time_now = self._clock.time_msec() return {"event": pdu.get_pdu_json(time_now), "room_version": room_version} - async def on_send_leave_request( - self, origin: str, content: JsonDict, room_id: str - ) -> dict: + async def on_send_leave_request(self, origin: str, content: JsonDict) -> dict: logger.debug("on_send_leave_request: content: %s", content) - room_version = await self.store.get_room_version(room_id) + assert_params_in_dict(content, ["room_id"]) + room_version = await self.store.get_room_version(content["room_id"]) pdu = event_from_pdu_json(content, room_version) origin_host, _ = parse_server_name(origin) @@ -748,12 +749,8 @@ class FederationServer(FederationBase): ) return ret - async def on_exchange_third_party_invite_request( - self, room_id: str, event_dict: Dict - ): - ret = await self.handler.on_exchange_third_party_invite_request( - room_id, event_dict - ) + async def on_exchange_third_party_invite_request(self, event_dict: Dict): + ret = await self.handler.on_exchange_third_party_invite_request(event_dict) return ret async def check_server_matches_acl(self, server_name: str, room_id: str): diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a0933fae88..b53e7a20ec 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -440,13 +440,13 @@ class FederationEventServlet(BaseFederationServlet): class FederationStateV1Servlet(BaseFederationServlet): - PATH = "/state/(?P[^/]*)/?" + PATH = "/state/(?P[^/]*)/?" - # This is when someone asks for all data for a given context. - async def on_GET(self, origin, content, query, context): - return await self.handler.on_context_state_request( + # This is when someone asks for all data for a given room. + async def on_GET(self, origin, content, query, room_id): + return await self.handler.on_room_state_request( origin, - context, + room_id, parse_string_from_args(query, "event_id", None, required=False), ) @@ -463,16 +463,16 @@ class FederationStateIdsServlet(BaseFederationServlet): class FederationBackfillServlet(BaseFederationServlet): - PATH = "/backfill/(?P[^/]*)/?" + PATH = "/backfill/(?P[^/]*)/?" - async def on_GET(self, origin, content, query, context): + async def on_GET(self, origin, content, query, room_id): versions = [x.decode("ascii") for x in query[b"v"]] limit = parse_integer_from_args(query, "limit", None) if not limit: return 400, {"error": "Did not include limit param"} - return await self.handler.on_backfill_request(origin, context, versions, limit) + return await self.handler.on_backfill_request(origin, room_id, versions, limit) class FederationQueryServlet(BaseFederationServlet): @@ -487,9 +487,9 @@ class FederationQueryServlet(BaseFederationServlet): class FederationMakeJoinServlet(BaseFederationServlet): - PATH = "/make_join/(?P[^/]*)/(?P[^/]*)" + PATH = "/make_join/(?P[^/]*)/(?P[^/]*)" - async def on_GET(self, origin, _content, query, context, user_id): + async def on_GET(self, origin, _content, query, room_id, user_id): """ Args: origin (unicode): The authenticated server_name of the calling server @@ -511,16 +511,16 @@ class FederationMakeJoinServlet(BaseFederationServlet): supported_versions = ["1"] content = await self.handler.on_make_join_request( - origin, context, user_id, supported_versions=supported_versions + origin, room_id, user_id, supported_versions=supported_versions ) return 200, content class FederationMakeLeaveServlet(BaseFederationServlet): - PATH = "/make_leave/(?P[^/]*)/(?P[^/]*)" + PATH = "/make_leave/(?P[^/]*)/(?P[^/]*)" - async def on_GET(self, origin, content, query, context, user_id): - content = await self.handler.on_make_leave_request(origin, context, user_id) + async def on_GET(self, origin, content, query, room_id, user_id): + content = await self.handler.on_make_leave_request(origin, room_id, user_id) return 200, content @@ -528,7 +528,7 @@ class FederationV1SendLeaveServlet(BaseFederationServlet): PATH = "/send_leave/(?P[^/]*)/(?P[^/]*)" async def on_PUT(self, origin, content, query, room_id, event_id): - content = await self.handler.on_send_leave_request(origin, content, room_id) + content = await self.handler.on_send_leave_request(origin, content) return 200, (200, content) @@ -538,43 +538,43 @@ class FederationV2SendLeaveServlet(BaseFederationServlet): PREFIX = FEDERATION_V2_PREFIX async def on_PUT(self, origin, content, query, room_id, event_id): - content = await self.handler.on_send_leave_request(origin, content, room_id) + content = await self.handler.on_send_leave_request(origin, content) return 200, content class FederationEventAuthServlet(BaseFederationServlet): - PATH = "/event_auth/(?P[^/]*)/(?P[^/]*)" + PATH = "/event_auth/(?P[^/]*)/(?P[^/]*)" - async def on_GET(self, origin, content, query, context, event_id): - return await self.handler.on_event_auth(origin, context, event_id) + async def on_GET(self, origin, content, query, room_id, event_id): + return await self.handler.on_event_auth(origin, room_id, event_id) class FederationV1SendJoinServlet(BaseFederationServlet): - PATH = "/send_join/(?P[^/]*)/(?P[^/]*)" + PATH = "/send_join/(?P[^/]*)/(?P[^/]*)" - async def on_PUT(self, origin, content, query, context, event_id): - # TODO(paul): assert that context/event_id parsed from path actually + async def on_PUT(self, origin, content, query, room_id, event_id): + # TODO(paul): assert that room_id/event_id parsed from path actually # match those given in content - content = await self.handler.on_send_join_request(origin, content, context) + content = await self.handler.on_send_join_request(origin, content) return 200, (200, content) class FederationV2SendJoinServlet(BaseFederationServlet): - PATH = "/send_join/(?P[^/]*)/(?P[^/]*)" + PATH = "/send_join/(?P[^/]*)/(?P[^/]*)" PREFIX = FEDERATION_V2_PREFIX - async def on_PUT(self, origin, content, query, context, event_id): - # TODO(paul): assert that context/event_id parsed from path actually + async def on_PUT(self, origin, content, query, room_id, event_id): + # TODO(paul): assert that room_id/event_id parsed from path actually # match those given in content - content = await self.handler.on_send_join_request(origin, content, context) + content = await self.handler.on_send_join_request(origin, content) return 200, content class FederationV1InviteServlet(BaseFederationServlet): - PATH = "/invite/(?P[^/]*)/(?P[^/]*)" + PATH = "/invite/(?P[^/]*)/(?P[^/]*)" - async def on_PUT(self, origin, content, query, context, event_id): + async def on_PUT(self, origin, content, query, room_id, event_id): # We don't get a room version, so we have to assume its EITHER v1 or # v2. This is "fine" as the only difference between V1 and V2 is the # state resolution algorithm, and we don't use that for processing @@ -589,12 +589,12 @@ class FederationV1InviteServlet(BaseFederationServlet): class FederationV2InviteServlet(BaseFederationServlet): - PATH = "/invite/(?P[^/]*)/(?P[^/]*)" + PATH = "/invite/(?P[^/]*)/(?P[^/]*)" PREFIX = FEDERATION_V2_PREFIX - async def on_PUT(self, origin, content, query, context, event_id): - # TODO(paul): assert that context/event_id parsed from path actually + async def on_PUT(self, origin, content, query, room_id, event_id): + # TODO(paul): assert that room_id/event_id parsed from path actually # match those given in content room_version = content["room_version"] @@ -616,9 +616,7 @@ class FederationThirdPartyInviteExchangeServlet(BaseFederationServlet): PATH = "/exchange_third_party_invite/(?P[^/]*)" async def on_PUT(self, origin, content, query, room_id): - content = await self.handler.on_exchange_third_party_invite_request( - room_id, content - ) + content = await self.handler.on_exchange_third_party_invite_request(content) return 200, content diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 69bc5ba44d..b9799090f7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -55,6 +55,7 @@ from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.events.validator import EventValidator from synapse.handlers._base import BaseHandler +from synapse.http.servlet import assert_params_in_dict from synapse.logging.context import ( make_deferred_yieldable, nested_logging_context, @@ -2688,7 +2689,7 @@ class FederationHandler(BaseHandler): ) async def on_exchange_third_party_invite_request( - self, room_id: str, event_dict: JsonDict + self, event_dict: JsonDict ) -> None: """Handle an exchange_third_party_invite request from a remote server @@ -2696,12 +2697,11 @@ class FederationHandler(BaseHandler): into a normal m.room.member invite. Args: - room_id: The ID of the room. - - event_dict (dict[str, Any]): Dictionary containing the event body. + event_dict: Dictionary containing the event body. """ - room_version = await self.store.get_room_version_id(room_id) + assert_params_in_dict(event_dict, ["room_id"]) + room_version = await self.store.get_room_version_id(event_dict["room_id"]) # NB: event_dict has a particular specced format we might need to fudge # if we change event formats too much. diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 9ef80fe502..bf866dacf3 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -59,7 +59,6 @@ class FederationTestCase(unittest.HomeserverTestCase): ) d = self.handler.on_exchange_third_party_invite_request( - room_id=room_id, event_dict={ "type": EventTypes.Member, "room_id": room_id, From 53a6f5ddf0c6bf2a8c8c3b757fb54a0c7755daf7 Mon Sep 17 00:00:00 2001 From: Ben Banfield-Zanin Date: Thu, 19 Nov 2020 14:57:13 +0000 Subject: [PATCH 37/57] SAML: Allow specifying the IdP entityid to use. (#8630) If the SAML metadata includes multiple IdPs it is necessary to specify which IdP to redirect users to for authentication. --- changelog.d/8630.feature | 1 + docs/sample_config.yaml | 8 ++++++++ synapse/config/saml2_config.py | 10 ++++++++++ synapse/handlers/saml_handler.py | 3 ++- 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changelog.d/8630.feature diff --git a/changelog.d/8630.feature b/changelog.d/8630.feature new file mode 100644 index 0000000000..706051f131 --- /dev/null +++ b/changelog.d/8630.feature @@ -0,0 +1 @@ +Allow specification of the SAML IdP if the metadata returns multiple IdPs. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index bedc147770..52a1d8b853 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1674,6 +1674,14 @@ saml2_config: # - attribute: department # value: "sales" + # If the metadata XML contains multiple IdP entities then the `idp_entityid` + # option must be set to the entity to redirect users to. + # + # Most deployments only have a single IdP entity and so should omit this + # option. + # + #idp_entityid: 'https://our_idp/entityid' + # Enable OpenID Connect (OIDC) / OAuth 2.0 for registration and login. # diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py index f233854941..c1b8e98ae0 100644 --- a/synapse/config/saml2_config.py +++ b/synapse/config/saml2_config.py @@ -90,6 +90,8 @@ class SAML2Config(Config): "grandfathered_mxid_source_attribute", "uid" ) + self.saml2_idp_entityid = saml2_config.get("idp_entityid", None) + # user_mapping_provider may be None if the key is present but has no value ump_dict = saml2_config.get("user_mapping_provider") or {} @@ -383,6 +385,14 @@ class SAML2Config(Config): # value: "staff" # - attribute: department # value: "sales" + + # If the metadata XML contains multiple IdP entities then the `idp_entityid` + # option must be set to the entity to redirect users to. + # + # Most deployments only have a single IdP entity and so should omit this + # option. + # + #idp_entityid: 'https://our_idp/entityid' """ % { "config_dir_path": config_dir_path } diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index aee772239a..9bf430b656 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -58,6 +58,7 @@ class SamlHandler(BaseHandler): def __init__(self, hs: "synapse.server.HomeServer"): super().__init__(hs) self._saml_client = Saml2Client(hs.config.saml2_sp_config) + self._saml_idp_entityid = hs.config.saml2_idp_entityid self._auth_handler = hs.get_auth_handler() self._registration_handler = hs.get_registration_handler() @@ -100,7 +101,7 @@ class SamlHandler(BaseHandler): URL to redirect to """ reqid, info = self._saml_client.prepare_for_authenticate( - relay_state=client_redirect_url + entityid=self._saml_idp_entityid, relay_state=client_redirect_url ) # Since SAML sessions timeout it is useful to log when they were created. From 79bfe966e08a2212cc2fae2b00f5efb2c2185543 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 19 Nov 2020 14:25:17 -0500 Subject: [PATCH 38/57] Improve error checking for OIDC/SAML mapping providers (#8774) Checks that the localpart returned by mapping providers for SAML and OIDC are valid before registering new users. Extends the OIDC tests for existing users and invalid data. --- UPGRADE.rst | 30 +++++++++++ changelog.d/8774.misc | 1 + docs/sso_mapping_providers.md | 9 +++- synapse/handlers/oidc_handler.py | 25 +++++++-- synapse/handlers/saml_handler.py | 6 +++ synapse/types.py | 6 +-- tests/handlers/test_oidc.py | 89 +++++++++++++++++++++++++------- 7 files changed, 137 insertions(+), 29 deletions(-) create mode 100644 changelog.d/8774.misc diff --git a/UPGRADE.rst b/UPGRADE.rst index 7c19cf2a70..4de1bb5841 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -75,6 +75,36 @@ for example: wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb +Upgrading to v1.24.0 +==================== + +Custom OpenID Connect mapping provider breaking change +------------------------------------------------------ + +This release allows the OpenID Connect mapping provider to perform normalisation +of the localpart of the Matrix ID. This allows for the mapping provider to +specify different algorithms, instead of the [default way](https://matrix.org/docs/spec/appendices#mapping-from-other-character-sets). + +If your Synapse configuration uses a custom mapping provider +(`oidc_config.user_mapping_provider.module` is specified and not equal to +`synapse.handlers.oidc_handler.JinjaOidcMappingProvider`) then you *must* ensure +that `map_user_attributes` of the mapping provider performs some normalisation +of the `localpart` returned. To match previous behaviour you can use the +`map_username_to_mxid_localpart` function provided by Synapse. An example is +shown below: + +.. code-block:: python + + from synapse.types import map_username_to_mxid_localpart + + class MyMappingProvider: + def map_user_attributes(self, userinfo, token): + # ... your custom logic ... + sso_user_id = ... + localpart = map_username_to_mxid_localpart(sso_user_id) + + return {"localpart": localpart} + Upgrading to v1.23.0 ==================== diff --git a/changelog.d/8774.misc b/changelog.d/8774.misc new file mode 100644 index 0000000000..57cca8fee5 --- /dev/null +++ b/changelog.d/8774.misc @@ -0,0 +1 @@ +Add additional error checking for OpenID Connect and SAML mapping providers. diff --git a/docs/sso_mapping_providers.md b/docs/sso_mapping_providers.md index 32b06aa2c5..707dd73978 100644 --- a/docs/sso_mapping_providers.md +++ b/docs/sso_mapping_providers.md @@ -15,8 +15,15 @@ where SAML mapping providers come into play. SSO mapping providers are currently supported for OpenID and SAML SSO configurations. Please see the details below for how to implement your own. +It is the responsibility of the mapping provider to normalise the SSO attributes +and map them to a valid Matrix ID. The +[specification for Matrix IDs](https://matrix.org/docs/spec/appendices#user-identifiers) +has some information about what is considered valid. Alternately an easy way to +ensure it is valid is to use a Synapse utility function: +`synapse.types.map_username_to_mxid_localpart`. + External mapping providers are provided to Synapse in the form of an external -Python module. You can retrieve this module from [PyPi](https://pypi.org) or elsewhere, +Python module. You can retrieve this module from [PyPI](https://pypi.org) or elsewhere, but it must be importable via Synapse (e.g. it must be in the same virtualenv as Synapse). The Synapse config is then modified to point to the mapping provider (and optionally provide additional configuration for it). diff --git a/synapse/handlers/oidc_handler.py b/synapse/handlers/oidc_handler.py index be8562d47b..4bfd8d5617 100644 --- a/synapse/handlers/oidc_handler.py +++ b/synapse/handlers/oidc_handler.py @@ -38,7 +38,12 @@ from synapse.handlers._base import BaseHandler from synapse.handlers.sso import MappingException from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable -from synapse.types import JsonDict, UserID, map_username_to_mxid_localpart +from synapse.types import ( + JsonDict, + UserID, + contains_invalid_mxid_characters, + map_username_to_mxid_localpart, +) from synapse.util import json_decoder if TYPE_CHECKING: @@ -885,10 +890,12 @@ class OidcHandler(BaseHandler): "Retrieved user attributes from user mapping provider: %r", attributes ) - if not attributes["localpart"]: - raise MappingException("localpart is empty") - - localpart = map_username_to_mxid_localpart(attributes["localpart"]) + localpart = attributes["localpart"] + if not localpart: + raise MappingException( + "Error parsing OIDC response: OIDC mapping provider plugin " + "did not return a localpart value" + ) user_id = UserID(localpart, self.server_name).to_string() users = await self.store.get_users_by_id_case_insensitive(user_id) @@ -908,6 +915,11 @@ class OidcHandler(BaseHandler): # This mxid is taken raise MappingException("mxid '{}' is already taken".format(user_id)) else: + # Since the localpart is provided via a potentially untrusted module, + # ensure the MXID is valid before registering. + if contains_invalid_mxid_characters(localpart): + raise MappingException("localpart is invalid: %s" % (localpart,)) + # It's the first time this user is logging in and the mapped mxid was # not taken, register the user registered_user_id = await self._registration_handler.register_user( @@ -1076,6 +1088,9 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]): ) -> UserAttribute: localpart = self._config.localpart_template.render(user=userinfo).strip() + # Ensure only valid characters are included in the MXID. + localpart = map_username_to_mxid_localpart(localpart) + display_name = None # type: Optional[str] if self._config.display_name_template is not None: display_name = self._config.display_name_template.render( diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index 9bf430b656..5d9b555b13 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -31,6 +31,7 @@ from synapse.http.site import SynapseRequest from synapse.module_api import ModuleApi from synapse.types import ( UserID, + contains_invalid_mxid_characters, map_username_to_mxid_localpart, mxid_localpart_allowed_characters, ) @@ -318,6 +319,11 @@ class SamlHandler(BaseHandler): "Unable to generate a Matrix ID from the SAML response" ) + # Since the localpart is provided via a potentially untrusted module, + # ensure the MXID is valid before registering. + if contains_invalid_mxid_characters(localpart): + raise MappingException("localpart is invalid: %s" % (localpart,)) + logger.info("Mapped SAML user to local part %s", localpart) registered_user_id = await self._registration_handler.register_user( localpart=localpart, diff --git a/synapse/types.py b/synapse/types.py index 66bb5bac8d..3ab6bdbe06 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -317,14 +317,14 @@ mxid_localpart_allowed_characters = set( ) -def contains_invalid_mxid_characters(localpart): +def contains_invalid_mxid_characters(localpart: str) -> bool: """Check for characters not allowed in an mxid or groupid localpart Args: - localpart (basestring): the localpart to be checked + localpart: the localpart to be checked Returns: - bool: True if there are any naughty characters + True if there are any naughty characters """ return any(c not in mxid_localpart_allowed_characters for c in localpart) diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index 630e6da808..b4fa02acc4 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import json from urllib.parse import parse_qs, urlparse @@ -24,12 +23,8 @@ import pymacaroons from twisted.python.failure import Failure from twisted.web._newclient import ResponseDone -from synapse.handlers.oidc_handler import ( - MappingException, - OidcError, - OidcHandler, - OidcMappingProvider, -) +from synapse.handlers.oidc_handler import OidcError, OidcHandler, OidcMappingProvider +from synapse.handlers.sso import MappingException from synapse.types import UserID from tests.unittest import HomeserverTestCase, override_config @@ -132,14 +127,13 @@ class OidcHandlerTestCase(HomeserverTestCase): config = self.default_config() config["public_baseurl"] = BASE_URL - oidc_config = {} - oidc_config["enabled"] = True - oidc_config["client_id"] = CLIENT_ID - oidc_config["client_secret"] = CLIENT_SECRET - oidc_config["issuer"] = ISSUER - oidc_config["scopes"] = SCOPES - oidc_config["user_mapping_provider"] = { - "module": __name__ + ".TestMappingProvider", + oidc_config = { + "enabled": True, + "client_id": CLIENT_ID, + "client_secret": CLIENT_SECRET, + "issuer": ISSUER, + "scopes": SCOPES, + "user_mapping_provider": {"module": __name__ + ".TestMappingProvider"}, } # Update this config with what's in the default config so that @@ -705,13 +699,13 @@ class OidcHandlerTestCase(HomeserverTestCase): def test_map_userinfo_to_existing_user(self): """Existing users can log in with OpenID Connect when allow_existing_users is True.""" store = self.hs.get_datastore() - user4 = UserID.from_string("@test_user_4:test") + user = UserID.from_string("@test_user:test") self.get_success( - store.register_user(user_id=user4.to_string(), password_hash=None) + store.register_user(user_id=user.to_string(), password_hash=None) ) userinfo = { - "sub": "test4", - "username": "test_user_4", + "sub": "test", + "username": "test_user", } token = {} mxid = self.get_success( @@ -719,4 +713,59 @@ class OidcHandlerTestCase(HomeserverTestCase): userinfo, token, "user-agent", "10.10.10.10" ) ) - self.assertEqual(mxid, "@test_user_4:test") + self.assertEqual(mxid, "@test_user:test") + + # Register some non-exact matching cases. + user2 = UserID.from_string("@TEST_user_2:test") + self.get_success( + store.register_user(user_id=user2.to_string(), password_hash=None) + ) + user2_caps = UserID.from_string("@test_USER_2:test") + self.get_success( + store.register_user(user_id=user2_caps.to_string(), password_hash=None) + ) + + # Attempting to login without matching a name exactly is an error. + userinfo = { + "sub": "test2", + "username": "TEST_USER_2", + } + e = self.get_failure( + self.handler._map_userinfo_to_user( + userinfo, token, "user-agent", "10.10.10.10" + ), + MappingException, + ) + self.assertTrue( + str(e.value).startswith( + "Attempted to login as '@TEST_USER_2:test' but it matches more than one user inexactly:" + ) + ) + + # Logging in when matching a name exactly should work. + user2 = UserID.from_string("@TEST_USER_2:test") + self.get_success( + store.register_user(user_id=user2.to_string(), password_hash=None) + ) + + mxid = self.get_success( + self.handler._map_userinfo_to_user( + userinfo, token, "user-agent", "10.10.10.10" + ) + ) + self.assertEqual(mxid, "@TEST_USER_2:test") + + def test_map_userinfo_to_invalid_localpart(self): + """If the mapping provider generates an invalid localpart it should be rejected.""" + userinfo = { + "sub": "test2", + "username": "föö", + } + token = {} + e = self.get_failure( + self.handler._map_userinfo_to_user( + userinfo, token, "user-agent", "10.10.10.10" + ), + MappingException, + ) + self.assertEqual(str(e.value), "localpart is invalid: föö") From 1091bcea3e9b76af80ae6e5f9a7193213aab39fb Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 22 Nov 2020 00:29:05 +0000 Subject: [PATCH 39/57] fix ancient changelog to be MD --- CHANGES.md | 296 ++++++++++++++++++++++++++--------------------------- 1 file changed, 146 insertions(+), 150 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 52b2fd6f8f..847c734c75 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6322,8 +6322,8 @@ Changes in synapse 0.5.1 (2014-11-26) See UPGRADES.rst for specific instructions on how to upgrade. -> - Fix bug where we served up an Event that did not match its signatures. -> - Fix regression where we no longer correctly handled the case where a homeserver receives an event for a room it doesn\'t recognise (but is in.) +- Fix bug where we served up an Event that did not match its signatures. +- Fix regression where we no longer correctly handled the case where a homeserver receives an event for a room it doesn\'t recognise (but is in.) Changes in synapse 0.5.0 (2014-11-19) ===================================== @@ -6334,44 +6334,44 @@ This release also changes the internal database schemas and so requires servers Homeserver: -: - Add authentication and authorization to the federation protocol. Events are now signed by their originating homeservers. - - Implement the new authorization model for rooms. - - Split out web client into a seperate repository: matrix-angular-sdk. - - Change the structure of PDUs. - - Fix bug where user could not join rooms via an alias containing 4-byte UTF-8 characters. - - Merge concept of PDUs and Events internally. - - Improve logging by adding request ids to log lines. - - Implement a very basic room initial sync API. - - Implement the new invite/join federation APIs. +- Add authentication and authorization to the federation protocol. Events are now signed by their originating homeservers. +- Implement the new authorization model for rooms. +- Split out web client into a seperate repository: matrix-angular-sdk. +- Change the structure of PDUs. +- Fix bug where user could not join rooms via an alias containing 4-byte UTF-8 characters. +- Merge concept of PDUs and Events internally. +- Improve logging by adding request ids to log lines. +- Implement a very basic room initial sync API. +- Implement the new invite/join federation APIs. Webclient: -: - The webclient has been moved to a seperate repository. +- The webclient has been moved to a seperate repository. Changes in synapse 0.4.2 (2014-10-31) ===================================== Homeserver: -: - Fix bugs where we did not notify users of correct presence updates. - - Fix bug where we did not handle sub second event stream timeouts. +- Fix bugs where we did not notify users of correct presence updates. +- Fix bug where we did not handle sub second event stream timeouts. Webclient: -: - Add ability to click on messages to see JSON. - - Add ability to redact messages. - - Add ability to view and edit all room state JSON. - - Handle incoming redactions. - - Improve feedback on errors. - - Fix bugs in mobile CSS. - - Fix bugs with desktop notifications. +- Add ability to click on messages to see JSON. +- Add ability to redact messages. +- Add ability to view and edit all room state JSON. +- Handle incoming redactions. +- Improve feedback on errors. +- Fix bugs in mobile CSS. +- Fix bugs with desktop notifications. Changes in synapse 0.4.1 (2014-10-17) ===================================== Webclient: -: - Fix bug with display of timestamps. +- Fix bug with display of timestamps. Changes in synpase 0.4.0 (2014-10-17) ===================================== @@ -6384,8 +6384,8 @@ You will also need an updated syutil and config. See UPGRADES.rst. Homeserver: -: - Sign federation transactions to assert strong identity over federation. - - Rename timestamp keys in PDUs and events from \'ts\' and \'hsob\_ts\' to \'origin\_server\_ts\'. +- Sign federation transactions to assert strong identity over federation. +- Rename timestamp keys in PDUs and events from \'ts\' and \'hsob\_ts\' to \'origin\_server\_ts\'. Changes in synapse 0.3.4 (2014-09-25) ===================================== @@ -6394,48 +6394,48 @@ This version adds support for using a TURN server. See docs/turn-howto.rst on ho Homeserver: -: - Add support for redaction of messages. - - Fix bug where inviting a user on a remote home server could take up to 20-30s. - - Implement a get current room state API. - - Add support specifying and retrieving turn server configuration. +- Add support for redaction of messages. +- Fix bug where inviting a user on a remote home server could take up to 20-30s. +- Implement a get current room state API. +- Add support specifying and retrieving turn server configuration. Webclient: -: - Add button to send messages to users from the home page. - - Add support for using TURN for VoIP calls. - - Show display name change messages. - - Fix bug where the client didn\'t get the state of a newly joined room until after it has been refreshed. - - Fix bugs with tab complete. - - Fix bug where holding down the down arrow caused chrome to chew 100% CPU. - - Fix bug where desktop notifications occasionally used \"Undefined\" as the display name. - - Fix more places where we sometimes saw room IDs incorrectly. - - Fix bug which caused lag when entering text in the text box. +- Add button to send messages to users from the home page. +- Add support for using TURN for VoIP calls. +- Show display name change messages. +- Fix bug where the client didn\'t get the state of a newly joined room until after it has been refreshed. +- Fix bugs with tab complete. +- Fix bug where holding down the down arrow caused chrome to chew 100% CPU. +- Fix bug where desktop notifications occasionally used \"Undefined\" as the display name. +- Fix more places where we sometimes saw room IDs incorrectly. +- Fix bug which caused lag when entering text in the text box. Changes in synapse 0.3.3 (2014-09-22) ===================================== Homeserver: -: - Fix bug where you continued to get events for rooms you had left. +- Fix bug where you continued to get events for rooms you had left. Webclient: -: - Add support for video calls with basic UI. - - Fix bug where one to one chats were named after your display name rather than the other person\'s. - - Fix bug which caused lag when typing in the textarea. - - Refuse to run on browsers we know won\'t work. - - Trigger pagination when joining new rooms. - - Fix bug where we sometimes didn\'t display invitations in recents. - - Automatically join room when accepting a VoIP call. - - Disable outgoing and reject incoming calls on browsers we don\'t support VoIP in. - - Don\'t display desktop notifications for messages in the room you are non-idle and speaking in. +- Add support for video calls with basic UI. +- Fix bug where one to one chats were named after your display name rather than the other person\'s. +- Fix bug which caused lag when typing in the textarea. +- Refuse to run on browsers we know won\'t work. +- Trigger pagination when joining new rooms. +- Fix bug where we sometimes didn\'t display invitations in recents. +- Automatically join room when accepting a VoIP call. +- Disable outgoing and reject incoming calls on browsers we don\'t support VoIP in. +- Don\'t display desktop notifications for messages in the room you are non-idle and speaking in. Changes in synapse 0.3.2 (2014-09-18) ===================================== Webclient: -: - Fix bug where an empty \"bing words\" list in old accounts didn\'t send notifications when it should have done. +- Fix bug where an empty \"bing words\" list in old accounts didn\'t send notifications when it should have done. Changes in synapse 0.3.1 (2014-09-18) ===================================== @@ -6444,8 +6444,8 @@ This is a release to hotfix v0.3.0 to fix two regressions. Webclient: -: - Fix a regression where we sometimes displayed duplicate events. - - Fix a regression where we didn\'t immediately remove rooms you were banned in from the recents list. +- Fix a regression where we sometimes displayed duplicate events. +- Fix a regression where we didn\'t immediately remove rooms you were banned in from the recents list. Changes in synapse 0.3.0 (2014-09-18) ===================================== @@ -6454,91 +6454,91 @@ See UPGRADE for information about changes to the client server API, including br Homeserver: -: - When a user changes their displayname or avatar the server will now update all their join states to reflect this. - - The server now adds \"age\" key to events to indicate how old they are. This is clock independent, so at no point does any server or webclient have to assume their clock is in sync with everyone else. - - Fix bug where we didn\'t correctly pull in missing PDUs. - - Fix bug where prev\_content key wasn\'t always returned. - - Add support for password resets. +- When a user changes their displayname or avatar the server will now update all their join states to reflect this. +- The server now adds \"age\" key to events to indicate how old they are. This is clock independent, so at no point does any server or webclient have to assume their clock is in sync with everyone else. +- Fix bug where we didn\'t correctly pull in missing PDUs. +- Fix bug where prev\_content key wasn\'t always returned. +- Add support for password resets. Webclient: -: - Improve page content loading. - - Join/parts now trigger desktop notifications. - - Always show room aliases in the UI if one is present. - - No longer show user-count in the recents side panel. - - Add up & down arrow support to the text box for message sending to step through your sent history. - - Don\'t display notifications for our own messages. - - Emotes are now formatted correctly in desktop notifications. - - The recents list now differentiates between public & private rooms. - - Fix bug where when switching between rooms the pagination flickered before the view jumped to the bottom of the screen. - - Add bing word support. +- Improve page content loading. +- Join/parts now trigger desktop notifications. +- Always show room aliases in the UI if one is present. +- No longer show user-count in the recents side panel. +- Add up & down arrow support to the text box for message sending to step through your sent history. +- Don\'t display notifications for our own messages. +- Emotes are now formatted correctly in desktop notifications. +- The recents list now differentiates between public & private rooms. +- Fix bug where when switching between rooms the pagination flickered before the view jumped to the bottom of the screen. +- Add bing word support. Registration API: -: - The registration API has been overhauled to function like the login API. In practice, this means registration requests must now include the following: \'type\':\'m.login.password\'. See UPGRADE for more information on this. - - The \'user\_id\' key has been renamed to \'user\' to better match the login API. - - There is an additional login type: \'m.login.email.identity\'. - - The command client and web client have been updated to reflect these changes. +- The registration API has been overhauled to function like the login API. In practice, this means registration requests must now include the following: \'type\':\'m.login.password\'. See UPGRADE for more information on this. +- The \'user\_id\' key has been renamed to \'user\' to better match the login API. +- There is an additional login type: \'m.login.email.identity\'. +- The command client and web client have been updated to reflect these changes. Changes in synapse 0.2.3 (2014-09-12) ===================================== Homeserver: -: - Fix bug where we stopped sending events to remote home servers if a user from that home server left, even if there were some still in the room. - - Fix bugs in the state conflict resolution where it was incorrectly rejecting events. +- Fix bug where we stopped sending events to remote home servers if a user from that home server left, even if there were some still in the room. +- Fix bugs in the state conflict resolution where it was incorrectly rejecting events. Webclient: -: - Display room names and topics. - - Allow setting/editing of room names and topics. - - Display information about rooms on the main page. - - Handle ban and kick events in real time. - - VoIP UI and reliability improvements. - - Add glare support for VoIP. - - Improvements to initial startup speed. - - Don\'t display duplicate join events. - - Local echo of messages. - - Differentiate sending and sent of local echo. - - Various minor bug fixes. +- Display room names and topics. +- Allow setting/editing of room names and topics. +- Display information about rooms on the main page. +- Handle ban and kick events in real time. +- VoIP UI and reliability improvements. +- Add glare support for VoIP. +- Improvements to initial startup speed. +- Don\'t display duplicate join events. +- Local echo of messages. +- Differentiate sending and sent of local echo. +- Various minor bug fixes. Changes in synapse 0.2.2 (2014-09-06) ===================================== Homeserver: -: - When the server returns state events it now also includes the previous content. - - Add support for inviting people when creating a new room. - - Make the homeserver inform the room via m.room.aliases when a new alias is added for a room. - - Validate m.room.power\_level events. +- When the server returns state events it now also includes the previous content. +- Add support for inviting people when creating a new room. +- Make the homeserver inform the room via m.room.aliases when a new alias is added for a room. +- Validate m.room.power\_level events. Webclient: -: - Add support for captchas on registration. - - Handle m.room.aliases events. - - Asynchronously send messages and show a local echo. - - Inform the UI when a message failed to send. - - Only autoscroll on receiving a new message if the user was already at the bottom of the screen. - - Add support for ban/kick reasons. +- Add support for captchas on registration. +- Handle m.room.aliases events. +- Asynchronously send messages and show a local echo. +- Inform the UI when a message failed to send. +- Only autoscroll on receiving a new message if the user was already at the bottom of the screen. +- Add support for ban/kick reasons. Changes in synapse 0.2.1 (2014-09-03) ===================================== Homeserver: -: - Added support for signing up with a third party id. - - Add synctl scripts. - - Added rate limiting. - - Add option to change the external address the content repo uses. - - Presence bug fixes. +- Added support for signing up with a third party id. +- Add synctl scripts. +- Added rate limiting. +- Add option to change the external address the content repo uses. +- Presence bug fixes. Webclient: -: - Added support for signing up with a third party id. - - Added support for banning and kicking users. - - Added support for displaying and setting ops. - - Added support for room names. - - Fix bugs with room membership event display. +- Added support for signing up with a third party id. +- Added support for banning and kicking users. +- Added support for displaying and setting ops. +- Added support for room names. +- Fix bugs with room membership event display. Changes in synapse 0.2.0 (2014-09-02) ===================================== @@ -6547,36 +6547,36 @@ This update changes many configuration options, updates the database schema and Homeserver: -: - Require SSL for server-server connections. - - Add SSL listener for client-server connections. - - Add ability to use config files. - - Add support for kicking/banning and power levels. - - Allow setting of room names and topics on creation. - - Change presence to include last seen time of the user. - - Change url path prefix to /\_matrix/\... - - Bug fixes to presence. +- Require SSL for server-server connections. +- Add SSL listener for client-server connections. +- Add ability to use config files. +- Add support for kicking/banning and power levels. +- Allow setting of room names and topics on creation. +- Change presence to include last seen time of the user. +- Change url path prefix to /\_matrix/\... +- Bug fixes to presence. Webclient: -: - Reskin the CSS for registration and login. - - Various improvements to rooms CSS. - - Support changes in client-server API. - - Bug fixes to VOIP UI. - - Various bug fixes to handling of changes to room member list. +- Reskin the CSS for registration and login. +- Various improvements to rooms CSS. +- Support changes in client-server API. +- Bug fixes to VOIP UI. +- Various bug fixes to handling of changes to room member list. Changes in synapse 0.1.2 (2014-08-29) ===================================== Webclient: -: - Add basic call state UI for VoIP calls. +- Add basic call state UI for VoIP calls. Changes in synapse 0.1.1 (2014-08-29) ===================================== Homeserver: -: - Fix bug that caused the event stream to not notify some clients about changes. +- Fix bug that caused the event stream to not notify some clients about changes. Changes in synapse 0.1.0 (2014-08-29) ===================================== @@ -6585,26 +6585,22 @@ Presence has been reenabled in this release. Homeserver: -: - - - Update client to server API, including: - - : - Use a more consistent url scheme. - - Provide more useful information in the initial sync api. - - - Change the presence handling to be much more efficient. - - Change the presence server to server API to not require explicit polling of all users who share a room with a user. - - Fix races in the event streaming logic. +- Update client to server API, including: + - Use a more consistent url scheme. + - Provide more useful information in the initial sync api. +- Change the presence handling to be much more efficient. +- Change the presence server to server API to not require explicit polling of all users who share a room with a user. +- Fix races in the event streaming logic. Webclient: -: - Update to use new client to server API. - - Add basic VOIP support. - - Add idle timers that change your status to away. - - Add recent rooms column when viewing a room. - - Various network efficiency improvements. - - Add basic mobile browser support. - - Add a settings page. +- Update to use new client to server API. +- Add basic VOIP support. +- Add idle timers that change your status to away. +- Add recent rooms column when viewing a room. +- Various network efficiency improvements. +- Add basic mobile browser support. +- Add a settings page. Changes in synapse 0.0.1 (2014-08-22) ===================================== @@ -6613,26 +6609,26 @@ Presence has been disabled in this release due to a bug that caused the homeserv Homeserver: -: - Completely change the database schema to support generic event types. - - Improve presence reliability. - - Improve reliability of joining remote rooms. - - Fix bug where room join events were duplicated. - - Improve initial sync API to return more information to the client. - - Stop generating fake messages for room membership events. +- Completely change the database schema to support generic event types. +- Improve presence reliability. +- Improve reliability of joining remote rooms. +- Fix bug where room join events were duplicated. +- Improve initial sync API to return more information to the client. +- Stop generating fake messages for room membership events. Webclient: -: - Add tab completion of names. - - Add ability to upload and send images. - - Add profile pages. - - Improve CSS layout of room. - - Disambiguate identical display names. - - Don\'t get remote users display names and avatars individually. - - Use the new initial sync API to reduce number of round trips to the homeserver. - - Change url scheme to use room aliases instead of room ids where known. - - Increase longpoll timeout. +- Add tab completion of names. +- Add ability to upload and send images. +- Add profile pages. +- Improve CSS layout of room. +- Disambiguate identical display names. +- Don\'t get remote users display names and avatars individually. +- Use the new initial sync API to reduce number of round trips to the homeserver. +- Change url scheme to use room aliases instead of room ids where known. +- Increase longpoll timeout. Changes in synapse 0.0.0 (2014-08-13) ===================================== -> - Initial alpha release +- Initial alpha release From 476b8c0ae6f5a7cfb9b3d74bafa2886a3337d940 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 22 Nov 2020 00:30:13 +0000 Subject: [PATCH 40/57] fix MD --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 847c734c75..4237550818 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6586,8 +6586,8 @@ Presence has been reenabled in this release. Homeserver: - Update client to server API, including: - - Use a more consistent url scheme. - - Provide more useful information in the initial sync api. + - Use a more consistent url scheme. + - Provide more useful information in the initial sync api. - Change the presence handling to be much more efficient. - Change the presence server to server API to not require explicit polling of all users who share a room with a user. - Fix races in the event streaming logic. From 8ca120df7c2964c73390dbfab9ceba03f76eadfe Mon Sep 17 00:00:00 2001 From: Daniele Sluijters Date: Mon, 23 Nov 2020 14:01:18 +0100 Subject: [PATCH 41/57] INSTALL: Fix setting content-type on well-known (#8793) When using `add_header` nginx will literally add a header. If a `content-type` header is already configured (for example through a server wide default), this means we end up with 2 content-type headers, like so: ``` content-type: text/html content-type: application/json access-control-allow-origin: * ``` That doesn't make sense. Instead, we want the content type of that block to only be `application/json` which we can achieve using `default_type` instead. Signed-off-by: Daniele Sluijters --- INSTALL.md | 2 +- changelog.d/8793.doc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/8793.doc diff --git a/INSTALL.md b/INSTALL.md index c6fcb3bd7f..eaeb690092 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -487,7 +487,7 @@ In nginx this would be something like: ``` location /.well-known/matrix/client { return 200 '{"m.homeserver": {"base_url": "https://"}}'; - add_header Content-Type application/json; + default_type application/json; add_header Access-Control-Allow-Origin *; } ``` diff --git a/changelog.d/8793.doc b/changelog.d/8793.doc new file mode 100644 index 0000000000..f6eee1ea73 --- /dev/null +++ b/changelog.d/8793.doc @@ -0,0 +1 @@ +Fix the example on how to set the `Content-Type` header in nginx for the Client Well-Known URI. From 59a995f38dab8d83aee09ed7eb3858390ed53a65 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 23 Nov 2020 13:45:23 +0000 Subject: [PATCH 42/57] Improve logging of the mapping from SSO IDs to Matrix IDs. (#8773) --- changelog.d/8773.misc | 1 + synapse/handlers/saml_handler.py | 5 +++-- synapse/handlers/sso.py | 12 +++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 changelog.d/8773.misc diff --git a/changelog.d/8773.misc b/changelog.d/8773.misc new file mode 100644 index 0000000000..62778ba410 --- /dev/null +++ b/changelog.d/8773.misc @@ -0,0 +1 @@ +Minor log line improvements for the SSO mapping code used to generate Matrix IDs from SSO IDs. diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index 5d9b555b13..f4e8cbeac8 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -268,7 +268,8 @@ class SamlHandler(BaseHandler): user_id = UserID( map_username_to_mxid_localpart(attrval), self.server_name ).to_string() - logger.info( + + logger.debug( "Looking for existing account based on mapped %s %s", self._grandfathered_mxid_source_attribute, user_id, @@ -324,7 +325,7 @@ class SamlHandler(BaseHandler): if contains_invalid_mxid_characters(localpart): raise MappingException("localpart is invalid: %s" % (localpart,)) - logger.info("Mapped SAML user to local part %s", localpart) + logger.debug("Mapped SAML user to local part %s", localpart) registered_user_id = await self._registration_handler.register_user( localpart=localpart, default_display_name=displayname, diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 9cb1866a71..cf7cb7754a 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -71,19 +71,25 @@ class SsoHandler(BaseHandler): Returns: The mxid of a previously seen user. """ - # Check if we already have a mapping for this user. - logger.info( + logger.debug( "Looking for existing mapping for user %s:%s", auth_provider_id, remote_user_id, ) + + # Check if we already have a mapping for this user. previously_registered_user_id = await self.store.get_user_by_external_id( auth_provider_id, remote_user_id, ) # A match was found, return the user ID. if previously_registered_user_id is not None: - logger.info("Found existing mapping %s", previously_registered_user_id) + logger.info( + "Found existing mapping for IdP '%s' and remote_user_id '%s': %s", + auth_provider_id, + remote_user_id, + previously_registered_user_id, + ) return previously_registered_user_id # No match. From 7127855741faaf9544339db4faa7d25b0158fdfc Mon Sep 17 00:00:00 2001 From: Waylon Cude Date: Mon, 23 Nov 2020 07:20:49 -0800 Subject: [PATCH 43/57] Fix synctl and duplicate worker spawning (#8798) Synctl did not check if a worker thread was already running when using `synctl start` and would naively start a fresh copy. This would sometimes lead to cases where many duplicate copies of a single worker would run. This fix adds a pid check when starting worker threads and synctl will now refuse to start individual workers if they're already running. --- changelog.d/8798.bugfix | 1 + synctl | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelog.d/8798.bugfix diff --git a/changelog.d/8798.bugfix b/changelog.d/8798.bugfix new file mode 100644 index 0000000000..9bdb2b51ea --- /dev/null +++ b/changelog.d/8798.bugfix @@ -0,0 +1 @@ +Fix a bug where synctl could spawn duplicate copies of a worker. Contributed by Waylon Cude. diff --git a/synctl b/synctl index 9395ebd048..cfa9cec0c4 100755 --- a/synctl +++ b/synctl @@ -358,6 +358,13 @@ def main(): for worker in workers: env = os.environ.copy() + # Skip starting a worker if its already running + if os.path.exists(worker.pidfile) and pid_running( + int(open(worker.pidfile).read()) + ): + print(worker.app + " already running") + continue + if worker.cache_factor: os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor) From 6fde6aa9c02d35e0a908437ea49b275df9b58427 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 23 Nov 2020 13:28:03 -0500 Subject: [PATCH 44/57] Properly report user-agent/IP during registration of SSO users. (#8784) This also expands type-hints to the SSO and registration code. Refactors the CAS code to more closely match OIDC/SAML. --- changelog.d/8784.misc | 1 + mypy.ini | 1 + synapse/handlers/cas_handler.py | 71 +++++++--- synapse/handlers/oidc_handler.py | 2 +- synapse/handlers/register.py | 214 +++++++++++++++++-------------- synapse/handlers/saml_handler.py | 6 +- 6 files changed, 173 insertions(+), 122 deletions(-) create mode 100644 changelog.d/8784.misc diff --git a/changelog.d/8784.misc b/changelog.d/8784.misc new file mode 100644 index 0000000000..18a4263398 --- /dev/null +++ b/changelog.d/8784.misc @@ -0,0 +1 @@ +Fix a bug introduced in v1.20.0 where the user-agent and IP address reported during user registration for CAS, OpenID Connect, and SAML were of the wrong form. diff --git a/mypy.ini b/mypy.ini index fc9f8d8050..0cf7c93f45 100644 --- a/mypy.ini +++ b/mypy.ini @@ -37,6 +37,7 @@ files = synapse/handlers/presence.py, synapse/handlers/profile.py, synapse/handlers/read_marker.py, + synapse/handlers/register.py, synapse/handlers/room.py, synapse/handlers/room_member.py, synapse/handlers/room_member_worker.py, diff --git a/synapse/handlers/cas_handler.py b/synapse/handlers/cas_handler.py index 048a3b3c0b..f4ea0a9767 100644 --- a/synapse/handlers/cas_handler.py +++ b/synapse/handlers/cas_handler.py @@ -14,7 +14,7 @@ # limitations under the License. import logging import urllib -from typing import Dict, Optional, Tuple +from typing import TYPE_CHECKING, Dict, Optional, Tuple from xml.etree import ElementTree as ET from twisted.web.client import PartialDownloadError @@ -23,6 +23,9 @@ from synapse.api.errors import Codes, LoginError from synapse.http.site import SynapseRequest from synapse.types import UserID, map_username_to_mxid_localpart +if TYPE_CHECKING: + from synapse.app.homeserver import HomeServer + logger = logging.getLogger(__name__) @@ -31,10 +34,10 @@ class CasHandler: Utility class for to handle the response from a CAS SSO service. Args: - hs (synapse.server.HomeServer) + hs """ - def __init__(self, hs): + def __init__(self, hs: "HomeServer"): self.hs = hs self._hostname = hs.hostname self._auth_handler = hs.get_auth_handler() @@ -200,27 +203,57 @@ class CasHandler: args["session"] = session username, user_display_name = await self._validate_ticket(ticket, args) - localpart = map_username_to_mxid_localpart(username) - user_id = UserID(localpart, self._hostname).to_string() - registered_user_id = await self._auth_handler.check_user_exists(user_id) + # Pull out the user-agent and IP from the request. + user_agent = request.get_user_agent("") + ip_address = self.hs.get_ip_from_request(request) + + # Get the matrix ID from the CAS username. + user_id = await self._map_cas_user_to_matrix_user( + username, user_display_name, user_agent, ip_address + ) if session: await self._auth_handler.complete_sso_ui_auth( - registered_user_id, session, request, + user_id, session, request, ) - else: - if not registered_user_id: - # Pull out the user-agent and IP from the request. - user_agent = request.get_user_agent("") - ip_address = self.hs.get_ip_from_request(request) - - registered_user_id = await self._registration_handler.register_user( - localpart=localpart, - default_display_name=user_display_name, - user_agent_ips=(user_agent, ip_address), - ) + # If this not a UI auth request than there must be a redirect URL. + assert client_redirect_url await self._auth_handler.complete_sso_login( - registered_user_id, request, client_redirect_url + user_id, request, client_redirect_url ) + + async def _map_cas_user_to_matrix_user( + self, + remote_user_id: str, + display_name: Optional[str], + user_agent: str, + ip_address: str, + ) -> str: + """ + Given a CAS username, retrieve the user ID for it and possibly register the user. + + Args: + remote_user_id: The username from the CAS response. + display_name: The display name from the CAS response. + user_agent: The user agent of the client making the request. + ip_address: The IP address of the client making the request. + + Returns: + The user ID associated with this response. + """ + + localpart = map_username_to_mxid_localpart(remote_user_id) + user_id = UserID(localpart, self._hostname).to_string() + registered_user_id = await self._auth_handler.check_user_exists(user_id) + + # If the user does not exist, register it. + if not registered_user_id: + registered_user_id = await self._registration_handler.register_user( + localpart=localpart, + default_display_name=display_name, + user_agent_ips=[(user_agent, ip_address)], + ) + + return registered_user_id diff --git a/synapse/handlers/oidc_handler.py b/synapse/handlers/oidc_handler.py index 4bfd8d5617..34de9109ea 100644 --- a/synapse/handlers/oidc_handler.py +++ b/synapse/handlers/oidc_handler.py @@ -925,7 +925,7 @@ class OidcHandler(BaseHandler): registered_user_id = await self._registration_handler.register_user( localpart=localpart, default_display_name=attributes["display_name"], - user_agent_ips=(user_agent, ip_address), + user_agent_ips=[(user_agent, ip_address)], ) await self.store.record_user_external_id( diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 252f700786..0d85fd0868 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -15,10 +15,12 @@ """Contains functions for registering clients.""" import logging +from typing import TYPE_CHECKING, List, Optional, Tuple from synapse import types from synapse.api.constants import MAX_USERID_LENGTH, EventTypes, JoinRules, LoginType from synapse.api.errors import AuthError, Codes, ConsentNotGivenError, SynapseError +from synapse.appservice import ApplicationService from synapse.config.server import is_threepid_reserved from synapse.http.servlet import assert_params_in_dict from synapse.replication.http.login import RegisterDeviceReplicationServlet @@ -32,16 +34,14 @@ from synapse.types import RoomAlias, UserID, create_requester from ._base import BaseHandler +if TYPE_CHECKING: + from synapse.app.homeserver import HomeServer + logger = logging.getLogger(__name__) class RegistrationHandler(BaseHandler): - def __init__(self, hs): - """ - - Args: - hs (synapse.server.HomeServer): - """ + def __init__(self, hs: "HomeServer"): super().__init__(hs) self.hs = hs self.auth = hs.get_auth() @@ -71,7 +71,10 @@ class RegistrationHandler(BaseHandler): self.session_lifetime = hs.config.session_lifetime async def check_username( - self, localpart, guest_access_token=None, assigned_user_id=None + self, + localpart: str, + guest_access_token: Optional[str] = None, + assigned_user_id: Optional[str] = None, ): if types.contains_invalid_mxid_characters(localpart): raise SynapseError( @@ -140,39 +143,45 @@ class RegistrationHandler(BaseHandler): async def register_user( self, - localpart=None, - password_hash=None, - guest_access_token=None, - make_guest=False, - admin=False, - threepid=None, - user_type=None, - default_display_name=None, - address=None, - bind_emails=[], - by_admin=False, - user_agent_ips=None, - ): + localpart: Optional[str] = None, + password_hash: Optional[str] = None, + guest_access_token: Optional[str] = None, + make_guest: bool = False, + admin: bool = False, + threepid: Optional[dict] = None, + user_type: Optional[str] = None, + default_display_name: Optional[str] = None, + address: Optional[str] = None, + bind_emails: List[str] = [], + by_admin: bool = False, + user_agent_ips: Optional[List[Tuple[str, str]]] = None, + ) -> str: """Registers a new client on the server. Args: localpart: The local part of the user ID to register. If None, one will be generated. - password_hash (str|None): The hashed password to assign to this user so they can + password_hash: The hashed password to assign to this user so they can login again. This can be None which means they cannot login again via a password (e.g. the user is an application service user). - user_type (str|None): type of user. One of the values from + guest_access_token: The access token used when this was a guest + account. + make_guest: True if the the new user should be guest, + false to add a regular user account. + admin: True if the user should be registered as a server admin. + threepid: The threepid used for registering, if any. + user_type: type of user. One of the values from api.constants.UserTypes, or None for a normal user. - default_display_name (unicode|None): if set, the new user's displayname + default_display_name: if set, the new user's displayname will be set to this. Defaults to 'localpart'. - address (str|None): the IP address used to perform the registration. - bind_emails (List[str]): list of emails to bind to this account. - by_admin (bool): True if this registration is being made via the + address: the IP address used to perform the registration. + bind_emails: list of emails to bind to this account. + by_admin: True if this registration is being made via the admin api, otherwise False. - user_agent_ips (List[(str, str)]): Tuples of IP addresses and user-agents used + user_agent_ips: Tuples of IP addresses and user-agents used during the registration process. Returns: - str: user_id + The registere user_id. Raises: SynapseError if there was a problem registering. """ @@ -236,8 +245,10 @@ class RegistrationHandler(BaseHandler): else: # autogen a sequential user ID fail_count = 0 - user = None - while not user: + # If a default display name is not given, generate one. + generate_display_name = default_display_name is None + # This breaks on successful registration *or* errors after 10 failures. + while True: # Fail after being unable to find a suitable ID a few times if fail_count > 10: raise SynapseError(500, "Unable to find a suitable guest user ID") @@ -246,7 +257,7 @@ class RegistrationHandler(BaseHandler): user = UserID(localpart, self.hs.hostname) user_id = user.to_string() self.check_user_id_not_appservice_exclusive(user_id) - if default_display_name is None: + if generate_display_name: default_display_name = localpart try: await self.register_with_store( @@ -262,8 +273,6 @@ class RegistrationHandler(BaseHandler): break except SynapseError: # if user id is taken, just generate another - user = None - user_id = None fail_count += 1 if not self.hs.config.user_consent_at_registration: @@ -295,7 +304,7 @@ class RegistrationHandler(BaseHandler): return user_id - async def _create_and_join_rooms(self, user_id: str): + async def _create_and_join_rooms(self, user_id: str) -> None: """ Create the auto-join rooms and join or invite the user to them. @@ -379,7 +388,7 @@ class RegistrationHandler(BaseHandler): except Exception as e: logger.error("Failed to join new user to %r: %r", r, e) - async def _join_rooms(self, user_id: str): + async def _join_rooms(self, user_id: str) -> None: """ Join or invite the user to the auto-join rooms. @@ -425,6 +434,9 @@ class RegistrationHandler(BaseHandler): # Send the invite, if necessary. if requires_invite: + # If an invite is required, there must be a auto-join user ID. + assert self.hs.config.registration.auto_join_user_id + await room_member_handler.update_membership( requester=create_requester( self.hs.config.registration.auto_join_user_id, @@ -456,7 +468,7 @@ class RegistrationHandler(BaseHandler): except Exception as e: logger.error("Failed to join new user to %r: %r", r, e) - async def _auto_join_rooms(self, user_id: str): + async def _auto_join_rooms(self, user_id: str) -> None: """Automatically joins users to auto join rooms - creating the room in the first place if the user is the first to be created. @@ -479,16 +491,16 @@ class RegistrationHandler(BaseHandler): else: await self._join_rooms(user_id) - async def post_consent_actions(self, user_id): + async def post_consent_actions(self, user_id: str) -> None: """A series of registration actions that can only be carried out once consent has been granted Args: - user_id (str): The user to join + user_id: The user to join """ await self._auto_join_rooms(user_id) - async def appservice_register(self, user_localpart, as_token): + async def appservice_register(self, user_localpart: str, as_token: str) -> str: user = UserID(user_localpart, self.hs.hostname) user_id = user.to_string() service = self.store.get_app_service_by_token(as_token) @@ -513,7 +525,9 @@ class RegistrationHandler(BaseHandler): ) return user_id - def check_user_id_not_appservice_exclusive(self, user_id, allowed_appservice=None): + def check_user_id_not_appservice_exclusive( + self, user_id: str, allowed_appservice: Optional[ApplicationService] = None + ) -> None: # don't allow people to register the server notices mxid if self._server_notices_mxid is not None: if user_id == self._server_notices_mxid: @@ -537,12 +551,12 @@ class RegistrationHandler(BaseHandler): errcode=Codes.EXCLUSIVE, ) - def check_registration_ratelimit(self, address): + def check_registration_ratelimit(self, address: Optional[str]) -> None: """A simple helper method to check whether the registration rate limit has been hit for a given IP address Args: - address (str|None): the IP address used to perform the registration. If this is + address: the IP address used to perform the registration. If this is None, no ratelimiting will be performed. Raises: @@ -553,42 +567,39 @@ class RegistrationHandler(BaseHandler): self.ratelimiter.ratelimit(address) - def register_with_store( + async def register_with_store( self, - user_id, - password_hash=None, - was_guest=False, - make_guest=False, - appservice_id=None, - create_profile_with_displayname=None, - admin=False, - user_type=None, - address=None, - shadow_banned=False, - ): + user_id: str, + password_hash: Optional[str] = None, + was_guest: bool = False, + make_guest: bool = False, + appservice_id: Optional[str] = None, + create_profile_with_displayname: Optional[str] = None, + admin: bool = False, + user_type: Optional[str] = None, + address: Optional[str] = None, + shadow_banned: bool = False, + ) -> None: """Register user in the datastore. Args: - user_id (str): The desired user ID to register. - password_hash (str|None): Optional. The password hash for this user. - was_guest (bool): Optional. Whether this is a guest account being + user_id: The desired user ID to register. + password_hash: Optional. The password hash for this user. + was_guest: Optional. Whether this is a guest account being upgraded to a non-guest account. - make_guest (boolean): True if the the new user should be guest, + make_guest: True if the the new user should be guest, false to add a regular user account. - appservice_id (str|None): The ID of the appservice registering the user. - create_profile_with_displayname (unicode|None): Optionally create a + appservice_id: The ID of the appservice registering the user. + create_profile_with_displayname: Optionally create a profile for the user, setting their displayname to the given value - admin (boolean): is an admin user? - user_type (str|None): type of user. One of the values from + admin: is an admin user? + user_type: type of user. One of the values from api.constants.UserTypes, or None for a normal user. - address (str|None): the IP address used to perform the registration. - shadow_banned (bool): Whether to shadow-ban the user - - Returns: - Awaitable + address: the IP address used to perform the registration. + shadow_banned: Whether to shadow-ban the user """ if self.hs.config.worker_app: - return self._register_client( + await self._register_client( user_id=user_id, password_hash=password_hash, was_guest=was_guest, @@ -601,7 +612,7 @@ class RegistrationHandler(BaseHandler): shadow_banned=shadow_banned, ) else: - return self.store.register_user( + await self.store.register_user( user_id=user_id, password_hash=password_hash, was_guest=was_guest, @@ -614,22 +625,24 @@ class RegistrationHandler(BaseHandler): ) async def register_device( - self, user_id, device_id, initial_display_name, is_guest=False - ): + self, + user_id: str, + device_id: Optional[str], + initial_display_name: Optional[str], + is_guest: bool = False, + ) -> Tuple[str, str]: """Register a device for a user and generate an access token. The access token will be limited by the homeserver's session_lifetime config. Args: - user_id (str): full canonical @user:id - device_id (str|None): The device ID to check, or None to generate - a new one. - initial_display_name (str|None): An optional display name for the - device. - is_guest (bool): Whether this is a guest account + user_id: full canonical @user:id + device_id: The device ID to check, or None to generate a new one. + initial_display_name: An optional display name for the device. + is_guest: Whether this is a guest account Returns: - tuple[str, str]: Tuple of device ID and access token + Tuple of device ID and access token """ if self.hs.config.worker_app: @@ -649,7 +662,7 @@ class RegistrationHandler(BaseHandler): ) valid_until_ms = self.clock.time_msec() + self.session_lifetime - device_id = await self.device_handler.check_device_registered( + registered_device_id = await self.device_handler.check_device_registered( user_id, device_id, initial_display_name ) if is_guest: @@ -659,20 +672,21 @@ class RegistrationHandler(BaseHandler): ) else: access_token = await self._auth_handler.get_access_token_for_user_id( - user_id, device_id=device_id, valid_until_ms=valid_until_ms + user_id, device_id=registered_device_id, valid_until_ms=valid_until_ms ) - return (device_id, access_token) + return (registered_device_id, access_token) - async def post_registration_actions(self, user_id, auth_result, access_token): + async def post_registration_actions( + self, user_id: str, auth_result: dict, access_token: Optional[str] + ) -> None: """A user has completed registration Args: - user_id (str): The user ID that consented - auth_result (dict): The authenticated credentials of the newly - registered user. - access_token (str|None): The access token of the newly logged in - device, or None if `inhibit_login` enabled. + user_id: The user ID that consented + auth_result: The authenticated credentials of the newly registered user. + access_token: The access token of the newly logged in device, or + None if `inhibit_login` enabled. """ if self.hs.config.worker_app: await self._post_registration_client( @@ -698,19 +712,20 @@ class RegistrationHandler(BaseHandler): if auth_result and LoginType.TERMS in auth_result: await self._on_user_consented(user_id, self.hs.config.user_consent_version) - async def _on_user_consented(self, user_id, consent_version): + async def _on_user_consented(self, user_id: str, consent_version: str) -> None: """A user consented to the terms on registration Args: - user_id (str): The user ID that consented. - consent_version (str): version of the policy the user has - consented to. + user_id: The user ID that consented. + consent_version: version of the policy the user has consented to. """ logger.info("%s has consented to the privacy policy", user_id) await self.store.user_set_consent_version(user_id, consent_version) await self.post_consent_actions(user_id) - async def _register_email_threepid(self, user_id, threepid, token): + async def _register_email_threepid( + self, user_id: str, threepid: dict, token: Optional[str] + ) -> None: """Add an email address as a 3pid identifier Also adds an email pusher for the email address, if configured in the @@ -719,10 +734,9 @@ class RegistrationHandler(BaseHandler): Must be called on master. Args: - user_id (str): id of user - threepid (object): m.login.email.identity auth response - token (str|None): access_token for the user, or None if not logged - in. + user_id: id of user + threepid: m.login.email.identity auth response + token: access_token for the user, or None if not logged in. """ reqd = ("medium", "address", "validated_at") if any(x not in threepid for x in reqd): @@ -748,6 +762,8 @@ class RegistrationHandler(BaseHandler): # up when the access token is saved, but that's quite an # invasive change I'd rather do separately. user_tuple = await self.store.get_user_by_access_token(token) + # The token better still exist. + assert user_tuple token_id = user_tuple.token_id await self.pusher_pool.add_pusher( @@ -762,14 +778,14 @@ class RegistrationHandler(BaseHandler): data={}, ) - async def _register_msisdn_threepid(self, user_id, threepid): + async def _register_msisdn_threepid(self, user_id: str, threepid: dict) -> None: """Add a phone number as a 3pid identifier Must be called on master. Args: - user_id (str): id of user - threepid (object): m.login.msisdn auth response + user_id: id of user + threepid: m.login.msisdn auth response """ try: assert_params_in_dict(threepid, ["medium", "address", "validated_at"]) diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index f4e8cbeac8..37ab42f050 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -39,7 +39,7 @@ from synapse.util.async_helpers import Linearizer from synapse.util.iterutils import chunk_seq if TYPE_CHECKING: - import synapse.server + from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -56,7 +56,7 @@ class Saml2SessionData: class SamlHandler(BaseHandler): - def __init__(self, hs: "synapse.server.HomeServer"): + def __init__(self, hs: "HomeServer"): super().__init__(hs) self._saml_client = Saml2Client(hs.config.saml2_sp_config) self._saml_idp_entityid = hs.config.saml2_idp_entityid @@ -330,7 +330,7 @@ class SamlHandler(BaseHandler): localpart=localpart, default_display_name=displayname, bind_emails=emails, - user_agent_ips=(user_agent, ip_address), + user_agent_ips=[(user_agent, ip_address)], ) await self.store.record_user_external_id( From e3d780670405ec5e58f3eb34d74d312f16b23629 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 24 Nov 2020 12:52:22 +0000 Subject: [PATCH 45/57] Update turn-howto (#8779) Some hopefully-useful notes on setting up a turnserver. --- changelog.d/8779.doc | 1 + docs/turn-howto.md | 131 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 changelog.d/8779.doc diff --git a/changelog.d/8779.doc b/changelog.d/8779.doc new file mode 100644 index 0000000000..3641ae7f91 --- /dev/null +++ b/changelog.d/8779.doc @@ -0,0 +1 @@ +Update `turn-howto.md` with troubleshooting notes. diff --git a/docs/turn-howto.md b/docs/turn-howto.md index d4a726be66..a470c274a5 100644 --- a/docs/turn-howto.md +++ b/docs/turn-howto.md @@ -42,10 +42,10 @@ This will install and start a systemd service called `coturn`. ./configure - > You may need to install `libevent2`: if so, you should do so in - > the way recommended by your operating system. You can ignore - > warnings about lack of database support: a database is unnecessary - > for this purpose. + You may need to install `libevent2`: if so, you should do so in + the way recommended by your operating system. You can ignore + warnings about lack of database support: a database is unnecessary + for this purpose. 1. Build and install it: @@ -66,6 +66,19 @@ This will install and start a systemd service called `coturn`. pwgen -s 64 1 + A `realm` must be specified, but its value is somewhat arbitrary. (It is + sent to clients as part of the authentication flow.) It is conventional to + set it to be your server name. + +1. You will most likely want to configure coturn to write logs somewhere. The + easiest way is normally to send them to the syslog: + + syslog + + (in which case, the logs will be available via `journalctl -u coturn` on a + systemd system). Alternatively, coturn can be configured to write to a + logfile - check the example config file supplied with coturn. + 1. Consider your security settings. TURN lets users request a relay which will connect to arbitrary IP addresses and ports. The following configuration is suggested as a minimum starting point: @@ -96,11 +109,31 @@ This will install and start a systemd service called `coturn`. # TLS private key file pkey=/path/to/privkey.pem + In this case, replace the `turn:` schemes in the `turn_uri` settings below + with `turns:`. + + We recommend that you only try to set up TLS/DTLS once you have set up a + basic installation and got it working. + 1. Ensure your firewall allows traffic into the TURN server on the ports - you've configured it to listen on (By default: 3478 and 5349 for the TURN(s) + you've configured it to listen on (By default: 3478 and 5349 for TURN traffic (remember to allow both TCP and UDP traffic), and ports 49152-65535 for the UDP relay.) +1. We do not recommend running a TURN server behind NAT, and are not aware of + anyone doing so successfully. + + If you want to try it anyway, you will at least need to tell coturn its + external IP address: + + external-ip=192.88.99.1 + + ... and your NAT gateway must forward all of the relayed ports directly + (eg, port 56789 on the external IP must be always be forwarded to port + 56789 on the internal IP). + + If you get this working, let us know! + 1. (Re)start the turn server: * If you used the Debian package (or have set up a systemd unit yourself): @@ -137,9 +170,10 @@ Your home server configuration file needs the following extra keys: without having gone through a CAPTCHA or similar to register a real account. -As an example, here is the relevant section of the config file for matrix.org: +As an example, here is the relevant section of the config file for `matrix.org`. The +`turn_uris` are appropriate for TURN servers listening on the default ports, with no TLS. - turn_uris: [ "turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp" ] + turn_uris: [ "turn:turn.matrix.org?transport=udp", "turn:turn.matrix.org?transport=tcp" ] turn_shared_secret: "n0t4ctuAllymatr1Xd0TorgSshar3d5ecret4obvIousreAsons" turn_user_lifetime: 86400000 turn_allow_guests: True @@ -155,5 +189,86 @@ After updating the homeserver configuration, you must restart synapse: ``` systemctl restart synapse.service ``` +... and then reload any clients (or wait an hour for them to refresh their +settings). -..and your Home Server now supports VoIP relaying! +## Troubleshooting + +The normal symptoms of a misconfigured TURN server are that calls between +devices on different networks ring, but get stuck at "call +connecting". Unfortunately, troubleshooting this can be tricky. + +Here are a few things to try: + + * Check that your TURN server is not behind NAT. As above, we're not aware of + anyone who has successfully set this up. + + * Check that you have opened your firewall to allow TCP and UDP traffic to the + TURN ports (normally 3478 and 5479). + + * Check that you have opened your firewall to allow UDP traffic to the UDP + relay ports (49152-65535 by default). + + * Some WebRTC implementations (notably, that of Google Chrome) appear to get + confused by TURN servers which are reachable over IPv6 (this appears to be + an unexpected side-effect of its handling of multiple IP addresses as + defined by + [`draft-ietf-rtcweb-ip-handling`](https://tools.ietf.org/html/draft-ietf-rtcweb-ip-handling-12)). + + Try removing any AAAA records for your TURN server, so that it is only + reachable over IPv4. + + * Enable more verbose logging in coturn via the `verbose` setting: + + ``` + verbose + ``` + + ... and then see if there are any clues in its logs. + + * If you are using a browser-based client under Chrome, check + `chrome://webrtc-internals/` for insights into the internals of the + negotiation. On Firefox, check the "Connection Log" on `about:webrtc`. + + (Understanding the output is beyond the scope of this document!) + + * There is a WebRTC test tool at + https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/. To + use it, you will need a username/password for your TURN server. You can + either: + + * look for the `GET /_matrix/client/r0/voip/turnServer` request made by a + matrix client to your homeserver in your browser's network inspector. In + the response you should see `username` and `password`. Or: + + * Use the following shell commands: + + ```sh + secret=staticAuthSecretHere + + u=$((`date +%s` + 3600)):test + p=$(echo -n $u | openssl dgst -hmac $secret -sha1 -binary | base64) + echo -e "username: $u\npassword: $p" + ``` + + Or: + + * Temporarily configure coturn to accept a static username/password. To do + this, comment out `use-auth-secret` and `static-auth-secret` and add the + following: + + ``` + lt-cred-mech + user=username:password + ``` + + **Note**: these settings will not take effect unless `use-auth-secret` + and `static-auth-secret` are disabled. + + Restart coturn after changing the configuration file. + + Remember to restore the original settings to go back to testing with + Matrix clients! + + If the TURN server is working correctly, you should see at least one `relay` + entry in the results. From 97b35ee259b062412ed2da8272dd200eb29ee93d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 24 Nov 2020 12:53:00 +0000 Subject: [PATCH 46/57] Add a script to sign arbitrary json objects. (#8772) --- changelog.d/8772.misc | 1 + mypy.ini | 1 + scripts-dev/sign_json | 127 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 changelog.d/8772.misc create mode 100755 scripts-dev/sign_json diff --git a/changelog.d/8772.misc b/changelog.d/8772.misc new file mode 100644 index 0000000000..d74d0a3d5d --- /dev/null +++ b/changelog.d/8772.misc @@ -0,0 +1 @@ +Add a commandline script to sign arbitrary json objects. diff --git a/mypy.ini b/mypy.ini index 0cf7c93f45..f4f981e813 100644 --- a/mypy.ini +++ b/mypy.ini @@ -8,6 +8,7 @@ show_traceback = True mypy_path = stubs warn_unreachable = True files = + scripts-dev/sign_json, synapse/api, synapse/appservice, synapse/config, diff --git a/scripts-dev/sign_json b/scripts-dev/sign_json new file mode 100755 index 0000000000..44553fb79a --- /dev/null +++ b/scripts-dev/sign_json @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# -*- coding: utf-8 -*- +# Copyright 2020 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import sys +from json import JSONDecodeError + +import yaml +from signedjson.key import read_signing_keys +from signedjson.sign import sign_json + +from synapse.util import json_encoder + + +def main(): + parser = argparse.ArgumentParser( + description="""Adds a signature to a JSON object. + +Example usage: + + $ scripts-dev/sign_json.py -N test -k localhost.signing.key "{}" + {"signatures":{"test":{"ed25519:a_ZnZh":"LmPnml6iM0iR..."}}} +""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "-N", + "--server-name", + help="Name to give as the local homeserver. If unspecified, will be " + "read from the config file.", + ) + + parser.add_argument( + "-k", + "--signing-key-path", + help="Path to the file containing the private ed25519 key to sign the " + "request with.", + ) + + parser.add_argument( + "-c", + "--config", + default="homeserver.yaml", + help=( + "Path to synapse config file, from which the server name and/or signing " + "key path will be read. Ignored if --server-name and --signing-key-path " + "are both given." + ), + ) + + input_args = parser.add_mutually_exclusive_group() + + input_args.add_argument("input_data", nargs="?", help="Raw JSON to be signed.") + + input_args.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help=( + "A file from which to read the JSON to be signed. If neither --input nor " + "input_data are given, JSON will be read from stdin." + ), + ) + + parser.add_argument( + "-o", + "--output", + type=argparse.FileType("w"), + default=sys.stdout, + help="Where to write the signed JSON. Defaults to stdout.", + ) + + args = parser.parse_args() + + if not args.server_name or not args.signing_key_path: + read_args_from_config(args) + + with open(args.signing_key_path) as f: + key = read_signing_keys(f)[0] + + json_to_sign = args.input_data + if json_to_sign is None: + json_to_sign = args.input.read() + + try: + obj = json.loads(json_to_sign) + except JSONDecodeError as e: + print("Unable to parse input as JSON: %s" % e, file=sys.stderr) + sys.exit(1) + + if not isinstance(obj, dict): + print("Input json was not an object", file=sys.stderr) + sys.exit(1) + + sign_json(obj, args.server_name, key) + for c in json_encoder.iterencode(obj): + args.output.write(c) + args.output.write("\n") + + +def read_args_from_config(args: argparse.Namespace) -> None: + with open(args.config, "r") as fh: + config = yaml.safe_load(fh) + if not args.server_name: + args.server_name = config["server_name"] + if not args.signing_key_path: + args.signing_key_path = config["signing_key_path"] + + +if __name__ == "__main__": + main() From b08dc7effed4a880384e386cdca50c5d15870904 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 24 Nov 2020 15:04:51 +0100 Subject: [PATCH 47/57] Clarify documentation of the admin list media API (#8795) Clarify that the list media API only shows media from unencrypted events. --- changelog.d/8795.doc | 1 + docs/admin_api/media_admin_api.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/8795.doc diff --git a/changelog.d/8795.doc b/changelog.d/8795.doc new file mode 100644 index 0000000000..f97a74efb5 --- /dev/null +++ b/changelog.d/8795.doc @@ -0,0 +1 @@ +Improve the documentation for the admin API to list all media in a room with respect to encrypted events. diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 3994e1f1a9..71137c6dfc 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -1,6 +1,7 @@ # List all media in a room This API gets a list of known media in a room. +However, it only shows media from unencrypted events or rooms. The API is: ``` From f38676d16143e399b654504486cf8cbecad12a5d Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 25 Nov 2020 07:07:21 -0500 Subject: [PATCH 48/57] Add type hints to matrix federation client / agent. (#8806) --- changelog.d/8806.misc | 1 + mypy.ini | 2 + .../federation/matrix_federation_agent.py | 100 +++--- .../http/federation/well_known_resolver.py | 16 +- synapse/http/matrixfederationclient.py | 304 +++++++++--------- synapse/server.py | 3 +- 6 files changed, 231 insertions(+), 195 deletions(-) create mode 100644 changelog.d/8806.misc diff --git a/changelog.d/8806.misc b/changelog.d/8806.misc new file mode 100644 index 0000000000..52457deb5e --- /dev/null +++ b/changelog.d/8806.misc @@ -0,0 +1 @@ +Add type hints to matrix federation client and agent. diff --git a/mypy.ini b/mypy.ini index f4f981e813..3e42235ac1 100644 --- a/mypy.ini +++ b/mypy.ini @@ -45,7 +45,9 @@ files = synapse/handlers/saml_handler.py, synapse/handlers/sync.py, synapse/handlers/ui_auth, + synapse/http/federation/matrix_federation_agent.py, synapse/http/federation/well_known_resolver.py, + synapse/http/matrixfederationclient.py, synapse/http/server.py, synapse/http/site.py, synapse/logging, diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 83d6196d4a..e77f9587d0 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -12,21 +12,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging -import urllib -from typing import List +import urllib.parse +from typing import List, Optional from netaddr import AddrFormatError, IPAddress from zope.interface import implementer from twisted.internet import defer from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS -from twisted.internet.interfaces import IStreamClientEndpoint -from twisted.web.client import Agent, HTTPConnectionPool +from twisted.internet.interfaces import ( + IProtocolFactory, + IReactorCore, + IStreamClientEndpoint, +) +from twisted.web.client import URI, Agent, HTTPConnectionPool from twisted.web.http_headers import Headers -from twisted.web.iweb import IAgent, IAgentEndpointFactory +from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer +from synapse.crypto.context_factory import FederationPolicyForHTTPS from synapse.http.federation.srv_resolver import Server, SrvResolver from synapse.http.federation.well_known_resolver import WellKnownResolver from synapse.logging.context import make_deferred_yieldable, run_in_background @@ -44,30 +48,30 @@ class MatrixFederationAgent: Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.) Args: - reactor (IReactor): twisted reactor to use for underlying requests + reactor: twisted reactor to use for underlying requests - tls_client_options_factory (FederationPolicyForHTTPS|None): + tls_client_options_factory: factory to use for fetching client tls options, or none to disable TLS. - user_agent (bytes): + user_agent: The user agent header to use for federation requests. - _srv_resolver (SrvResolver|None): - SRVResolver impl to use for looking up SRV records. None to use a default - implementation. + _srv_resolver: + SrvResolver implementation to use for looking up SRV records. None + to use a default implementation. - _well_known_resolver (WellKnownResolver|None): + _well_known_resolver: WellKnownResolver to use to perform well-known lookups. None to use a default implementation. """ def __init__( self, - reactor, - tls_client_options_factory, - user_agent, - _srv_resolver=None, - _well_known_resolver=None, + reactor: IReactorCore, + tls_client_options_factory: Optional[FederationPolicyForHTTPS], + user_agent: bytes, + _srv_resolver: Optional[SrvResolver] = None, + _well_known_resolver: Optional[WellKnownResolver] = None, ): self._reactor = reactor self._clock = Clock(reactor) @@ -99,15 +103,20 @@ class MatrixFederationAgent: self._well_known_resolver = _well_known_resolver @defer.inlineCallbacks - def request(self, method, uri, headers=None, bodyProducer=None): + def request( + self, + method: bytes, + uri: bytes, + headers: Optional[Headers] = None, + bodyProducer: Optional[IBodyProducer] = None, + ) -> defer.Deferred: """ Args: - method (bytes): HTTP method: GET/POST/etc - uri (bytes): Absolute URI to be retrieved - headers (twisted.web.http_headers.Headers|None): - HTTP headers to send with the request, or None to - send no extra headers. - bodyProducer (twisted.web.iweb.IBodyProducer|None): + method: HTTP method: GET/POST/etc + uri: Absolute URI to be retrieved + headers: + HTTP headers to send with the request, or None to send no extra headers. + bodyProducer: An object which can generate bytes to make up the body of this request (for example, the properly encoded contents of a file for a file upload). Or None if the request is to have @@ -123,6 +132,9 @@ class MatrixFederationAgent: # explicit port. parsed_uri = urllib.parse.urlparse(uri) + # There must be a valid hostname. + assert parsed_uri.hostname + # If this is a matrix:// URI check if the server has delegated matrix # traffic using well-known delegation. # @@ -179,7 +191,12 @@ class MatrixHostnameEndpointFactory: """Factory for MatrixHostnameEndpoint for parsing to an Agent. """ - def __init__(self, reactor, tls_client_options_factory, srv_resolver): + def __init__( + self, + reactor: IReactorCore, + tls_client_options_factory: Optional[FederationPolicyForHTTPS], + srv_resolver: Optional[SrvResolver], + ): self._reactor = reactor self._tls_client_options_factory = tls_client_options_factory @@ -203,15 +220,20 @@ class MatrixHostnameEndpoint: resolution (i.e. via SRV). Does not check for well-known delegation. Args: - reactor (IReactor) - tls_client_options_factory (ClientTLSOptionsFactory|None): + reactor: twisted reactor to use for underlying requests + tls_client_options_factory: factory to use for fetching client tls options, or none to disable TLS. - srv_resolver (SrvResolver): The SRV resolver to use - parsed_uri (twisted.web.client.URI): The parsed URI that we're wanting - to connect to. + srv_resolver: The SRV resolver to use + parsed_uri: The parsed URI that we're wanting to connect to. """ - def __init__(self, reactor, tls_client_options_factory, srv_resolver, parsed_uri): + def __init__( + self, + reactor: IReactorCore, + tls_client_options_factory: Optional[FederationPolicyForHTTPS], + srv_resolver: SrvResolver, + parsed_uri: URI, + ): self._reactor = reactor self._parsed_uri = parsed_uri @@ -231,13 +253,13 @@ class MatrixHostnameEndpoint: self._srv_resolver = srv_resolver - def connect(self, protocol_factory): + def connect(self, protocol_factory: IProtocolFactory) -> defer.Deferred: """Implements IStreamClientEndpoint interface """ return run_in_background(self._do_connect, protocol_factory) - async def _do_connect(self, protocol_factory): + async def _do_connect(self, protocol_factory: IProtocolFactory) -> None: first_exception = None server_list = await self._resolve_server() @@ -303,20 +325,20 @@ class MatrixHostnameEndpoint: return [Server(host, 8448)] -def _is_ip_literal(host): +def _is_ip_literal(host: bytes) -> bool: """Test if the given host name is either an IPv4 or IPv6 literal. Args: - host (bytes) + host: The host name to check Returns: - bool + True if the hostname is an IP address literal. """ - host = host.decode("ascii") + host_str = host.decode("ascii") try: - IPAddress(host) + IPAddress(host_str) return True except AddrFormatError: return False diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 1cc666fbf6..5e08ef1664 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging import random import time @@ -21,10 +20,11 @@ from typing import Callable, Dict, Optional, Tuple import attr from twisted.internet import defer +from twisted.internet.interfaces import IReactorTime from twisted.web.client import RedirectAgent, readBody from twisted.web.http import stringToDatetime from twisted.web.http_headers import Headers -from twisted.web.iweb import IResponse +from twisted.web.iweb import IAgent, IResponse from synapse.logging.context import make_deferred_yieldable from synapse.util import Clock, json_decoder @@ -81,11 +81,11 @@ class WellKnownResolver: def __init__( self, - reactor, - agent, - user_agent, - well_known_cache=None, - had_well_known_cache=None, + reactor: IReactorTime, + agent: IAgent, + user_agent: bytes, + well_known_cache: Optional[TTLCache] = None, + had_well_known_cache: Optional[TTLCache] = None, ): self._reactor = reactor self._clock = Clock(reactor) @@ -127,7 +127,7 @@ class WellKnownResolver: with Measure(self._clock, "get_well_known"): result, cache_period = await self._fetch_well_known( server_name - ) # type: Tuple[Optional[bytes], float] + ) # type: Optional[bytes], float except _FetchWellKnownFailure as e: if prev_result and e.temporary: diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 7e17cdb73e..b2ccae90df 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -17,8 +17,9 @@ import cgi import logging import random import sys -import urllib +import urllib.parse from io import BytesIO +from typing import BinaryIO, Callable, Dict, List, Optional, Tuple, Union import attr import treq @@ -31,9 +32,10 @@ from twisted.internet import defer, protocol from twisted.internet.error import DNSLookupError from twisted.internet.interfaces import IReactorPluggableNameResolver, IReactorTime from twisted.internet.task import _EPSILON, Cooperator +from twisted.python.failure import Failure from twisted.web._newclient import ResponseDone from twisted.web.http_headers import Headers -from twisted.web.iweb import IResponse +from twisted.web.iweb import IBodyProducer, IResponse import synapse.metrics import synapse.util.retryutils @@ -54,6 +56,7 @@ from synapse.logging.opentracing import ( start_active_span, tags, ) +from synapse.types import JsonDict from synapse.util import json_decoder from synapse.util.async_helpers import timeout_deferred from synapse.util.metrics import Measure @@ -76,47 +79,44 @@ MAXINT = sys.maxsize _next_id = 1 +QueryArgs = Dict[str, Union[str, List[str]]] + + @attr.s(slots=True, frozen=True) class MatrixFederationRequest: - method = attr.ib() + method = attr.ib(type=str) """HTTP method - :type: str """ - path = attr.ib() + path = attr.ib(type=str) """HTTP path - :type: str """ - destination = attr.ib() + destination = attr.ib(type=str) """The remote server to send the HTTP request to. - :type: str""" + """ - json = attr.ib(default=None) + json = attr.ib(default=None, type=Optional[JsonDict]) """JSON to send in the body. - :type: dict|None """ - json_callback = attr.ib(default=None) + json_callback = attr.ib(default=None, type=Optional[Callable[[], JsonDict]]) """A callback to generate the JSON. - :type: func|None """ - query = attr.ib(default=None) + query = attr.ib(default=None, type=Optional[dict]) """Query arguments. - :type: dict|None """ - txn_id = attr.ib(default=None) + txn_id = attr.ib(default=None, type=Optional[str]) """Unique ID for this request (for logging) - :type: str|None """ uri = attr.ib(init=False, type=bytes) """The URI of this request """ - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: global _next_id txn_id = "%s-O-%s" % (self.method, _next_id) _next_id = (_next_id + 1) % (MAXINT - 1) @@ -136,7 +136,7 @@ class MatrixFederationRequest: ) object.__setattr__(self, "uri", uri) - def get_json(self): + def get_json(self) -> Optional[JsonDict]: if self.json_callback: return self.json_callback() return self.json @@ -148,7 +148,7 @@ async def _handle_json_response( request: MatrixFederationRequest, response: IResponse, start_ms: int, -): +) -> JsonDict: """ Reads the JSON body of a response, with a timeout @@ -160,7 +160,7 @@ async def _handle_json_response( start_ms: Timestamp when request was made Returns: - dict: parsed JSON response + The parsed JSON response """ try: check_content_type_is_json(response.headers) @@ -266,27 +266,29 @@ class MatrixFederationHttpClient: self._cooperator = Cooperator(scheduler=schedule) async def _send_request_with_optional_trailing_slash( - self, request, try_trailing_slash_on_400=False, **send_request_args - ): + self, + request: MatrixFederationRequest, + try_trailing_slash_on_400: bool = False, + **send_request_args + ) -> IResponse: """Wrapper for _send_request which can optionally retry the request upon receiving a combination of a 400 HTTP response code and a 'M_UNRECOGNIZED' errcode. This is a workaround for Synapse <= v0.99.3 due to #3622. Args: - request (MatrixFederationRequest): details of request to be sent - try_trailing_slash_on_400 (bool): Whether on receiving a 400 + request: details of request to be sent + try_trailing_slash_on_400: Whether on receiving a 400 'M_UNRECOGNIZED' from the server to retry the request with a trailing slash appended to the request path. - send_request_args (Dict): A dictionary of arguments to pass to - `_send_request()`. + send_request_args: A dictionary of arguments to pass to `_send_request()`. Raises: HttpResponseException: If we get an HTTP response code >= 300 (except 429). Returns: - Dict: Parsed JSON response body. + Parsed JSON response body. """ try: response = await self._send_request(request, **send_request_args) @@ -313,24 +315,26 @@ class MatrixFederationHttpClient: async def _send_request( self, - request, - retry_on_dns_fail=True, - timeout=None, - long_retries=False, - ignore_backoff=False, - backoff_on_404=False, - ): + request: MatrixFederationRequest, + retry_on_dns_fail: bool = True, + timeout: Optional[int] = None, + long_retries: bool = False, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + ) -> IResponse: """ Sends a request to the given server. Args: - request (MatrixFederationRequest): details of request to be sent + request: details of request to be sent - timeout (int|None): number of milliseconds to wait for the response headers + retry_on_dns_fail: true if the request should be retied on DNS failures + + timeout: number of milliseconds to wait for the response headers (including connecting to the server), *for each attempt*. 60s by default. - long_retries (bool): whether to use the long retry algorithm. + long_retries: whether to use the long retry algorithm. The regular retry algorithm makes 4 attempts, with intervals [0.5s, 1s, 2s]. @@ -346,14 +350,13 @@ class MatrixFederationHttpClient: NB: the long retry algorithm takes over 20 minutes to complete, with a default timeout of 60s! - ignore_backoff (bool): true to ignore the historical backoff data + ignore_backoff: true to ignore the historical backoff data and try the request anyway. - backoff_on_404 (bool): Back off if we get a 404 + backoff_on_404: Back off if we get a 404 Returns: - twisted.web.client.Response: resolves with the HTTP - response object on success. + Resolves with the HTTP response object on success. Raises: HttpResponseException: If we get an HTTP response code >= 300 @@ -404,7 +407,7 @@ class MatrixFederationHttpClient: ) # Inject the span into the headers - headers_dict = {} + headers_dict = {} # type: Dict[bytes, List[bytes]] inject_active_span_byte_dict(headers_dict, request.destination) headers_dict[b"User-Agent"] = [self.version_string_bytes] @@ -435,7 +438,7 @@ class MatrixFederationHttpClient: data = encode_canonical_json(json) producer = QuieterFileBodyProducer( BytesIO(data), cooperator=self._cooperator - ) + ) # type: Optional[IBodyProducer] else: producer = None auth_headers = self.build_auth_headers( @@ -524,14 +527,16 @@ class MatrixFederationHttpClient: ) body = None - e = HttpResponseException(response.code, response_phrase, body) + exc = HttpResponseException( + response.code, response_phrase, body + ) # Retry if the error is a 429 (Too Many Requests), # otherwise just raise a standard HttpResponseException if response.code == 429: - raise RequestSendFailed(e, can_retry=True) from e + raise RequestSendFailed(exc, can_retry=True) from exc else: - raise e + raise exc break except RequestSendFailed as e: @@ -582,22 +587,27 @@ class MatrixFederationHttpClient: return response def build_auth_headers( - self, destination, method, url_bytes, content=None, destination_is=None - ): + self, + destination: Optional[bytes], + method: bytes, + url_bytes: bytes, + content: Optional[JsonDict] = None, + destination_is: Optional[bytes] = None, + ) -> List[bytes]: """ Builds the Authorization headers for a federation request Args: - destination (bytes|None): The destination homeserver of the request. + destination: The destination homeserver of the request. May be None if the destination is an identity server, in which case destination_is must be non-None. - method (bytes): The HTTP method of the request - url_bytes (bytes): The URI path of the request - content (object): The body of the request - destination_is (bytes): As 'destination', but if the destination is an + method: The HTTP method of the request + url_bytes: The URI path of the request + content: The body of the request + destination_is: As 'destination', but if the destination is an identity server Returns: - list[bytes]: a list of headers to be added as "Authorization:" headers + A list of headers to be added as "Authorization:" headers """ request = { "method": method.decode("ascii"), @@ -629,33 +639,32 @@ class MatrixFederationHttpClient: async def put_json( self, - destination, - path, - args={}, - data={}, - json_data_callback=None, - long_retries=False, - timeout=None, - ignore_backoff=False, - backoff_on_404=False, - try_trailing_slash_on_400=False, - ): + destination: str, + path: str, + args: Optional[QueryArgs] = None, + data: Optional[JsonDict] = None, + json_data_callback: Optional[Callable[[], JsonDict]] = None, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + try_trailing_slash_on_400: bool = False, + ) -> Union[JsonDict, list]: """ Sends the specified json data using PUT Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - args (dict): query params - data (dict): A dict containing the data that will be used as + destination: The remote server to send the HTTP request to. + path: The HTTP path. + args: query params + data: A dict containing the data that will be used as the request body. This will be encoded as JSON. - json_data_callback (callable): A callable returning the dict to + json_data_callback: A callable returning the dict to use as the request body. - long_retries (bool): whether to use the long retry algorithm. See + long_retries: whether to use the long retry algorithm. See docs on _send_request for details. - timeout (int|None): number of milliseconds to wait for the response. + timeout: number of milliseconds to wait for the response. self._default_timeout (60s) by default. Note that we may make several attempts to send the request; this @@ -663,19 +672,19 @@ class MatrixFederationHttpClient: *each* attempt (including connection time) as well as the time spent reading the response body after a 200 response. - ignore_backoff (bool): true to ignore the historical backoff data + ignore_backoff: true to ignore the historical backoff data and try the request anyway. - backoff_on_404 (bool): True if we should count a 404 response as + backoff_on_404: True if we should count a 404 response as a failure of the server (and should therefore back off future requests). - try_trailing_slash_on_400 (bool): True if on a 400 M_UNRECOGNIZED + try_trailing_slash_on_400: True if on a 400 M_UNRECOGNIZED response we should try appending a trailing slash to the end of the request. Workaround for #3622 in Synapse <= v0.99.3. This will be attempted before backing off if backing off has been enabled. Returns: - dict|list: Succeeds when we get a 2xx HTTP response. The + Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. Raises: @@ -721,29 +730,28 @@ class MatrixFederationHttpClient: async def post_json( self, - destination, - path, - data={}, - long_retries=False, - timeout=None, - ignore_backoff=False, - args={}, - ): + destination: str, + path: str, + data: Optional[JsonDict] = None, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + args: Optional[QueryArgs] = None, + ) -> Union[JsonDict, list]: """ Sends the specified json data using POST Args: - destination (str): The remote server to send the HTTP request - to. + destination: The remote server to send the HTTP request to. - path (str): The HTTP path. + path: The HTTP path. - data (dict): A dict containing the data that will be used as + data: A dict containing the data that will be used as the request body. This will be encoded as JSON. - long_retries (bool): whether to use the long retry algorithm. See + long_retries: whether to use the long retry algorithm. See docs on _send_request for details. - timeout (int|None): number of milliseconds to wait for the response. + timeout: number of milliseconds to wait for the response. self._default_timeout (60s) by default. Note that we may make several attempts to send the request; this @@ -751,10 +759,10 @@ class MatrixFederationHttpClient: *each* attempt (including connection time) as well as the time spent reading the response body after a 200 response. - ignore_backoff (bool): true to ignore the historical backoff data and + ignore_backoff: true to ignore the historical backoff data and try the request anyway. - args (dict): query params + args: query params Returns: dict|list: Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. @@ -795,26 +803,25 @@ class MatrixFederationHttpClient: async def get_json( self, - destination, - path, - args=None, - retry_on_dns_fail=True, - timeout=None, - ignore_backoff=False, - try_trailing_slash_on_400=False, - ): + destination: str, + path: str, + args: Optional[QueryArgs] = None, + retry_on_dns_fail: bool = True, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + try_trailing_slash_on_400: bool = False, + ) -> Union[JsonDict, list]: """ GETs some json from the given host homeserver and path Args: - destination (str): The remote server to send the HTTP request - to. + destination: The remote server to send the HTTP request to. - path (str): The HTTP path. + path: The HTTP path. - args (dict|None): A dictionary used to create query strings, defaults to + args: A dictionary used to create query strings, defaults to None. - timeout (int|None): number of milliseconds to wait for the response. + timeout: number of milliseconds to wait for the response. self._default_timeout (60s) by default. Note that we may make several attempts to send the request; this @@ -822,14 +829,14 @@ class MatrixFederationHttpClient: *each* attempt (including connection time) as well as the time spent reading the response body after a 200 response. - ignore_backoff (bool): true to ignore the historical backoff data + ignore_backoff: true to ignore the historical backoff data and try the request anyway. - try_trailing_slash_on_400 (bool): True if on a 400 M_UNRECOGNIZED + try_trailing_slash_on_400: True if on a 400 M_UNRECOGNIZED response we should try appending a trailing slash to the end of the request. Workaround for #3622 in Synapse <= v0.99.3. Returns: - dict|list: Succeeds when we get a 2xx HTTP response. The + Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. Raises: @@ -870,24 +877,23 @@ class MatrixFederationHttpClient: async def delete_json( self, - destination, - path, - long_retries=False, - timeout=None, - ignore_backoff=False, - args={}, - ): + destination: str, + path: str, + long_retries: bool = False, + timeout: Optional[int] = None, + ignore_backoff: bool = False, + args: Optional[QueryArgs] = None, + ) -> Union[JsonDict, list]: """Send a DELETE request to the remote expecting some json response Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. + destination: The remote server to send the HTTP request to. + path: The HTTP path. - long_retries (bool): whether to use the long retry algorithm. See + long_retries: whether to use the long retry algorithm. See docs on _send_request for details. - timeout (int|None): number of milliseconds to wait for the response. + timeout: number of milliseconds to wait for the response. self._default_timeout (60s) by default. Note that we may make several attempts to send the request; this @@ -895,12 +901,12 @@ class MatrixFederationHttpClient: *each* attempt (including connection time) as well as the time spent reading the response body after a 200 response. - ignore_backoff (bool): true to ignore the historical backoff data and + ignore_backoff: true to ignore the historical backoff data and try the request anyway. - args (dict): query params + args: query params Returns: - dict|list: Succeeds when we get a 2xx HTTP response. The + Succeeds when we get a 2xx HTTP response. The result will be the decoded JSON body. Raises: @@ -938,25 +944,25 @@ class MatrixFederationHttpClient: async def get_file( self, - destination, - path, + destination: str, + path: str, output_stream, - args={}, - retry_on_dns_fail=True, - max_size=None, - ignore_backoff=False, - ): + args: Optional[QueryArgs] = None, + retry_on_dns_fail: bool = True, + max_size: Optional[int] = None, + ignore_backoff: bool = False, + ) -> Tuple[int, Dict[bytes, List[bytes]]]: """GETs a file from a given homeserver Args: - destination (str): The remote server to send the HTTP request to. - path (str): The HTTP path to GET. - output_stream (file): File to write the response body to. - args (dict): Optional dictionary used to create the query string. - ignore_backoff (bool): true to ignore the historical backoff data + destination: The remote server to send the HTTP request to. + path: The HTTP path to GET. + output_stream: File to write the response body to. + args: Optional dictionary used to create the query string. + ignore_backoff: true to ignore the historical backoff data and try the request anyway. Returns: - tuple[int, dict]: Resolves with an (int,dict) tuple of + Resolves with an (int,dict) tuple of the file length and a dict of the response headers. Raises: @@ -1005,13 +1011,15 @@ class MatrixFederationHttpClient: class _ReadBodyToFileProtocol(protocol.Protocol): - def __init__(self, stream, deferred, max_size): + def __init__( + self, stream: BinaryIO, deferred: defer.Deferred, max_size: Optional[int] + ): self.stream = stream self.deferred = deferred self.length = 0 self.max_size = max_size - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: self.stream.write(data) self.length += len(data) if self.max_size is not None and self.length >= self.max_size: @@ -1025,14 +1033,16 @@ class _ReadBodyToFileProtocol(protocol.Protocol): self.deferred = defer.Deferred() self.transport.loseConnection() - def connectionLost(self, reason): + def connectionLost(self, reason: Failure) -> None: if reason.check(ResponseDone): self.deferred.callback(self.length) else: self.deferred.errback(reason) -def _readBodyToFile(response, stream, max_size): +def _readBodyToFile( + response: IResponse, stream: BinaryIO, max_size: Optional[int] +) -> defer.Deferred: d = defer.Deferred() response.deliverBody(_ReadBodyToFileProtocol(stream, d, max_size)) return d @@ -1049,13 +1059,13 @@ def _flatten_response_never_received(e): return repr(e) -def check_content_type_is_json(headers): +def check_content_type_is_json(headers: Headers) -> None: """ Check that a set of HTTP headers have a Content-Type header, and that it is application/json. Args: - headers (twisted.web.http_headers.Headers): headers to check + headers: headers to check Raises: RequestSendFailed: if the Content-Type header is missing or isn't JSON @@ -1080,7 +1090,7 @@ def check_content_type_is_json(headers): ) -def encode_query_args(args): +def encode_query_args(args: Optional[QueryArgs]) -> bytes: if args is None: return b"" @@ -1088,8 +1098,8 @@ def encode_query_args(args): for k, vs in args.items(): if isinstance(vs, str): vs = [vs] - encoded_args[k] = [v.encode("UTF-8") for v in vs] + encoded_args[k] = [v.encode("utf8") for v in vs] - query_bytes = urllib.parse.urlencode(encoded_args, True) + query_str = urllib.parse.urlencode(encoded_args, True) - return query_bytes.encode("utf8") + return query_str.encode("utf8") diff --git a/synapse/server.py b/synapse/server.py index 12a783de17..c82d8f9fad 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -27,7 +27,8 @@ import logging import os from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar, cast -import twisted +import twisted.internet.base +import twisted.internet.tcp from twisted.mail.smtp import sendmail from twisted.web.iweb import IPolicyForHTTPS From 4fd222ad704767e08c41a60690c4b499ed788b63 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 25 Nov 2020 10:04:22 -0500 Subject: [PATCH 49/57] Support trying multiple localparts for OpenID Connect. (#8801) Abstracts the SAML and OpenID Connect code which attempts to regenerate the localpart of a matrix ID if it is already in use. --- changelog.d/8801.feature | 1 + docs/sso_mapping_providers.md | 11 ++- synapse/handlers/oidc_handler.py | 120 ++++++++++-------------- synapse/handlers/saml_handler.py | 93 ++++++------------- synapse/handlers/sso.py | 155 ++++++++++++++++++++++++++++++- tests/handlers/test_oidc.py | 88 +++++++++++++++++- 6 files changed, 331 insertions(+), 137 deletions(-) create mode 100644 changelog.d/8801.feature diff --git a/changelog.d/8801.feature b/changelog.d/8801.feature new file mode 100644 index 0000000000..77f7fe4e5d --- /dev/null +++ b/changelog.d/8801.feature @@ -0,0 +1 @@ +Add support for re-trying generation of a localpart for OpenID Connect mapping providers. diff --git a/docs/sso_mapping_providers.md b/docs/sso_mapping_providers.md index 707dd73978..dee53b5d40 100644 --- a/docs/sso_mapping_providers.md +++ b/docs/sso_mapping_providers.md @@ -63,13 +63,22 @@ A custom mapping provider must specify the following methods: information from. - This method must return a string, which is the unique identifier for the user. Commonly the ``sub`` claim of the response. -* `map_user_attributes(self, userinfo, token)` +* `map_user_attributes(self, userinfo, token, failures)` - This method must be async. - Arguments: - `userinfo` - A `authlib.oidc.core.claims.UserInfo` object to extract user information from. - `token` - A dictionary which includes information necessary to make further requests to the OpenID provider. + - `failures` - An `int` that represents the amount of times the returned + mxid localpart mapping has failed. This should be used + to create a deduplicated mxid localpart which should be + returned instead. For example, if this method returns + `john.doe` as the value of `localpart` in the returned + dict, and that is already taken on the homeserver, this + method will be called again with the same parameters but + with failures=1. The method should then return a different + `localpart` value, such as `john.doe1`. - Returns a dictionary with two keys: - localpart: A required string, used to generate the Matrix ID. - displayname: An optional string, the display name for the user. diff --git a/synapse/handlers/oidc_handler.py b/synapse/handlers/oidc_handler.py index 34de9109ea..78c4e94a9d 100644 --- a/synapse/handlers/oidc_handler.py +++ b/synapse/handlers/oidc_handler.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import inspect import logging from typing import TYPE_CHECKING, Dict, Generic, List, Optional, Tuple, TypeVar from urllib.parse import urlencode @@ -35,15 +36,10 @@ from twisted.web.client import readBody from synapse.config import ConfigError from synapse.handlers._base import BaseHandler -from synapse.handlers.sso import MappingException +from synapse.handlers.sso import MappingException, UserAttributes from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable -from synapse.types import ( - JsonDict, - UserID, - contains_invalid_mxid_characters, - map_username_to_mxid_localpart, -) +from synapse.types import JsonDict, map_username_to_mxid_localpart from synapse.util import json_decoder if TYPE_CHECKING: @@ -869,73 +865,51 @@ class OidcHandler(BaseHandler): # to be strings. remote_user_id = str(remote_user_id) - # first of all, check if we already have a mapping for this user - previously_registered_user_id = await self._sso_handler.get_sso_user_by_remote_user_id( - self._auth_provider_id, remote_user_id, + # Older mapping providers don't accept the `failures` argument, so we + # try and detect support. + mapper_signature = inspect.signature( + self._user_mapping_provider.map_user_attributes ) - if previously_registered_user_id: - return previously_registered_user_id + supports_failures = "failures" in mapper_signature.parameters - # Otherwise, generate a new user. - try: - attributes = await self._user_mapping_provider.map_user_attributes( - userinfo, token - ) - except Exception as e: - raise MappingException( - "Could not extract user attributes from OIDC response: " + str(e) - ) + async def oidc_response_to_user_attributes(failures: int) -> UserAttributes: + """ + Call the mapping provider to map the OIDC userinfo and token to user attributes. - logger.debug( - "Retrieved user attributes from user mapping provider: %r", attributes - ) - - localpart = attributes["localpart"] - if not localpart: - raise MappingException( - "Error parsing OIDC response: OIDC mapping provider plugin " - "did not return a localpart value" - ) - - user_id = UserID(localpart, self.server_name).to_string() - users = await self.store.get_users_by_id_case_insensitive(user_id) - if users: - if self._allow_existing_users: - if len(users) == 1: - registered_user_id = next(iter(users)) - elif user_id in users: - registered_user_id = user_id - else: - raise MappingException( - "Attempted to login as '{}' but it matches more than one user inexactly: {}".format( - user_id, list(users.keys()) - ) - ) + This is backwards compatibility for abstraction for the SSO handler. + """ + if supports_failures: + attributes = await self._user_mapping_provider.map_user_attributes( + userinfo, token, failures + ) else: - # This mxid is taken - raise MappingException("mxid '{}' is already taken".format(user_id)) - else: - # Since the localpart is provided via a potentially untrusted module, - # ensure the MXID is valid before registering. - if contains_invalid_mxid_characters(localpart): - raise MappingException("localpart is invalid: %s" % (localpart,)) + # If the mapping provider does not support processing failures, + # do not continually generate the same Matrix ID since it will + # continue to already be in use. Note that the error raised is + # arbitrary and will get turned into a MappingException. + if failures: + raise RuntimeError( + "Mapping provider does not support de-duplicating Matrix IDs" + ) - # It's the first time this user is logging in and the mapped mxid was - # not taken, register the user - registered_user_id = await self._registration_handler.register_user( - localpart=localpart, - default_display_name=attributes["display_name"], - user_agent_ips=[(user_agent, ip_address)], - ) + attributes = await self._user_mapping_provider.map_user_attributes( # type: ignore + userinfo, token + ) - await self.store.record_user_external_id( - self._auth_provider_id, remote_user_id, registered_user_id, + return UserAttributes(**attributes) + + return await self._sso_handler.get_mxid_from_sso( + self._auth_provider_id, + remote_user_id, + user_agent, + ip_address, + oidc_response_to_user_attributes, + self._allow_existing_users, ) - return registered_user_id -UserAttribute = TypedDict( - "UserAttribute", {"localpart": str, "display_name": Optional[str]} +UserAttributeDict = TypedDict( + "UserAttributeDict", {"localpart": str, "display_name": Optional[str]} ) C = TypeVar("C") @@ -978,13 +952,15 @@ class OidcMappingProvider(Generic[C]): raise NotImplementedError() async def map_user_attributes( - self, userinfo: UserInfo, token: Token - ) -> UserAttribute: + self, userinfo: UserInfo, token: Token, failures: int + ) -> UserAttributeDict: """Map a `UserInfo` object into user attributes. Args: userinfo: An object representing the user given by the OIDC provider token: A dict with the tokens returned by the provider + failures: How many times a call to this function with this + UserInfo has resulted in a failure. Returns: A dict containing the ``localpart`` and (optionally) the ``display_name`` @@ -1084,13 +1060,17 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]): return userinfo[self._config.subject_claim] async def map_user_attributes( - self, userinfo: UserInfo, token: Token - ) -> UserAttribute: + self, userinfo: UserInfo, token: Token, failures: int + ) -> UserAttributeDict: localpart = self._config.localpart_template.render(user=userinfo).strip() # Ensure only valid characters are included in the MXID. localpart = map_username_to_mxid_localpart(localpart) + # Append suffix integer if last call to this function failed to produce + # a usable mxid. + localpart += str(failures) if failures else "" + display_name = None # type: Optional[str] if self._config.display_name_template is not None: display_name = self._config.display_name_template.render( @@ -1100,7 +1080,7 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]): if display_name == "": display_name = None - return UserAttribute(localpart=localpart, display_name=display_name) + return UserAttributeDict(localpart=localpart, display_name=display_name) async def get_extra_attributes(self, userinfo: UserInfo, token: Token) -> JsonDict: extras = {} # type: Dict[str, str] diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py index 37ab42f050..34db10ffe4 100644 --- a/synapse/handlers/saml_handler.py +++ b/synapse/handlers/saml_handler.py @@ -25,13 +25,12 @@ from synapse.api.errors import SynapseError from synapse.config import ConfigError from synapse.config.saml2_config import SamlAttributeRequirement from synapse.handlers._base import BaseHandler -from synapse.handlers.sso import MappingException +from synapse.handlers.sso import MappingException, UserAttributes from synapse.http.servlet import parse_string from synapse.http.site import SynapseRequest from synapse.module_api import ModuleApi from synapse.types import ( UserID, - contains_invalid_mxid_characters, map_username_to_mxid_localpart, mxid_localpart_allowed_characters, ) @@ -250,14 +249,26 @@ class SamlHandler(BaseHandler): "Failed to extract remote user id from SAML response" ) - with (await self._mapping_lock.queue(self._auth_provider_id)): - # first of all, check if we already have a mapping for this user - previously_registered_user_id = await self._sso_handler.get_sso_user_by_remote_user_id( - self._auth_provider_id, remote_user_id, - ) - if previously_registered_user_id: - return previously_registered_user_id + async def saml_response_to_remapped_user_attributes( + failures: int, + ) -> UserAttributes: + """ + Call the mapping provider to map a SAML response to user attributes and coerce the result into the standard form. + This is backwards compatibility for abstraction for the SSO handler. + """ + # Call the mapping provider. + result = self._user_mapping_provider.saml_response_to_user_attributes( + saml2_auth, failures, client_redirect_url + ) + # Remap some of the results. + return UserAttributes( + localpart=result.get("mxid_localpart"), + display_name=result.get("displayname"), + emails=result.get("emails"), + ) + + with (await self._mapping_lock.queue(self._auth_provider_id)): # backwards-compatibility hack: see if there is an existing user with a # suitable mapping from the uid if ( @@ -284,60 +295,14 @@ class SamlHandler(BaseHandler): ) return registered_user_id - # Map saml response to user attributes using the configured mapping provider - for i in range(1000): - attribute_dict = self._user_mapping_provider.saml_response_to_user_attributes( - saml2_auth, i, client_redirect_url=client_redirect_url, - ) - - logger.debug( - "Retrieved SAML attributes from user mapping provider: %s " - "(attempt %d)", - attribute_dict, - i, - ) - - localpart = attribute_dict.get("mxid_localpart") - if not localpart: - raise MappingException( - "Error parsing SAML2 response: SAML mapping provider plugin " - "did not return a mxid_localpart value" - ) - - displayname = attribute_dict.get("displayname") - emails = attribute_dict.get("emails", []) - - # Check if this mxid already exists - if not await self.store.get_users_by_id_case_insensitive( - UserID(localpart, self.server_name).to_string() - ): - # This mxid is free - break - else: - # Unable to generate a username in 1000 iterations - # Break and return error to the user - raise MappingException( - "Unable to generate a Matrix ID from the SAML response" - ) - - # Since the localpart is provided via a potentially untrusted module, - # ensure the MXID is valid before registering. - if contains_invalid_mxid_characters(localpart): - raise MappingException("localpart is invalid: %s" % (localpart,)) - - logger.debug("Mapped SAML user to local part %s", localpart) - registered_user_id = await self._registration_handler.register_user( - localpart=localpart, - default_display_name=displayname, - bind_emails=emails, - user_agent_ips=[(user_agent, ip_address)], + return await self._sso_handler.get_mxid_from_sso( + self._auth_provider_id, + remote_user_id, + user_agent, + ip_address, + saml_response_to_remapped_user_attributes, ) - await self.store.record_user_external_id( - self._auth_provider_id, remote_user_id, registered_user_id - ) - return registered_user_id - def expire_sessions(self): expire_before = self.clock.time_msec() - self._saml2_session_lifetime to_expire = set() @@ -451,11 +416,11 @@ class DefaultSamlMappingProvider: ) # Use the configured mapper for this mxid_source - base_mxid_localpart = self._mxid_mapper(mxid_source) + localpart = self._mxid_mapper(mxid_source) # Append suffix integer if last call to this function failed to produce - # a usable mxid - localpart = base_mxid_localpart + (str(failures) if failures else "") + # a usable mxid. + localpart += str(failures) if failures else "" # Retrieve the display name from the saml response # If displayname is None, the mxid_localpart will be used instead diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index cf7cb7754a..d963082210 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -13,10 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional + +import attr from synapse.handlers._base import BaseHandler from synapse.http.server import respond_with_html +from synapse.types import UserID, contains_invalid_mxid_characters if TYPE_CHECKING: from synapse.server import HomeServer @@ -29,9 +32,20 @@ class MappingException(Exception): """ +@attr.s +class UserAttributes: + localpart = attr.ib(type=str) + display_name = attr.ib(type=Optional[str], default=None) + emails = attr.ib(type=List[str], default=attr.Factory(list)) + + class SsoHandler(BaseHandler): + # The number of attempts to ask the mapping provider for when generating an MXID. + _MAP_USERNAME_RETRIES = 1000 + def __init__(self, hs: "HomeServer"): super().__init__(hs) + self._registration_handler = hs.get_registration_handler() self._error_template = hs.config.sso_error_template def render_error( @@ -94,3 +108,142 @@ class SsoHandler(BaseHandler): # No match. return None + + async def get_mxid_from_sso( + self, + auth_provider_id: str, + remote_user_id: str, + user_agent: str, + ip_address: str, + sso_to_matrix_id_mapper: Callable[[int], Awaitable[UserAttributes]], + allow_existing_users: bool = False, + ) -> str: + """ + Given an SSO ID, retrieve the user ID for it and possibly register the user. + + This first checks if the SSO ID has previously been linked to a matrix ID, + if it has that matrix ID is returned regardless of the current mapping + logic. + + The mapping function is called (potentially multiple times) to generate + a localpart for the user. + + If an unused localpart is generated, the user is registered from the + given user-agent and IP address and the SSO ID is linked to this matrix + ID for subsequent calls. + + If allow_existing_users is true the mapping function is only called once + and results in: + + 1. The use of a previously registered matrix ID. In this case, the + SSO ID is linked to the matrix ID. (Note it is possible that + other SSO IDs are linked to the same matrix ID.) + 2. An unused localpart, in which case the user is registered (as + discussed above). + 3. An error if the generated localpart matches multiple pre-existing + matrix IDs. Generally this should not happen. + + Args: + auth_provider_id: A unique identifier for this SSO provider, e.g. + "oidc" or "saml". + remote_user_id: The unique identifier from the SSO provider. + user_agent: The user agent of the client making the request. + ip_address: The IP address of the client making the request. + sso_to_matrix_id_mapper: A callable to generate the user attributes. + The only parameter is an integer which represents the amount of + times the returned mxid localpart mapping has failed. + allow_existing_users: True if the localpart returned from the + mapping provider can be linked to an existing matrix ID. + + Returns: + The user ID associated with the SSO response. + + Raises: + MappingException if there was a problem mapping the response to a user. + RedirectException: some mapping providers may raise this if they need + to redirect to an interstitial page. + + """ + # first of all, check if we already have a mapping for this user + previously_registered_user_id = await self.get_sso_user_by_remote_user_id( + auth_provider_id, remote_user_id, + ) + if previously_registered_user_id: + return previously_registered_user_id + + # Otherwise, generate a new user. + for i in range(self._MAP_USERNAME_RETRIES): + try: + attributes = await sso_to_matrix_id_mapper(i) + except Exception as e: + raise MappingException( + "Could not extract user attributes from SSO response: " + str(e) + ) + + logger.debug( + "Retrieved user attributes from user mapping provider: %r (attempt %d)", + attributes, + i, + ) + + if not attributes.localpart: + raise MappingException( + "Error parsing SSO response: SSO mapping provider plugin " + "did not return a localpart value" + ) + + # Check if this mxid already exists + user_id = UserID(attributes.localpart, self.server_name).to_string() + users = await self.store.get_users_by_id_case_insensitive(user_id) + # Note, if allow_existing_users is true then the loop is guaranteed + # to end on the first iteration: either by matching an existing user, + # raising an error, or registering a new user. See the docstring for + # more in-depth an explanation. + if users and allow_existing_users: + # If an existing matrix ID is returned, then use it. + if len(users) == 1: + previously_registered_user_id = next(iter(users)) + elif user_id in users: + previously_registered_user_id = user_id + else: + # Do not attempt to continue generating Matrix IDs. + raise MappingException( + "Attempted to login as '{}' but it matches more than one user inexactly: {}".format( + user_id, users + ) + ) + + # Future logins should also match this user ID. + await self.store.record_user_external_id( + auth_provider_id, remote_user_id, previously_registered_user_id + ) + + return previously_registered_user_id + + elif not users: + # This mxid is free + break + else: + # Unable to generate a username in 1000 iterations + # Break and return error to the user + raise MappingException( + "Unable to generate a Matrix ID from the SSO response" + ) + + # Since the localpart is provided via a potentially untrusted module, + # ensure the MXID is valid before registering. + if contains_invalid_mxid_characters(attributes.localpart): + raise MappingException("localpart is invalid: %s" % (attributes.localpart,)) + + logger.debug("Mapped SSO user to local part %s", attributes.localpart) + registered_user_id = await self._registration_handler.register_user( + localpart=attributes.localpart, + default_display_name=attributes.display_name, + bind_emails=attributes.emails, + user_agent_ips=[(user_agent, ip_address)], + ) + + await self.store.record_user_external_id( + auth_provider_id, remote_user_id, registered_user_id + ) + return registered_user_id diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index b4fa02acc4..e880d32be6 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -89,6 +89,14 @@ class TestMappingProviderExtra(TestMappingProvider): return {"phone": userinfo["phone"]} +class TestMappingProviderFailures(TestMappingProvider): + async def map_user_attributes(self, userinfo, token, failures): + return { + "localpart": userinfo["username"] + (str(failures) if failures else ""), + "display_name": None, + } + + def simple_async_mock(return_value=None, raises=None): # AsyncMock is not available in python3.5, this mimics part of its behaviour async def cb(*args, **kwargs): @@ -152,6 +160,9 @@ class OidcHandlerTestCase(HomeserverTestCase): self.render_error = Mock(return_value=None) self.handler._sso_handler.render_error = self.render_error + # Reduce the number of attempts when generating MXIDs. + self.handler._sso_handler._MAP_USERNAME_RETRIES = 3 + return hs def metadata_edit(self, values): @@ -693,7 +704,10 @@ class OidcHandlerTestCase(HomeserverTestCase): ), MappingException, ) - self.assertEqual(str(e.value), "mxid '@test_user_3:test' is already taken") + self.assertEqual( + str(e.value), + "Could not extract user attributes from SSO response: Mapping provider does not support de-duplicating Matrix IDs", + ) @override_config({"oidc_config": {"allow_existing_users": True}}) def test_map_userinfo_to_existing_user(self): @@ -703,6 +717,8 @@ class OidcHandlerTestCase(HomeserverTestCase): self.get_success( store.register_user(user_id=user.to_string(), password_hash=None) ) + + # Map a user via SSO. userinfo = { "sub": "test", "username": "test_user", @@ -715,6 +731,23 @@ class OidcHandlerTestCase(HomeserverTestCase): ) self.assertEqual(mxid, "@test_user:test") + # Note that a second SSO user can be mapped to the same Matrix ID. (This + # requires a unique sub, but something that maps to the same matrix ID, + # in this case we'll just use the same username. A more realistic example + # would be subs which are email addresses, and mapping from the localpart + # of the email, e.g. bob@foo.com and bob@bar.com -> @bob:test.) + userinfo = { + "sub": "test1", + "username": "test_user", + } + token = {} + mxid = self.get_success( + self.handler._map_userinfo_to_user( + userinfo, token, "user-agent", "10.10.10.10" + ) + ) + self.assertEqual(mxid, "@test_user:test") + # Register some non-exact matching cases. user2 = UserID.from_string("@TEST_user_2:test") self.get_success( @@ -762,6 +795,7 @@ class OidcHandlerTestCase(HomeserverTestCase): "username": "föö", } token = {} + e = self.get_failure( self.handler._map_userinfo_to_user( userinfo, token, "user-agent", "10.10.10.10" @@ -769,3 +803,55 @@ class OidcHandlerTestCase(HomeserverTestCase): MappingException, ) self.assertEqual(str(e.value), "localpart is invalid: föö") + + @override_config( + { + "oidc_config": { + "user_mapping_provider": { + "module": __name__ + ".TestMappingProviderFailures" + } + } + } + ) + def test_map_userinfo_to_user_retries(self): + """The mapping provider can retry generating an MXID if the MXID is already in use.""" + store = self.hs.get_datastore() + self.get_success( + store.register_user(user_id="@test_user:test", password_hash=None) + ) + userinfo = { + "sub": "test", + "username": "test_user", + } + token = {} + mxid = self.get_success( + self.handler._map_userinfo_to_user( + userinfo, token, "user-agent", "10.10.10.10" + ) + ) + # test_user is already taken, so test_user1 gets registered instead. + self.assertEqual(mxid, "@test_user1:test") + + # Register all of the potential users for a particular username. + self.get_success( + store.register_user(user_id="@tester:test", password_hash=None) + ) + for i in range(1, 3): + self.get_success( + store.register_user(user_id="@tester%d:test" % i, password_hash=None) + ) + + # Now attempt to map to a username, this will fail since all potential usernames are taken. + userinfo = { + "sub": "tester", + "username": "tester", + } + e = self.get_failure( + self.handler._map_userinfo_to_user( + userinfo, token, "user-agent", "10.10.10.10" + ), + MappingException, + ) + self.assertEqual( + str(e.value), "Unable to generate a Matrix ID from the SSO response" + ) From 968939bdacc66be91aeba440a6b2ae7bc84731f1 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 25 Nov 2020 13:30:47 -0500 Subject: [PATCH 50/57] Add additional type hints to HTTP client. (#8812) This also removes some duplicated code between the simple HTTP client and matrix federation client. --- changelog.d/8806.misc | 2 +- changelog.d/8812.misc | 1 + mypy.ini | 3 +- synapse/http/client.py | 211 +++++++++++++++---------- synapse/http/matrixfederationclient.py | 74 ++------- 5 files changed, 142 insertions(+), 149 deletions(-) create mode 100644 changelog.d/8812.misc diff --git a/changelog.d/8806.misc b/changelog.d/8806.misc index 52457deb5e..ee144846a5 100644 --- a/changelog.d/8806.misc +++ b/changelog.d/8806.misc @@ -1 +1 @@ -Add type hints to matrix federation client and agent. +Add type hints to HTTP abstractions. diff --git a/changelog.d/8812.misc b/changelog.d/8812.misc new file mode 100644 index 0000000000..ee144846a5 --- /dev/null +++ b/changelog.d/8812.misc @@ -0,0 +1 @@ +Add type hints to HTTP abstractions. diff --git a/mypy.ini b/mypy.ini index 3e42235ac1..a5503abe26 100644 --- a/mypy.ini +++ b/mypy.ini @@ -45,6 +45,7 @@ files = synapse/handlers/saml_handler.py, synapse/handlers/sync.py, synapse/handlers/ui_auth, + synapse/http/client.py, synapse/http/federation/matrix_federation_agent.py, synapse/http/federation/well_known_resolver.py, synapse/http/matrixfederationclient.py, @@ -109,7 +110,7 @@ ignore_missing_imports = True [mypy-opentracing] ignore_missing_imports = True -[mypy-OpenSSL] +[mypy-OpenSSL.*] ignore_missing_imports = True [mypy-netaddr] diff --git a/synapse/http/client.py b/synapse/http/client.py index f409368802..e5b13593f2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import urllib +import urllib.parse from io import BytesIO from typing import ( + TYPE_CHECKING, Any, BinaryIO, Dict, @@ -31,7 +32,7 @@ from typing import ( import treq from canonicaljson import encode_canonical_json -from netaddr import IPAddress +from netaddr import IPAddress, IPSet from prometheus_client import Counter from zope.interface import implementer, provider @@ -39,6 +40,8 @@ from OpenSSL import SSL from OpenSSL.SSL import VERIFY_NONE from twisted.internet import defer, error as twisted_error, protocol, ssl from twisted.internet.interfaces import ( + IAddress, + IHostResolution, IReactorPluggableNameResolver, IResolutionReceiver, ) @@ -53,7 +56,7 @@ from twisted.web.client import ( ) from twisted.web.http import PotentialDataLoss from twisted.web.http_headers import Headers -from twisted.web.iweb import IResponse +from twisted.web.iweb import IAgent, IBodyProducer, IResponse from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.http import QuieterFileBodyProducer, RequestTimedOutError, redact_uri @@ -63,6 +66,9 @@ from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.util import json_decoder from synapse.util.async_helpers import timeout_deferred +if TYPE_CHECKING: + from synapse.app.homeserver import HomeServer + logger = logging.getLogger(__name__) outgoing_requests_counter = Counter("synapse_http_client_requests", "", ["method"]) @@ -84,12 +90,19 @@ QueryParamValue = Union[str, bytes, Iterable[Union[str, bytes]]] QueryParams = Union[Mapping[str, QueryParamValue], Mapping[bytes, QueryParamValue]] -def check_against_blacklist(ip_address, ip_whitelist, ip_blacklist): +def check_against_blacklist( + ip_address: IPAddress, ip_whitelist: Optional[IPSet], ip_blacklist: IPSet +) -> bool: """ + Compares an IP address to allowed and disallowed IP sets. + Args: - ip_address (netaddr.IPAddress) - ip_whitelist (netaddr.IPSet) - ip_blacklist (netaddr.IPSet) + ip_address: The IP address to check + ip_whitelist: Allowed IP addresses. + ip_blacklist: Disallowed IP addresses. + + Returns: + True if the IP address is in the blacklist and not in the whitelist. """ if ip_address in ip_blacklist: if ip_whitelist is None or ip_address not in ip_whitelist: @@ -118,23 +131,30 @@ class IPBlacklistingResolver: addresses, preventing DNS rebinding attacks on URL preview. """ - def __init__(self, reactor, ip_whitelist, ip_blacklist): + def __init__( + self, + reactor: IReactorPluggableNameResolver, + ip_whitelist: Optional[IPSet], + ip_blacklist: IPSet, + ): """ Args: - reactor (twisted.internet.reactor) - ip_whitelist (netaddr.IPSet) - ip_blacklist (netaddr.IPSet) + reactor: The twisted reactor. + ip_whitelist: IP addresses to allow. + ip_blacklist: IP addresses to disallow. """ self._reactor = reactor self._ip_whitelist = ip_whitelist self._ip_blacklist = ip_blacklist - def resolveHostName(self, recv, hostname, portNumber=0): + def resolveHostName( + self, recv: IResolutionReceiver, hostname: str, portNumber: int = 0 + ) -> IResolutionReceiver: r = recv() - addresses = [] + addresses = [] # type: List[IAddress] - def _callback(): + def _callback() -> None: r.resolutionBegan(None) has_bad_ip = False @@ -161,15 +181,15 @@ class IPBlacklistingResolver: @provider(IResolutionReceiver) class EndpointReceiver: @staticmethod - def resolutionBegan(resolutionInProgress): + def resolutionBegan(resolutionInProgress: IHostResolution) -> None: pass @staticmethod - def addressResolved(address): + def addressResolved(address: IAddress) -> None: addresses.append(address) @staticmethod - def resolutionComplete(): + def resolutionComplete() -> None: _callback() self._reactor.nameResolver.resolveHostName( @@ -185,19 +205,29 @@ class BlacklistingAgentWrapper(Agent): directly (without an IP address lookup). """ - def __init__(self, agent, reactor, ip_whitelist=None, ip_blacklist=None): + def __init__( + self, + agent: IAgent, + ip_whitelist: Optional[IPSet] = None, + ip_blacklist: Optional[IPSet] = None, + ): """ Args: - agent (twisted.web.client.Agent): The Agent to wrap. - reactor (twisted.internet.reactor) - ip_whitelist (netaddr.IPSet) - ip_blacklist (netaddr.IPSet) + agent: The Agent to wrap. + ip_whitelist: IP addresses to allow. + ip_blacklist: IP addresses to disallow. """ self._agent = agent self._ip_whitelist = ip_whitelist self._ip_blacklist = ip_blacklist - def request(self, method, uri, headers=None, bodyProducer=None): + def request( + self, + method: bytes, + uri: bytes, + headers: Optional[Headers] = None, + bodyProducer: Optional[IBodyProducer] = None, + ) -> defer.Deferred: h = urllib.parse.urlparse(uri.decode("ascii")) try: @@ -226,23 +256,23 @@ class SimpleHttpClient: def __init__( self, - hs, - treq_args={}, - ip_whitelist=None, - ip_blacklist=None, - http_proxy=None, - https_proxy=None, + hs: "HomeServer", + treq_args: Dict[str, Any] = {}, + ip_whitelist: Optional[IPSet] = None, + ip_blacklist: Optional[IPSet] = None, + http_proxy: Optional[bytes] = None, + https_proxy: Optional[bytes] = None, ): """ Args: - hs (synapse.server.HomeServer) - treq_args (dict): Extra keyword arguments to be given to treq.request. - ip_blacklist (netaddr.IPSet): The IP addresses that are blacklisted that + hs + treq_args: Extra keyword arguments to be given to treq.request. + ip_blacklist: The IP addresses that are blacklisted that we may not request. - ip_whitelist (netaddr.IPSet): The whitelisted IP addresses, that we can + ip_whitelist: The whitelisted IP addresses, that we can request if it were otherwise caught in a blacklist. - http_proxy (bytes): proxy server to use for http connections. host[:port] - https_proxy (bytes): proxy server to use for https connections. host[:port] + http_proxy: proxy server to use for http connections. host[:port] + https_proxy: proxy server to use for https connections. host[:port] """ self.hs = hs @@ -306,7 +336,6 @@ class SimpleHttpClient: # by the DNS resolution. self.agent = BlacklistingAgentWrapper( self.agent, - self.reactor, ip_whitelist=self._ip_whitelist, ip_blacklist=self._ip_blacklist, ) @@ -397,7 +426,7 @@ class SimpleHttpClient: async def post_urlencoded_get_json( self, uri: str, - args: Mapping[str, Union[str, List[str]]] = {}, + args: Optional[Mapping[str, Union[str, List[str]]]] = None, headers: Optional[RawHeaders] = None, ) -> Any: """ @@ -422,9 +451,7 @@ class SimpleHttpClient: # TODO: Do we ever want to log message contents? logger.debug("post_urlencoded_get_json args: %s", args) - query_bytes = urllib.parse.urlencode(encode_urlencode_args(args), True).encode( - "utf8" - ) + query_bytes = encode_query_args(args) actual_headers = { b"Content-Type": [b"application/x-www-form-urlencoded"], @@ -432,7 +459,7 @@ class SimpleHttpClient: b"Accept": [b"application/json"], } if headers: - actual_headers.update(headers) + actual_headers.update(headers) # type: ignore response = await self.request( "POST", uri, headers=Headers(actual_headers), data=query_bytes @@ -479,7 +506,7 @@ class SimpleHttpClient: b"Accept": [b"application/json"], } if headers: - actual_headers.update(headers) + actual_headers.update(headers) # type: ignore response = await self.request( "POST", uri, headers=Headers(actual_headers), data=json_str @@ -495,7 +522,10 @@ class SimpleHttpClient: ) async def get_json( - self, uri: str, args: QueryParams = {}, headers: Optional[RawHeaders] = None, + self, + uri: str, + args: Optional[QueryParams] = None, + headers: Optional[RawHeaders] = None, ) -> Any: """Gets some json from the given URI. @@ -516,7 +546,7 @@ class SimpleHttpClient: """ actual_headers = {b"Accept": [b"application/json"]} if headers: - actual_headers.update(headers) + actual_headers.update(headers) # type: ignore body = await self.get_raw(uri, args, headers=headers) return json_decoder.decode(body.decode("utf-8")) @@ -525,7 +555,7 @@ class SimpleHttpClient: self, uri: str, json_body: Any, - args: QueryParams = {}, + args: Optional[QueryParams] = None, headers: RawHeaders = None, ) -> Any: """Puts some json to the given URI. @@ -546,9 +576,9 @@ class SimpleHttpClient: ValueError: if the response was not JSON """ - if len(args): - query_bytes = urllib.parse.urlencode(args, True) - uri = "%s?%s" % (uri, query_bytes) + if args: + query_str = urllib.parse.urlencode(args, True) + uri = "%s?%s" % (uri, query_str) json_str = encode_canonical_json(json_body) @@ -558,7 +588,7 @@ class SimpleHttpClient: b"Accept": [b"application/json"], } if headers: - actual_headers.update(headers) + actual_headers.update(headers) # type: ignore response = await self.request( "PUT", uri, headers=Headers(actual_headers), data=json_str @@ -574,7 +604,10 @@ class SimpleHttpClient: ) async def get_raw( - self, uri: str, args: QueryParams = {}, headers: Optional[RawHeaders] = None + self, + uri: str, + args: Optional[QueryParams] = None, + headers: Optional[RawHeaders] = None, ) -> bytes: """Gets raw text from the given URI. @@ -592,13 +625,13 @@ class SimpleHttpClient: HttpResponseException on a non-2xx HTTP response. """ - if len(args): - query_bytes = urllib.parse.urlencode(args, True) - uri = "%s?%s" % (uri, query_bytes) + if args: + query_str = urllib.parse.urlencode(args, True) + uri = "%s?%s" % (uri, query_str) actual_headers = {b"User-Agent": [self.user_agent]} if headers: - actual_headers.update(headers) + actual_headers.update(headers) # type: ignore response = await self.request("GET", uri, headers=Headers(actual_headers)) @@ -641,7 +674,7 @@ class SimpleHttpClient: actual_headers = {b"User-Agent": [self.user_agent]} if headers: - actual_headers.update(headers) + actual_headers.update(headers) # type: ignore response = await self.request("GET", url, headers=Headers(actual_headers)) @@ -649,12 +682,13 @@ class SimpleHttpClient: if ( b"Content-Length" in resp_headers + and max_size and int(resp_headers[b"Content-Length"][0]) > max_size ): - logger.warning("Requested URL is too large > %r bytes" % (self.max_size,)) + logger.warning("Requested URL is too large > %r bytes" % (max_size,)) raise SynapseError( 502, - "Requested file is too large > %r bytes" % (self.max_size,), + "Requested file is too large > %r bytes" % (max_size,), Codes.TOO_LARGE, ) @@ -668,7 +702,7 @@ class SimpleHttpClient: try: length = await make_deferred_yieldable( - _readBodyToFile(response, output_stream, max_size) + readBodyToFile(response, output_stream, max_size) ) except SynapseError: # This can happen e.g. because the body is too large. @@ -696,18 +730,16 @@ def _timeout_to_request_timed_out_error(f: Failure): return f -# XXX: FIXME: This is horribly copy-pasted from matrixfederationclient. -# The two should be factored out. - - class _ReadBodyToFileProtocol(protocol.Protocol): - def __init__(self, stream, deferred, max_size): + def __init__( + self, stream: BinaryIO, deferred: defer.Deferred, max_size: Optional[int] + ): self.stream = stream self.deferred = deferred self.length = 0 self.max_size = max_size - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: self.stream.write(data) self.length += len(data) if self.max_size is not None and self.length >= self.max_size: @@ -721,7 +753,7 @@ class _ReadBodyToFileProtocol(protocol.Protocol): self.deferred = defer.Deferred() self.transport.loseConnection() - def connectionLost(self, reason): + def connectionLost(self, reason: Failure) -> None: if reason.check(ResponseDone): self.deferred.callback(self.length) elif reason.check(PotentialDataLoss): @@ -732,35 +764,48 @@ class _ReadBodyToFileProtocol(protocol.Protocol): self.deferred.errback(reason) -# XXX: FIXME: This is horribly copy-pasted from matrixfederationclient. -# The two should be factored out. +def readBodyToFile( + response: IResponse, stream: BinaryIO, max_size: Optional[int] +) -> defer.Deferred: + """ + Read a HTTP response body to a file-object. Optionally enforcing a maximum file size. + Args: + response: The HTTP response to read from. + stream: The file-object to write to. + max_size: The maximum file size to allow. + + Returns: + A Deferred which resolves to the length of the read body. + """ -def _readBodyToFile(response, stream, max_size): d = defer.Deferred() response.deliverBody(_ReadBodyToFileProtocol(stream, d, max_size)) return d -def encode_urlencode_args(args): - return {k: encode_urlencode_arg(v) for k, v in args.items()} +def encode_query_args(args: Optional[Mapping[str, Union[str, List[str]]]]) -> bytes: + """ + Encodes a map of query arguments to bytes which can be appended to a URL. + Args: + args: The query arguments, a mapping of string to string or list of strings. -def encode_urlencode_arg(arg): - if isinstance(arg, str): - return arg.encode("utf-8") - elif isinstance(arg, list): - return [encode_urlencode_arg(i) for i in arg] - else: - return arg + Returns: + The query arguments encoded as bytes. + """ + if args is None: + return b"" + encoded_args = {} + for k, vs in args.items(): + if isinstance(vs, str): + vs = [vs] + encoded_args[k] = [v.encode("utf8") for v in vs] -def _print_ex(e): - if hasattr(e, "reasons") and e.reasons: - for ex in e.reasons: - _print_ex(ex) - else: - logger.exception(e) + query_str = urllib.parse.urlencode(encoded_args, True) + + return query_str.encode("utf8") class InsecureInterceptableContextFactory(ssl.ContextFactory): diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b2ccae90df..4e27f93b7a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -19,7 +19,7 @@ import random import sys import urllib.parse from io import BytesIO -from typing import BinaryIO, Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, List, Optional, Tuple, Union import attr import treq @@ -28,26 +28,27 @@ from prometheus_client import Counter from signedjson.sign import sign_json from zope.interface import implementer -from twisted.internet import defer, protocol +from twisted.internet import defer from twisted.internet.error import DNSLookupError from twisted.internet.interfaces import IReactorPluggableNameResolver, IReactorTime from twisted.internet.task import _EPSILON, Cooperator -from twisted.python.failure import Failure -from twisted.web._newclient import ResponseDone from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer, IResponse import synapse.metrics import synapse.util.retryutils from synapse.api.errors import ( - Codes, FederationDeniedError, HttpResponseException, RequestSendFailed, - SynapseError, ) from synapse.http import QuieterFileBodyProducer -from synapse.http.client import BlacklistingAgentWrapper, IPBlacklistingResolver +from synapse.http.client import ( + BlacklistingAgentWrapper, + IPBlacklistingResolver, + encode_query_args, + readBodyToFile, +) from synapse.http.federation.matrix_federation_agent import MatrixFederationAgent from synapse.logging.context import make_deferred_yieldable from synapse.logging.opentracing import ( @@ -250,9 +251,7 @@ class MatrixFederationHttpClient: # Use a BlacklistingAgentWrapper to prevent circumventing the IP # blacklist via IP literals in server names self.agent = BlacklistingAgentWrapper( - self.agent, - self.reactor, - ip_blacklist=hs.config.federation_ip_range_blacklist, + self.agent, ip_blacklist=hs.config.federation_ip_range_blacklist, ) self.clock = hs.get_clock() @@ -986,7 +985,7 @@ class MatrixFederationHttpClient: headers = dict(response.headers.getAllRawHeaders()) try: - d = _readBodyToFile(response, output_stream, max_size) + d = readBodyToFile(response, output_stream, max_size) d.addTimeout(self.default_timeout, self.reactor) length = await make_deferred_yieldable(d) except Exception as e: @@ -1010,44 +1009,6 @@ class MatrixFederationHttpClient: return (length, headers) -class _ReadBodyToFileProtocol(protocol.Protocol): - def __init__( - self, stream: BinaryIO, deferred: defer.Deferred, max_size: Optional[int] - ): - self.stream = stream - self.deferred = deferred - self.length = 0 - self.max_size = max_size - - def dataReceived(self, data: bytes) -> None: - self.stream.write(data) - self.length += len(data) - if self.max_size is not None and self.length >= self.max_size: - self.deferred.errback( - SynapseError( - 502, - "Requested file is too large > %r bytes" % (self.max_size,), - Codes.TOO_LARGE, - ) - ) - self.deferred = defer.Deferred() - self.transport.loseConnection() - - def connectionLost(self, reason: Failure) -> None: - if reason.check(ResponseDone): - self.deferred.callback(self.length) - else: - self.deferred.errback(reason) - - -def _readBodyToFile( - response: IResponse, stream: BinaryIO, max_size: Optional[int] -) -> defer.Deferred: - d = defer.Deferred() - response.deliverBody(_ReadBodyToFileProtocol(stream, d, max_size)) - return d - - def _flatten_response_never_received(e): if hasattr(e, "reasons"): reasons = ", ".join( @@ -1088,18 +1049,3 @@ def check_content_type_is_json(headers: Headers) -> None: ), can_retry=False, ) - - -def encode_query_args(args: Optional[QueryArgs]) -> bytes: - if args is None: - return b"" - - encoded_args = {} - for k, vs in args.items(): - if isinstance(vs, str): - vs = [vs] - encoded_args[k] = [v.encode("utf8") for v in vs] - - query_str = urllib.parse.urlencode(encoded_args, True) - - return query_str.encode("utf8") From d963c69ba56ea45276ec3d11d191a20e8a38881d Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 25 Nov 2020 20:06:13 +0000 Subject: [PATCH 51/57] Speed up remote invite rejection database call (#8815) This is another PR that grew out of #6739. The existing code for checking whether a user is currently invited to a room when they want to leave the room looks like the following: https://github.com/matrix-org/synapse/blob/f737368a26bb9eea401fcc3a5bdd7e0b59e91f09/synapse/handlers/room_member.py#L518-L540 It calls `get_invite_for_local_user_in_room`, which will actually query *all* rooms the user has been invited to, before iterating over them and matching via the room ID. It will then return a tuple of a lot of information which we pull the event ID out of. I need to do a similar check for knocking, but this code wasn't very efficient. I then tried to write a different implementation using `StateHandler.get_current_state` but this actually didn't work as we haven't *joined* the room yet - we've only been invited to it. That means that only certain tables in Synapse have our desired `invite` membership state. One of those tables is `local_current_membership`. So I wrote a store method that just queries that table instead --- changelog.d/8815.misc | 1 + synapse/handlers/room_member.py | 16 ++++++--- synapse/storage/databases/main/roommember.py | 34 +++++++++++++++++++- 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 changelog.d/8815.misc diff --git a/changelog.d/8815.misc b/changelog.d/8815.misc new file mode 100644 index 0000000000..647edeb568 --- /dev/null +++ b/changelog.d/8815.misc @@ -0,0 +1 @@ +Optimise the lookup for an invite from another homeserver when trying to reject it. \ No newline at end of file diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 70f8966267..13a793c05a 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -31,7 +31,6 @@ from synapse.api.errors import ( from synapse.api.ratelimiting import Ratelimiter from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.storage.roommember import RoomsForUser from synapse.types import JsonDict, Requester, RoomAlias, RoomID, StateMap, UserID from synapse.util.async_helpers import Linearizer from synapse.util.distributor import user_left_room @@ -515,10 +514,16 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): elif effective_membership_state == Membership.LEAVE: if not is_host_in_room: # perhaps we've been invited - invite = await self.store.get_invite_for_local_user_in_room( - user_id=target.to_string(), room_id=room_id - ) # type: Optional[RoomsForUser] - if not invite: + ( + current_membership_type, + current_membership_event_id, + ) = await self.store.get_local_current_membership_for_user_in_room( + target.to_string(), room_id + ) + if ( + current_membership_type != Membership.INVITE + or not current_membership_event_id + ): logger.info( "%s sent a leave request to %s, but that is not an active room " "on this server, and there is no pending invite", @@ -528,6 +533,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): raise SynapseError(404, "Not a known room") + invite = await self.store.get_event(current_membership_event_id) logger.info( "%s rejects invite to %s from %s", target, room_id, invite.sender ) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 01d9dbb36f..dcdaf09682 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import TYPE_CHECKING, Dict, FrozenSet, Iterable, List, Optional, Set +from typing import TYPE_CHECKING, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple from synapse.api.constants import EventTypes, Membership from synapse.events import EventBase @@ -350,6 +350,38 @@ class RoomMemberWorkerStore(EventsWorkerStore): return results + async def get_local_current_membership_for_user_in_room( + self, user_id: str, room_id: str + ) -> Tuple[Optional[str], Optional[str]]: + """Retrieve the current local membership state and event ID for a user in a room. + + Args: + user_id: The ID of the user. + room_id: The ID of the room. + + Returns: + A tuple of (membership_type, event_id). Both will be None if a + room_id/user_id pair is not found. + """ + # Paranoia check. + if not self.hs.is_mine_id(user_id): + raise Exception( + "Cannot call 'get_local_current_membership_for_user_in_room' on " + "non-local user %s" % (user_id,), + ) + + results_dict = await self.db_pool.simple_select_one( + "local_current_membership", + {"room_id": room_id, "user_id": user_id}, + ("membership", "event_id"), + allow_none=True, + desc="get_local_current_membership_for_user_in_room", + ) + if not results_dict: + return None, None + + return results_dict.get("membership"), results_dict.get("event_id") + @cached(max_entries=500000, iterable=True) async def get_rooms_for_user_with_stream_ordering( self, user_id: str From 2b110dda2a66f8bcd69b68411ebab94c9e2593c2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 25 Nov 2020 21:02:53 +0000 Subject: [PATCH 52/57] Fix the formatting of push config section (#8818) This PR updates the push config's formatting to better align with our [code style guidelines](https://github.com/matrix-org/synapse/blob/develop/docs/code_style.md#configuration-file-format). --- changelog.d/8818.doc | 1 + docs/sample_config.yaml | 33 +++++++++++++++++++-------------- synapse/config/push.py | 35 ++++++++++++++++++++--------------- 3 files changed, 40 insertions(+), 29 deletions(-) create mode 100644 changelog.d/8818.doc diff --git a/changelog.d/8818.doc b/changelog.d/8818.doc new file mode 100644 index 0000000000..571b0e3f60 --- /dev/null +++ b/changelog.d/8818.doc @@ -0,0 +1 @@ +Update the formatting of the `push` section of the homeserver config file to better align with the [code style guidelines](https://github.com/matrix-org/synapse/blob/develop/docs/code_style.md#configuration-file-format). \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 52a1d8b853..df0f3e1d8e 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -2251,20 +2251,25 @@ password_providers: -# Clients requesting push notifications can either have the body of -# the message sent in the notification poke along with other details -# like the sender, or just the event ID and room ID (`event_id_only`). -# If clients choose the former, this option controls whether the -# notification request includes the content of the event (other details -# like the sender are still included). For `event_id_only` push, it -# has no effect. -# -# For modern android devices the notification content will still appear -# because it is loaded by the app. iPhone, however will send a -# notification saying only that a message arrived and who it came from. -# -#push: -# include_content: true +## Push ## + +push: + # Clients requesting push notifications can either have the body of + # the message sent in the notification poke along with other details + # like the sender, or just the event ID and room ID (`event_id_only`). + # If clients choose the former, this option controls whether the + # notification request includes the content of the event (other details + # like the sender are still included). For `event_id_only` push, it + # has no effect. + # + # For modern android devices the notification content will still appear + # because it is loaded by the app. iPhone, however will send a + # notification saying only that a message arrived and who it came from. + # + # The default value is "true" to include message details. Uncomment to only + # include the event ID and room ID in push notification payloads. + # + #include_content: false # Spam checkers are third-party modules that can block specific actions diff --git a/synapse/config/push.py b/synapse/config/push.py index a1f3752c8a..a71baac89c 100644 --- a/synapse/config/push.py +++ b/synapse/config/push.py @@ -21,7 +21,7 @@ class PushConfig(Config): section = "push" def read_config(self, config, **kwargs): - push_config = config.get("push", {}) + push_config = config.get("push") or {} self.push_include_content = push_config.get("include_content", True) pusher_instances = config.get("pusher_instances") or [] @@ -49,18 +49,23 @@ class PushConfig(Config): def generate_config_section(self, config_dir_path, server_name, **kwargs): return """ - # Clients requesting push notifications can either have the body of - # the message sent in the notification poke along with other details - # like the sender, or just the event ID and room ID (`event_id_only`). - # If clients choose the former, this option controls whether the - # notification request includes the content of the event (other details - # like the sender are still included). For `event_id_only` push, it - # has no effect. - # - # For modern android devices the notification content will still appear - # because it is loaded by the app. iPhone, however will send a - # notification saying only that a message arrived and who it came from. - # - #push: - # include_content: true + ## Push ## + + push: + # Clients requesting push notifications can either have the body of + # the message sent in the notification poke along with other details + # like the sender, or just the event ID and room ID (`event_id_only`). + # If clients choose the former, this option controls whether the + # notification request includes the content of the event (other details + # like the sender are still included). For `event_id_only` push, it + # has no effect. + # + # For modern android devices the notification content will still appear + # because it is loaded by the app. iPhone, however will send a + # notification saying only that a message arrived and who it came from. + # + # The default value is "true" to include message details. Uncomment to only + # include the event ID and room ID in push notification payloads. + # + #include_content: false """ From 3f0ff53158cc07b481c701077357d9d09254845b Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Wed, 25 Nov 2020 22:26:11 +0100 Subject: [PATCH 53/57] Remove deprecated `/_matrix/client/*/admin` endpoints (#8785) These are now only available via `/_synapse/admin/v1`. --- UPGRADE.rst | 22 ++++ changelog.d/8785.removal | 1 + docs/admin_api/user_admin_api.rst | 7 ++ synapse/_scripts/register_new_matrix_user.py | 2 +- synapse/rest/admin/__init__.py | 12 +- synapse/rest/admin/_base.py | 22 ---- synapse/rest/admin/groups.py | 7 +- synapse/rest/admin/media.py | 15 +-- synapse/rest/admin/rooms.py | 3 +- synapse/rest/admin/users.py | 25 +++-- tests/rest/admin/test_admin.py | 2 +- tests/rest/admin/test_room.py | 4 +- tests/rest/admin/test_user.py | 110 ++++++++++++++++++- tests/rest/client/v2_alpha/test_register.py | 6 +- tests/storage/test_client_ips.py | 2 +- tests/unittest.py | 4 +- 16 files changed, 176 insertions(+), 68 deletions(-) create mode 100644 changelog.d/8785.removal diff --git a/UPGRADE.rst b/UPGRADE.rst index 4de1bb5841..6825b567e9 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -105,6 +105,28 @@ shown below: return {"localpart": localpart} +Removal historical Synapse Admin API +------------------------------------ + +Historically, the Synapse Admin API has been accessible under: + +* ``/_matrix/client/api/v1/admin`` +* ``/_matrix/client/unstable/admin`` +* ``/_matrix/client/r0/admin`` +* ``/_synapse/admin/v1`` + +The endpoints with ``/_matrix/client/*`` prefixes have been removed as of v1.24.0. +The Admin API is now only accessible under: + +* ``/_synapse/admin/v1`` + +The only exception is the `/admin/whois` endpoint, which is +`also available via the client-server API `_. + +The deprecation of the old endpoints was announced with Synapse 1.20.0 (released +on 2020-09-22) and makes it easier for homeserver admins to lock down external +access to the Admin API endpoints. + Upgrading to v1.23.0 ==================== diff --git a/changelog.d/8785.removal b/changelog.d/8785.removal new file mode 100644 index 0000000000..ee8ee32598 --- /dev/null +++ b/changelog.d/8785.removal @@ -0,0 +1 @@ +Remove old `/_matrix/client/*/admin` endpoints which was deprecated since Synapse 1.20.0. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.rst b/docs/admin_api/user_admin_api.rst index 84863296e3..1473a3d4e3 100644 --- a/docs/admin_api/user_admin_api.rst +++ b/docs/admin_api/user_admin_api.rst @@ -176,6 +176,13 @@ The api is:: GET /_synapse/admin/v1/whois/ +and:: + + GET /_matrix/client/r0/admin/whois/ + +See also: `Client Server API Whois +`_ + To use it, you will need to authenticate by providing an ``access_token`` for a server admin: see `README.rst `_. diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py index da0996edbc..d37ccccd5b 100644 --- a/synapse/_scripts/register_new_matrix_user.py +++ b/synapse/_scripts/register_new_matrix_user.py @@ -37,7 +37,7 @@ def request_registration( exit=sys.exit, ): - url = "%s/_matrix/client/r0/admin/register" % (server_location,) + url = "%s/_synapse/admin/v1/register" % (server_location,) # Get the nonce r = requests.get(url, verify=False) diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 7a3a5c46ca..55ddebb4fe 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -21,11 +21,7 @@ import synapse from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.server import JsonResource from synapse.http.servlet import RestServlet, parse_json_object_from_request -from synapse.rest.admin._base import ( - admin_patterns, - assert_requester_is_admin, - historical_admin_path_patterns, -) +from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin from synapse.rest.admin.devices import ( DeleteDevicesRestServlet, DeviceRestServlet, @@ -84,7 +80,7 @@ class VersionServlet(RestServlet): class PurgeHistoryRestServlet(RestServlet): - PATTERNS = historical_admin_path_patterns( + PATTERNS = admin_patterns( "/purge_history/(?P[^/]*)(/(?P[^/]+))?" ) @@ -169,9 +165,7 @@ class PurgeHistoryRestServlet(RestServlet): class PurgeHistoryStatusRestServlet(RestServlet): - PATTERNS = historical_admin_path_patterns( - "/purge_history_status/(?P[^/]+)" - ) + PATTERNS = admin_patterns("/purge_history_status/(?P[^/]+)") def __init__(self, hs): """ diff --git a/synapse/rest/admin/_base.py b/synapse/rest/admin/_base.py index db9fea263a..e09234c644 100644 --- a/synapse/rest/admin/_base.py +++ b/synapse/rest/admin/_base.py @@ -22,28 +22,6 @@ from synapse.api.errors import AuthError from synapse.types import UserID -def historical_admin_path_patterns(path_regex): - """Returns the list of patterns for an admin endpoint, including historical ones - - This is a backwards-compatibility hack. Previously, the Admin API was exposed at - various paths under /_matrix/client. This function returns a list of patterns - matching those paths (as well as the new one), so that existing scripts which rely - on the endpoints being available there are not broken. - - Note that this should only be used for existing endpoints: new ones should just - register for the /_synapse/admin path. - """ - return [ - re.compile(prefix + path_regex) - for prefix in ( - "^/_synapse/admin/v1", - "^/_matrix/client/api/v1/admin", - "^/_matrix/client/unstable/admin", - "^/_matrix/client/r0/admin", - ) - ] - - def admin_patterns(path_regex: str, version: str = "v1"): """Returns the list of patterns for an admin endpoint diff --git a/synapse/rest/admin/groups.py b/synapse/rest/admin/groups.py index 0b54ca09f4..d0c86b204a 100644 --- a/synapse/rest/admin/groups.py +++ b/synapse/rest/admin/groups.py @@ -16,10 +16,7 @@ import logging from synapse.api.errors import SynapseError from synapse.http.servlet import RestServlet -from synapse.rest.admin._base import ( - assert_user_is_admin, - historical_admin_path_patterns, -) +from synapse.rest.admin._base import admin_patterns, assert_user_is_admin logger = logging.getLogger(__name__) @@ -28,7 +25,7 @@ class DeleteGroupAdminRestServlet(RestServlet): """Allows deleting of local groups """ - PATTERNS = historical_admin_path_patterns("/delete_group/(?P[^/]*)") + PATTERNS = admin_patterns("/delete_group/(?P[^/]*)") def __init__(self, hs): self.group_server = hs.get_groups_server_handler() diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index ba50cb876d..c82b4f87d6 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -22,7 +22,6 @@ from synapse.rest.admin._base import ( admin_patterns, assert_requester_is_admin, assert_user_is_admin, - historical_admin_path_patterns, ) logger = logging.getLogger(__name__) @@ -34,10 +33,10 @@ class QuarantineMediaInRoom(RestServlet): """ PATTERNS = ( - historical_admin_path_patterns("/room/(?P[^/]+)/media/quarantine") + admin_patterns("/room/(?P[^/]+)/media/quarantine") + # This path kept around for legacy reasons - historical_admin_path_patterns("/quarantine_media/(?P[^/]+)") + admin_patterns("/quarantine_media/(?P[^/]+)") ) def __init__(self, hs): @@ -63,9 +62,7 @@ class QuarantineMediaByUser(RestServlet): this server. """ - PATTERNS = historical_admin_path_patterns( - "/user/(?P[^/]+)/media/quarantine" - ) + PATTERNS = admin_patterns("/user/(?P[^/]+)/media/quarantine") def __init__(self, hs): self.store = hs.get_datastore() @@ -90,7 +87,7 @@ class QuarantineMediaByID(RestServlet): it via this server. """ - PATTERNS = historical_admin_path_patterns( + PATTERNS = admin_patterns( "/media/quarantine/(?P[^/]+)/(?P[^/]+)" ) @@ -116,7 +113,7 @@ class ListMediaInRoom(RestServlet): """Lists all of the media in a given room. """ - PATTERNS = historical_admin_path_patterns("/room/(?P[^/]+)/media") + PATTERNS = admin_patterns("/room/(?P[^/]+)/media") def __init__(self, hs): self.store = hs.get_datastore() @@ -134,7 +131,7 @@ class ListMediaInRoom(RestServlet): class PurgeMediaCacheRestServlet(RestServlet): - PATTERNS = historical_admin_path_patterns("/purge_media_cache") + PATTERNS = admin_patterns("/purge_media_cache") def __init__(self, hs): self.media_repository = hs.get_media_repository() diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index ee345e12ce..353151169a 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -29,7 +29,6 @@ from synapse.rest.admin._base import ( admin_patterns, assert_requester_is_admin, assert_user_is_admin, - historical_admin_path_patterns, ) from synapse.storage.databases.main.room import RoomSortOrder from synapse.types import RoomAlias, RoomID, UserID, create_requester @@ -44,7 +43,7 @@ class ShutdownRoomRestServlet(RestServlet): joined to the new room. """ - PATTERNS = historical_admin_path_patterns("/shutdown_room/(?P[^/]+)") + PATTERNS = admin_patterns("/shutdown_room/(?P[^/]+)") def __init__(self, hs): self.hs = hs diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index fa8d8e6d91..b0ff5e1ead 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -33,8 +33,8 @@ from synapse.rest.admin._base import ( admin_patterns, assert_requester_is_admin, assert_user_is_admin, - historical_admin_path_patterns, ) +from synapse.rest.client.v2_alpha._base import client_patterns from synapse.types import JsonDict, UserID if TYPE_CHECKING: @@ -55,7 +55,7 @@ _GET_PUSHERS_ALLOWED_KEYS = { class UsersRestServlet(RestServlet): - PATTERNS = historical_admin_path_patterns("/users/(?P[^/]*)$") + PATTERNS = admin_patterns("/users/(?P[^/]*)$") def __init__(self, hs): self.hs = hs @@ -338,7 +338,7 @@ class UserRegisterServlet(RestServlet): nonce to the time it was generated, in int seconds. """ - PATTERNS = historical_admin_path_patterns("/register") + PATTERNS = admin_patterns("/register") NONCE_TIMEOUT = 60 def __init__(self, hs): @@ -461,7 +461,14 @@ class UserRegisterServlet(RestServlet): class WhoisRestServlet(RestServlet): - PATTERNS = historical_admin_path_patterns("/whois/(?P[^/]*)") + path_regex = "/whois/(?P[^/]*)$" + PATTERNS = ( + admin_patterns(path_regex) + + + # URL for spec reason + # https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid + client_patterns("/admin" + path_regex, v1=True) + ) def __init__(self, hs): self.hs = hs @@ -485,7 +492,7 @@ class WhoisRestServlet(RestServlet): class DeactivateAccountRestServlet(RestServlet): - PATTERNS = historical_admin_path_patterns("/deactivate/(?P[^/]*)") + PATTERNS = admin_patterns("/deactivate/(?P[^/]*)") def __init__(self, hs): self._deactivate_account_handler = hs.get_deactivate_account_handler() @@ -516,7 +523,7 @@ class DeactivateAccountRestServlet(RestServlet): class AccountValidityRenewServlet(RestServlet): - PATTERNS = historical_admin_path_patterns("/account_validity/validity$") + PATTERNS = admin_patterns("/account_validity/validity$") def __init__(self, hs): """ @@ -559,9 +566,7 @@ class ResetPasswordRestServlet(RestServlet): 200 OK with empty object if success otherwise an error. """ - PATTERNS = historical_admin_path_patterns( - "/reset_password/(?P[^/]*)" - ) + PATTERNS = admin_patterns("/reset_password/(?P[^/]*)") def __init__(self, hs): self.store = hs.get_datastore() @@ -603,7 +608,7 @@ class SearchUsersRestServlet(RestServlet): 200 OK with json object {list[dict[str, Any]], count} or empty object. """ - PATTERNS = historical_admin_path_patterns("/search_users/(?P[^/]*)") + PATTERNS = admin_patterns("/search_users/(?P[^/]*)") def __init__(self, hs): self.hs = hs diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 898e43411e..4f76f8f768 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -100,7 +100,7 @@ class DeleteGroupTestCase(unittest.HomeserverTestCase): self.assertIn(group_id, self._get_groups_user_is_in(self.other_user_token)) # Now delete the group - url = "/admin/delete_group/" + group_id + url = "/_synapse/admin/v1/delete_group/" + group_id request, channel = self.make_request( "POST", url.encode("ascii"), diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 54824a5410..46933a0493 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -78,7 +78,7 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase): ) # Test that the admin can still send shutdown - url = "admin/shutdown_room/" + room_id + url = "/_synapse/admin/v1/shutdown_room/" + room_id request, channel = self.make_request( "POST", url.encode("ascii"), @@ -112,7 +112,7 @@ class ShutdownRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) # Test that the admin can still send shutdown - url = "admin/shutdown_room/" + room_id + url = "/_synapse/admin/v1/shutdown_room/" + room_id request, channel = self.make_request( "POST", url.encode("ascii"), diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 9661af7e79..54d46f4bd3 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -41,7 +41,7 @@ class UserRegisterTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): - self.url = "/_matrix/client/r0/admin/register" + self.url = "/_synapse/admin/v1/register" self.registration_handler = Mock() self.identity_handler = Mock() @@ -1768,3 +1768,111 @@ class UserTokenRestTestCase(unittest.HomeserverTestCase): # though the MAU limit would stop the user doing so. puppet_token = self._get_token() self.helper.join(room_id, user=self.other_user, tok=puppet_token) + + +class WhoisRestTestCase(unittest.HomeserverTestCase): + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor, clock, hs): + self.store = hs.get_datastore() + + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.other_user = self.register_user("user", "pass") + self.url1 = "/_synapse/admin/v1/whois/%s" % urllib.parse.quote(self.other_user) + self.url2 = "/_matrix/client/r0/admin/whois/%s" % urllib.parse.quote( + self.other_user + ) + + def test_no_auth(self): + """ + Try to get information of an user without authentication. + """ + request, channel = self.make_request("GET", self.url1, b"{}") + self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + request, channel = self.make_request("GET", self.url2, b"{}") + self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_not_admin(self): + """ + If the user is not a server admin, an error is returned. + """ + self.register_user("user2", "pass") + other_user2_token = self.login("user2", "pass") + + request, channel = self.make_request( + "GET", self.url1, access_token=other_user2_token, + ) + self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + request, channel = self.make_request( + "GET", self.url2, access_token=other_user2_token, + ) + self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_user_is_not_local(self): + """ + Tests that a lookup for a user that is not a local returns a 400 + """ + url1 = "/_synapse/admin/v1/whois/@unknown_person:unknown_domain" + url2 = "/_matrix/client/r0/admin/whois/@unknown_person:unknown_domain" + + request, channel = self.make_request( + "GET", url1, access_token=self.admin_user_tok, + ) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual("Can only whois a local user", channel.json_body["error"]) + + request, channel = self.make_request( + "GET", url2, access_token=self.admin_user_tok, + ) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual("Can only whois a local user", channel.json_body["error"]) + + def test_get_whois_admin(self): + """ + The lookup should succeed for an admin. + """ + request, channel = self.make_request( + "GET", self.url1, access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(self.other_user, channel.json_body["user_id"]) + self.assertIn("devices", channel.json_body) + + request, channel = self.make_request( + "GET", self.url2, access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(self.other_user, channel.json_body["user_id"]) + self.assertIn("devices", channel.json_body) + + def test_get_whois_user(self): + """ + The lookup should succeed for a normal user looking up their own information. + """ + other_user_token = self.login("user", "pass") + + request, channel = self.make_request( + "GET", self.url1, access_token=other_user_token, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(self.other_user, channel.json_body["user_id"]) + self.assertIn("devices", channel.json_body) + + request, channel = self.make_request( + "GET", self.url2, access_token=other_user_token, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(self.other_user, channel.json_body["user_id"]) + self.assertIn("devices", channel.json_body) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 88923fcea4..699a40c3df 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -342,7 +342,7 @@ class AccountValidityTestCase(unittest.HomeserverTestCase): self.register_user("admin", "adminpassword", admin=True) admin_tok = self.login("admin", "adminpassword") - url = "/_matrix/client/unstable/admin/account_validity/validity" + url = "/_synapse/admin/v1/account_validity/validity" params = {"user_id": user_id} request_data = json.dumps(params) request, channel = self.make_request( @@ -362,7 +362,7 @@ class AccountValidityTestCase(unittest.HomeserverTestCase): self.register_user("admin", "adminpassword", admin=True) admin_tok = self.login("admin", "adminpassword") - url = "/_matrix/client/unstable/admin/account_validity/validity" + url = "/_synapse/admin/v1/account_validity/validity" params = { "user_id": user_id, "expiration_ts": 0, @@ -389,7 +389,7 @@ class AccountValidityTestCase(unittest.HomeserverTestCase): self.register_user("admin", "adminpassword", admin=True) admin_tok = self.login("admin", "adminpassword") - url = "/_matrix/client/unstable/admin/account_validity/validity" + url = "/_synapse/admin/v1/account_validity/validity" params = { "user_id": user_id, "expiration_ts": 0, diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index 6bdde1a2ba..a69117c5a9 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -416,7 +416,7 @@ class ClientIpAuthTestCase(unittest.HomeserverTestCase): self.reactor, self.site, "GET", - "/_matrix/client/r0/admin/users/" + self.user_id, + "/_synapse/admin/v1/users/" + self.user_id, access_token=access_token, custom_headers=headers1.items(), **make_request_args, diff --git a/tests/unittest.py b/tests/unittest.py index c7c889c405..a9d59e31f7 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -554,7 +554,7 @@ class HomeserverTestCase(TestCase): self.hs.config.registration_shared_secret = "shared" # Create the user - request, channel = self.make_request("GET", "/_matrix/client/r0/admin/register") + request, channel = self.make_request("GET", "/_synapse/admin/v1/register") self.assertEqual(channel.code, 200, msg=channel.result) nonce = channel.json_body["nonce"] @@ -580,7 +580,7 @@ class HomeserverTestCase(TestCase): } ) request, channel = self.make_request( - "POST", "/_matrix/client/r0/admin/register", body.encode("utf8") + "POST", "/_synapse/admin/v1/register", body.encode("utf8") ) self.assertEqual(channel.code, 200, channel.json_body) From 14f81a6d242204229934c9b7bff0ec6efb09c840 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 26 Nov 2020 11:42:55 +0100 Subject: [PATCH 54/57] Improve documentation how to configure prometheus for workers (#8822) --- changelog.d/8822.doc | 1 + contrib/prometheus/README.md | 10 +++-- docs/metrics-howto.md | 74 +++++++++++++++++++++++++++--------- 3 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 changelog.d/8822.doc diff --git a/changelog.d/8822.doc b/changelog.d/8822.doc new file mode 100644 index 0000000000..4299245990 --- /dev/null +++ b/changelog.d/8822.doc @@ -0,0 +1 @@ +Improve documentation how to configure prometheus for workers. \ No newline at end of file diff --git a/contrib/prometheus/README.md b/contrib/prometheus/README.md index e646cb7ea7..b3f23bcc80 100644 --- a/contrib/prometheus/README.md +++ b/contrib/prometheus/README.md @@ -20,6 +20,7 @@ Add a new job to the main prometheus.conf file: ``` ### for Prometheus v2 + Add a new job to the main prometheus.yml file: ```yaml @@ -29,14 +30,17 @@ Add a new job to the main prometheus.yml file: scheme: "https" static_configs: - - targets: ['SERVER.LOCATION:PORT'] + - targets: ["my.server.here:port"] ``` +An example of a Prometheus configuration with workers can be found in +[metrics-howto.md](https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md). + To use `synapse.rules` add ```yaml - rule_files: - - "/PATH/TO/synapse-v2.rules" + rule_files: + - "/PATH/TO/synapse-v2.rules" ``` Metrics are disabled by default when running synapse; they must be enabled diff --git a/docs/metrics-howto.md b/docs/metrics-howto.md index fb71af4911..6b84153274 100644 --- a/docs/metrics-howto.md +++ b/docs/metrics-howto.md @@ -13,10 +13,12 @@ can be enabled by adding the \"metrics\" resource to the existing listener as such: - resources: - - names: - - client - - metrics + ```yaml + resources: + - names: + - client + - metrics + ``` This provides a simple way of adding metrics to your Synapse installation, and serves under `/_synapse/metrics`. If you do not @@ -31,11 +33,13 @@ Add a new listener to homeserver.yaml: - listeners: - - type: metrics - port: 9000 - bind_addresses: - - '0.0.0.0' + ```yaml + listeners: + - type: metrics + port: 9000 + bind_addresses: + - '0.0.0.0' + ``` For both options, you will need to ensure that `enable_metrics` is set to `True`. @@ -47,10 +51,13 @@ It needs to set the `metrics_path` to a non-default value (under `scrape_configs`): - - job_name: "synapse" - metrics_path: "/_synapse/metrics" - static_configs: - - targets: ["my.server.here:port"] + ```yaml + - job_name: "synapse" + scrape_interval: 15s + metrics_path: "/_synapse/metrics" + static_configs: + - targets: ["my.server.here:port"] + ``` where `my.server.here` is the IP address of Synapse, and `port` is the listener port configured with the `metrics` resource. @@ -60,7 +67,8 @@ 1. Restart Prometheus. -1. Consider using the [grafana dashboard](https://github.com/matrix-org/synapse/tree/master/contrib/grafana/) and required [recording rules](https://github.com/matrix-org/synapse/tree/master/contrib/prometheus/) +1. Consider using the [grafana dashboard](https://github.com/matrix-org/synapse/tree/master/contrib/grafana/) + and required [recording rules](https://github.com/matrix-org/synapse/tree/master/contrib/prometheus/) ## Monitoring workers @@ -76,9 +84,9 @@ To allow collecting metrics from a worker, you need to add a under `worker_listeners`: ```yaml - - type: metrics - bind_address: '' - port: 9101 + - type: metrics + bind_address: '' + port: 9101 ``` The `bind_address` and `port` parameters should be set so that @@ -87,6 +95,38 @@ don't clash with an existing worker. With this example, the worker's metrics would then be available on `http://127.0.0.1:9101`. +Example Prometheus target for Synapse with workers: + +```yaml + - job_name: "synapse" + scrape_interval: 15s + metrics_path: "/_synapse/metrics" + static_configs: + - targets: ["my.server.here:port"] + labels: + instance: "my.server" + job: "master" + index: 1 + - targets: ["my.workerserver.here:port"] + labels: + instance: "my.server" + job: "generic_worker" + index: 1 + - targets: ["my.workerserver.here:port"] + labels: + instance: "my.server" + job: "generic_worker" + index: 2 + - targets: ["my.workerserver.here:port"] + labels: + instance: "my.server" + job: "media_repository" + index: 1 +``` + +Labels (`instance`, `job`, `index`) can be defined as anything. +The labels are used to group graphs in grafana. + ## Renaming of metrics & deprecation of old names in 1.2 Synapse 1.2 updates the Prometheus metrics to match the naming From 7c4344747709e9a03e96f85f96affd5faa22e0ee Mon Sep 17 00:00:00 2001 From: Dmitry Borodaenko Date: Thu, 26 Nov 2020 02:57:26 -0800 Subject: [PATCH 55/57] Strip trailing / from server_url in register_new_matrix_user (#8823) When server URL provided to register_new_matrix_user includes path component (e.g. "http://localhost:8008/"), the command fails with "ERROR! Received 400 Bad Request". Stripping trailing slash from the server_url command argument makes sure combined endpoint URL remains valid. Signed-off-by: Dmitry Borodaenko angdraug@debian.org --- changelog.d/8823.bugfix | 1 + synapse/_scripts/register_new_matrix_user.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/8823.bugfix diff --git a/changelog.d/8823.bugfix b/changelog.d/8823.bugfix new file mode 100644 index 0000000000..74af1c20b6 --- /dev/null +++ b/changelog.d/8823.bugfix @@ -0,0 +1 @@ +Fix `register_new_matrix_user` failing with "Bad Request" when trailing slash is included in server URL. Contributed by @angdraug. diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py index d37ccccd5b..dfe26dea6d 100644 --- a/synapse/_scripts/register_new_matrix_user.py +++ b/synapse/_scripts/register_new_matrix_user.py @@ -37,7 +37,7 @@ def request_registration( exit=sys.exit, ): - url = "%s/_synapse/admin/v1/register" % (server_location,) + url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),) # Get the nonce r = requests.get(url, verify=False) From 382b4e83f1e5f4c9c1e233193241db8eb6e6ffa4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 26 Nov 2020 11:18:10 +0000 Subject: [PATCH 56/57] Defer SIGHUP handlers to reactor. (#8817) We can get a SIGHUP at any point, including times where we are not in a sane state. By deferring calling the handlers until the next reactor tick we ensure that we don't get unexpected conflicts, e.g. trying to flush logs from the signal handler while the code was in the process of writing a log entry. Fixes #8769. --- changelog.d/8817.bugfix | 1 + synapse/app/_base.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 changelog.d/8817.bugfix diff --git a/changelog.d/8817.bugfix b/changelog.d/8817.bugfix new file mode 100644 index 0000000000..e45dbd2ba4 --- /dev/null +++ b/changelog.d/8817.bugfix @@ -0,0 +1 @@ +Fix bug where logging could break after a call to SIGHUP. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 9c8dc785c6..895b38ae76 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -32,6 +32,7 @@ from synapse.app.phone_stats_home import start_phone_stats_home from synapse.config.server import ListenerConfig from synapse.crypto import context_factory from synapse.logging.context import PreserveLoggingContext +from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.util.async_helpers import Linearizer from synapse.util.daemonize import daemonize_process from synapse.util.rlimit import change_resource_limit @@ -244,6 +245,7 @@ def start(hs: "synapse.server.HomeServer", listeners: Iterable[ListenerConfig]): # Set up the SIGHUP machinery. if hasattr(signal, "SIGHUP"): + @wrap_as_background_process("sighup") def handle_sighup(*args, **kwargs): # Tell systemd our state, if we're using it. This will silently fail if # we're not using systemd. @@ -254,7 +256,13 @@ def start(hs: "synapse.server.HomeServer", listeners: Iterable[ListenerConfig]): sdnotify(b"READY=1") - signal.signal(signal.SIGHUP, handle_sighup) + # We defer running the sighup handlers until next reactor tick. This + # is so that we're in a sane state, e.g. flushing the logs may fail + # if the sighup happens in the middle of writing a log entry. + def run_sighup(*args, **kwargs): + hs.get_clock().call_later(0, handle_sighup, *args, **kwargs) + + signal.signal(signal.SIGHUP, run_sighup) register_sighup(refresh_certificate, hs) From 1cd356765e20f7193813604cbbec94cc88668cb2 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 26 Nov 2020 18:41:20 +0100 Subject: [PATCH 57/57] Update example prometheus console (#8824) Signed-off-by: Dirk Klimpel dirk@klimpel.org --- changelog.d/8824.doc | 1 + contrib/prometheus/consoles/synapse.html | 101 +++++++++++------------ 2 files changed, 51 insertions(+), 51 deletions(-) create mode 100644 changelog.d/8824.doc diff --git a/changelog.d/8824.doc b/changelog.d/8824.doc new file mode 100644 index 0000000000..683b436328 --- /dev/null +++ b/changelog.d/8824.doc @@ -0,0 +1 @@ +Update example prometheus console. \ No newline at end of file diff --git a/contrib/prometheus/consoles/synapse.html b/contrib/prometheus/consoles/synapse.html index 69aa87f85e..cd9ad15231 100644 --- a/contrib/prometheus/consoles/synapse.html +++ b/contrib/prometheus/consoles/synapse.html @@ -9,7 +9,7 @@ new PromConsole.Graph({ node: document.querySelector("#process_resource_utime"), expr: "rate(process_cpu_seconds_total[2m]) * 100", - name: "[[job]]", + name: "[[job]]-[[index]]", min: 0, max: 100, renderer: "line", @@ -22,12 +22,12 @@ new PromConsole.Graph({

Memory

-
+
@@ -115,7 +115,7 @@ new PromConsole.Graph({

Transaction execution time

-
+
-

Database scheduling latency

-
+

Average time waiting for database connection

+
-

Cache hit ratio

-
+

Cache request rate

+
@@ -191,7 +190,7 @@ new PromConsole.Graph({ new PromConsole.Graph({ node: document.querySelector("#synapse_cache_size"), expr: "synapse_util_caches_cache:size", - name: "[[name]]", + name: "[[job]]-[[index]] [[name]]", yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, yUnits: "", @@ -206,8 +205,8 @@ new PromConsole.Graph({