diff --git a/.circleci/config.yml b/.circleci/config.yml index 5395028426..d4aafd4b2c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,8 +4,8 @@ jobs: machine: true steps: - checkout - - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG} . - - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 . + - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} . + - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 . - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - run: docker push matrixdotorg/synapse:${CIRCLE_TAG} - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3 @@ -13,13 +13,9 @@ jobs: machine: true steps: - checkout - - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1} . - - run: docker build -f docker/Dockerfile -t matrixdotorg/synapse:${CIRCLE_SHA1}-py3 --build-arg PYTHON_VERSION=3.6 . + - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_SHA1} . + - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_SHA1}-py3 --build-arg PYTHON_VERSION=3.6 . - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1} matrixdotorg/synapse:latest - - run: docker tag matrixdotorg/synapse:${CIRCLE_SHA1}-py3 matrixdotorg/synapse:latest-py3 - - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1} - - run: docker push matrixdotorg/synapse:${CIRCLE_SHA1}-py3 - run: docker push matrixdotorg/synapse:latest - run: docker push matrixdotorg/synapse:latest-py3 sytestpy2: diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..3edf9e717c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +# EditorConfig https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# 4 space indentation +[*.py] +indent_style = space +indent_size = 4 diff --git a/.travis.yml b/.travis.yml index 655fab9d8e..84d5efff9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,24 +36,24 @@ matrix: env: TOX_ENV="pep8,check_isort" - python: 2.7 - env: TOX_ENV=py27 TRIAL_FLAGS="-j 2" + env: TOX_ENV=py27,codecov TRIAL_FLAGS="-j 2" - python: 2.7 env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2" - python: 2.7 - env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4" + env: TOX_ENV=py27-postgres,codecov TRIAL_FLAGS="-j 4" services: - postgresql - python: 3.5 - env: TOX_ENV=py35 TRIAL_FLAGS="-j 2" + env: TOX_ENV=py35,codecov TRIAL_FLAGS="-j 2" - python: 3.6 - env: TOX_ENV=py36 TRIAL_FLAGS="-j 2" + env: TOX_ENV=py36,codecov TRIAL_FLAGS="-j 2" - python: 3.6 - env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4" + env: TOX_ENV=py36-postgres,codecov TRIAL_FLAGS="-j 4" services: - postgresql diff --git a/CHANGES.md b/CHANGES.md index 1c3d575c37..bd420ba5ef 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,57 @@ +Synapse 0.34.0rc1 (2018-12-04) +============================== + +Synapse 0.34 is the first release to fully support Python 3. We recommend +upgrading to Python 3, but make sure to read the +[upgrade notes](UPGRADE.rst#upgrading-to-v0340) when doing so. + +Features +-------- + +- Add option to track MAU stats (but not limit people) ([\#3830](https://github.com/matrix-org/synapse/issues/3830)) +- Add an option to enable recording IPs for appservice users ([\#3831](https://github.com/matrix-org/synapse/issues/3831)) +- Rename login type m.login.cas to m.login.sso ([\#4220](https://github.com/matrix-org/synapse/issues/4220)) +- Add an option to disable search for homeservers that may not be interested in it. ([\#4230](https://github.com/matrix-org/synapse/issues/4230)) + + +Bugfixes +-------- + +- Pushrules can now again be made with non-ASCII rule IDs. ([\#4165](https://github.com/matrix-org/synapse/issues/4165)) +- The media repository now no longer fails to decode UTF-8 filenames when downloading remote media. ([\#4176](https://github.com/matrix-org/synapse/issues/4176)) +- URL previews now correctly decode non-UTF-8 text if the header contains a ``_). This bug - is now fixed, but means that federating with older servers can be unreliable + is now fixed, but means that federating with older servers can be unreliable when using name-based virtual hosting. Furthermore, a number of the normal reasons for using a reverse-proxy do not @@ -828,7 +797,7 @@ Password reset ============== If a user has registered an email address to their account using an identity -server, they can request a password-reset token via clients such as Vector. +server, they can request a password-reset token via clients such as Riot. A manual password reset can be done via direct database access as follows. diff --git a/UPGRADE.rst b/UPGRADE.rst index 55c77eedde..9d68a64058 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -48,6 +48,74 @@ returned by the Client-Server API: # configured on port 443. curl -kv https:///_matrix/client/versions 2>&1 | grep "Server:" +Upgrading to v0.34.0 +==================== + +1. This release is the first to fully support Python 3. We recommend switching + to Python 3, as it has been shown to give performance improvements. + + For users who have installed Synapse into a virtualenv, we recommend doing + this by creating a new virtualenv. For example:: + + virtualenv -p python3 ~/synapse/env3 + source ~/synapse/env3/bin/activate + pip install matrix-synapse + + You can then start synapse as normal, having activated the new virtualenv:: + + cd ~/synapse + source env3/bin/activate + synctl start + + Users who have installed from distribution packages should see the relevant + package documentation. + + * When upgrading to Python 3, you **must** make sure that your log files are + configured as UTF-8, by adding ``encoding: utf8`` to the + ``RotatingFileHandler`` configuration (if you have one) in your + ``.log.config`` file. For example, if your ``log.config`` file + contains:: + + handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: homeserver.log + maxBytes: 104857600 + backupCount: 10 + filters: [context] + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + + Then you should update this to be:: + + handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: homeserver.log + maxBytes: 104857600 + backupCount: 10 + filters: [context] + encoding: utf8 + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + + There is no need to revert this change if downgrading to Python 2. + +2. This release removes the ``riot.im`` from the default list of trusted + identity servers. + + If ``riot.im`` is in your homeserver's list of + ``trusted_third_party_id_servers``, you should remove it. It was added in + case a hypothetical future identity server was put there. If you don't + remove it, users may be unable to deactivate their accounts. + + Upgrading to v0.33.7 ==================== diff --git a/changelog.d/3831.feature b/changelog.d/3831.feature new file mode 100644 index 0000000000..6395586458 --- /dev/null +++ b/changelog.d/3831.feature @@ -0,0 +1 @@ +Add an option to enable recording IPs for appservice users diff --git a/changelog.d/4165.bugfix b/changelog.d/4165.bugfix new file mode 100644 index 0000000000..fe31c60683 --- /dev/null +++ b/changelog.d/4165.bugfix @@ -0,0 +1 @@ +Pushrules can now again be made with non-ASCII rule IDs. diff --git a/changelog.d/4189.misc b/changelog.d/4189.misc new file mode 100644 index 0000000000..4a41357d94 --- /dev/null +++ b/changelog.d/4189.misc @@ -0,0 +1,2 @@ +Run the AS senders as background processes to fix warnings + diff --git a/changelog.d/4190.misc b/changelog.d/4190.misc new file mode 100644 index 0000000000..6700a5150d --- /dev/null +++ b/changelog.d/4190.misc @@ -0,0 +1 @@ +Add some diagnostics to the tests to detect logcontext problems diff --git a/changelog.d/4205.misc b/changelog.d/4205.misc new file mode 100644 index 0000000000..bbdce2c7aa --- /dev/null +++ b/changelog.d/4205.misc @@ -0,0 +1 @@ +More logcontext checking in unittests diff --git a/changelog.d/4209.misc b/changelog.d/4209.misc new file mode 100644 index 0000000000..efd1f4abd6 --- /dev/null +++ b/changelog.d/4209.misc @@ -0,0 +1 @@ +Fix logcontext leaks in EmailPusher and in tests \ No newline at end of file diff --git a/changelog.d/4211.bugfix b/changelog.d/4211.bugfix new file mode 100644 index 0000000000..376f80c55a --- /dev/null +++ b/changelog.d/4211.bugfix @@ -0,0 +1,2 @@ +fix start up failure when mau_limit_reserved_threepids set and db is postgres + diff --git a/changelog.d/4214.misc b/changelog.d/4214.misc new file mode 100644 index 0000000000..b2f62060e3 --- /dev/null +++ b/changelog.d/4214.misc @@ -0,0 +1 @@ +Ignore __pycache__ directories in the database schema folder diff --git a/changelog.d/4220.feature b/changelog.d/4220.feature new file mode 100644 index 0000000000..e7a3e40483 --- /dev/null +++ b/changelog.d/4220.feature @@ -0,0 +1 @@ +Rename login type m.login.cas to m.login.sso diff --git a/changelog.d/4223.bugfix b/changelog.d/4223.bugfix new file mode 100644 index 0000000000..bab591a765 --- /dev/null +++ b/changelog.d/4223.bugfix @@ -0,0 +1 @@ +Fix auto join failures for servers that require user consent diff --git a/changelog.d/4224.misc b/changelog.d/4224.misc new file mode 100644 index 0000000000..1bfe2e5c31 --- /dev/null +++ b/changelog.d/4224.misc @@ -0,0 +1 @@ +Add note to UPGRADE.rst about removing riot.im from list of trusted identity servers diff --git a/changelog.d/4225.misc b/changelog.d/4225.misc new file mode 100644 index 0000000000..39062696ea --- /dev/null +++ b/changelog.d/4225.misc @@ -0,0 +1 @@ +Added automated coverage reporting to CI. diff --git a/changelog.d/4227.misc b/changelog.d/4227.misc new file mode 100644 index 0000000000..7ebd51b6a4 --- /dev/null +++ b/changelog.d/4227.misc @@ -0,0 +1 @@ +Garbage-collect after each unit test to fix logcontext leaks \ No newline at end of file diff --git a/changelog.d/4230.feature b/changelog.d/4230.feature new file mode 100644 index 0000000000..0ecb1d5ec6 --- /dev/null +++ b/changelog.d/4230.feature @@ -0,0 +1 @@ +Add an option to disable search for homeservers that may not be interested in it. diff --git a/changelog.d/4234.misc b/changelog.d/4234.misc new file mode 100644 index 0000000000..b5a01d38af --- /dev/null +++ b/changelog.d/4234.misc @@ -0,0 +1 @@ +add more detail to logging regarding "More than one row matched" error \ No newline at end of file diff --git a/changelog.d/4241.bugfix b/changelog.d/4241.bugfix new file mode 100644 index 0000000000..1158a5aa16 --- /dev/null +++ b/changelog.d/4241.bugfix @@ -0,0 +1 @@ +Fix exception caused by non-ascii event IDs diff --git a/changelog.d/4244.misc b/changelog.d/4244.misc new file mode 100644 index 0000000000..8236bbc28d --- /dev/null +++ b/changelog.d/4244.misc @@ -0,0 +1 @@ +Drop sent_transactions table diff --git a/changelog.d/4250.bugfix b/changelog.d/4250.bugfix new file mode 100644 index 0000000000..1f60f5bd0a --- /dev/null +++ b/changelog.d/4250.bugfix @@ -0,0 +1 @@ +Pushers can now be unsubscribed from on Python 3. diff --git a/changelog.d/4253.bugfix b/changelog.d/4253.bugfix new file mode 100644 index 0000000000..1796e95b86 --- /dev/null +++ b/changelog.d/4253.bugfix @@ -0,0 +1 @@ +Fix UnicodeDecodeError when postgres is configured to give non-English errors diff --git a/changelog.d/4257.misc b/changelog.d/4257.misc new file mode 100644 index 0000000000..43ac24cb7d --- /dev/null +++ b/changelog.d/4257.misc @@ -0,0 +1 @@ +Add a basic .editorconfig diff --git a/changelog.d/4260.misc b/changelog.d/4260.misc new file mode 100644 index 0000000000..0f7c4faf28 --- /dev/null +++ b/changelog.d/4260.misc @@ -0,0 +1 @@ +Update README.rst and UPGRADE.rst for Python 3. diff --git a/changelog.d/4261.misc b/changelog.d/4261.misc new file mode 100644 index 0000000000..2a88c8c856 --- /dev/null +++ b/changelog.d/4261.misc @@ -0,0 +1 @@ +Remove obsolete `verbose` and `log_file` settings from `homeserver.yaml` for Docker image. \ No newline at end of file diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index d5074be6dd..c2b8576a32 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -14,6 +14,7 @@ server_name: "{{ SYNAPSE_SERVER_NAME }}" pid_file: /homeserver.pid web_client: False soft_file_limit: 0 +log_config: "/compiled/log.config" ## Ports ## @@ -67,9 +68,6 @@ database: ## Performance ## event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" -verbose: 0 -log_file: "/data/homeserver.log" -log_config: "/compiled/log.config" ## Ratelimiting ## diff --git a/docs/log_contexts.rst b/docs/log_contexts.rst index 82ac4f91e5..27cde11cf7 100644 --- a/docs/log_contexts.rst +++ b/docs/log_contexts.rst @@ -163,7 +163,7 @@ the logcontext was set, this will make things work out ok: provided It's all too easy to forget to ``yield``: for instance if we forgot that ``do_some_stuff`` returned a deferred, we might plough on regardless. This leads to a mess; it will probably work itself out eventually, but not before -a load of stuff has been logged against the wrong content. (Normally, other +a load of stuff has been logged against the wrong context. (Normally, other things will break, more obviously, if you forget to ``yield``, so this tends not to be a major problem in practice.) @@ -440,3 +440,59 @@ To conclude: I think this scheme would have worked equally well, with less danger of messing it up, and probably made some more esoteric code easier to write. But again — changing the conventions of the entire Synapse codebase is not a sensible option for the marginal improvement offered. + + +A note on garbage-collection of Deferred chains +----------------------------------------------- + +It turns out that our logcontext rules do not play nicely with Deferred +chains which get orphaned and garbage-collected. + +Imagine we have some code that looks like this: + +.. code:: python + + listener_queue = [] + + def on_something_interesting(): + for d in listener_queue: + d.callback("foo") + + @defer.inlineCallbacks + def await_something_interesting(): + new_deferred = defer.Deferred() + listener_queue.append(new_deferred) + + with PreserveLoggingContext(): + yield new_deferred + +Obviously, the idea here is that we have a bunch of things which are waiting +for an event. (It's just an example of the problem here, but a relatively +common one.) + +Now let's imagine two further things happen. First of all, whatever was +waiting for the interesting thing goes away. (Perhaps the request times out, +or something *even more* interesting happens.) + +Secondly, let's suppose that we decide that the interesting thing is never +going to happen, and we reset the listener queue: + +.. code:: python + + def reset_listener_queue(): + listener_queue.clear() + +So, both ends of the deferred chain have now dropped their references, and the +deferred chain is now orphaned, and will be garbage-collected at some point. +Note that ``await_something_interesting`` is a generator function, and when +Python garbage-collects generator functions, it gives them a chance to clean +up by making the ``yield`` raise a ``GeneratorExit`` exception. In our case, +that means that the ``__exit__`` handler of ``PreserveLoggingContext`` will +carefully restore the request context, but there is now nothing waiting for +its return, so the request context is never cleared. + +To reiterate, this problem only arises when *both* ends of a deferred chain +are dropped. Dropping the the reference to a deferred you're supposed to be +calling is probably bad practice, so this doesn't actually happen too much. +Unfortunately, when it does happen, it will lead to leaked logcontexts which +are incredibly hard to track down. diff --git a/synapse/__init__.py b/synapse/__init__.py index 5a28fe2b82..df0504ac2c 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.33.9" +__version__ = "0.34.0rc1" diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 34382e4e3c..5309899703 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -188,17 +188,33 @@ class Auth(object): """ # Can optionally look elsewhere in the request (e.g. headers) try: - user_id, app_service = yield self._get_appservice_user_id(request) - if user_id: - request.authenticated_entity = user_id - defer.returnValue( - synapse.types.create_requester(user_id, app_service=app_service) - ) + ip_addr = self.hs.get_ip_from_request(request) + user_agent = request.requestHeaders.getRawHeaders( + b"User-Agent", + default=[b""] + )[0].decode('ascii', 'surrogateescape') access_token = self.get_access_token_from_request( request, self.TOKEN_NOT_FOUND_HTTP_STATUS ) + user_id, app_service = yield self._get_appservice_user_id(request) + if user_id: + request.authenticated_entity = user_id + + if ip_addr and self.hs.config.track_appservice_user_ips: + yield self.store.insert_client_ip( + user_id=user_id, + access_token=access_token, + ip=ip_addr, + user_agent=user_agent, + device_id="dummy-device", # stubbed + ) + + defer.returnValue( + synapse.types.create_requester(user_id, app_service=app_service) + ) + user_info = yield self.get_user_by_access_token(access_token, rights) user = user_info["user"] token_id = user_info["token_id"] @@ -208,11 +224,6 @@ class Auth(object): # stubbed out. device_id = user_info.get("device_id") - ip_addr = self.hs.get_ip_from_request(request) - user_agent = request.requestHeaders.getRawHeaders( - b"User-Agent", - default=[b""] - )[0].decode('ascii', 'surrogateescape') if user and access_token and ip_addr: yield self.store.insert_client_ip( user_id=user.to_string(), diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 2430814796..685f15c061 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -53,8 +53,8 @@ import logging from twisted.internet import defer from synapse.appservice import ApplicationServiceState +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.logcontext import run_in_background -from synapse.util.metrics import Measure logger = logging.getLogger(__name__) @@ -104,14 +104,23 @@ class _ServiceQueuer(object): self.clock = clock def enqueue(self, service, event): - # if this service isn't being sent something self.queued_events.setdefault(service.id, []).append(event) - run_in_background(self._send_request, service) + + # start a sender for this appservice if we don't already have one + + if service.id in self.requests_in_flight: + return + + run_as_background_process( + "as-sender-%s" % (service.id, ), + self._send_request, service, + ) @defer.inlineCallbacks def _send_request(self, service): - if service.id in self.requests_in_flight: - return + # sanity-check: we shouldn't get here if this service already has a sender + # running. + assert(service.id not in self.requests_in_flight) self.requests_in_flight.add(service.id) try: @@ -119,12 +128,10 @@ class _ServiceQueuer(object): events = self.queued_events.pop(service.id, []) if not events: return - - with Measure(self.clock, "servicequeuer.send"): - try: - yield self.txn_ctrl.send(service, events) - except Exception: - logger.exception("AS request failed") + try: + yield self.txn_ctrl.send(service, events) + except Exception: + logger.exception("AS request failed") finally: self.requests_in_flight.discard(service.id) @@ -223,7 +230,12 @@ class _Recoverer(object): self.backoff_counter = 1 def recover(self): - self.clock.call_later((2 ** self.backoff_counter), self.retry) + def _retry(): + run_as_background_process( + "as-recoverer-%s" % (self.service.id,), + self.retry, + ) + self.clock.call_later((2 ** self.backoff_counter), _retry) def _backoff(self): # cap the backoff to be around 8.5min => (2^9) = 512 secs diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 3b161d708a..c21cb3dd87 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -33,11 +33,16 @@ class AppServiceConfig(Config): def read_config(self, config): self.app_service_config_files = config.get("app_service_config_files", []) self.notify_appservices = config.get("notify_appservices", True) + self.track_appservice_user_ips = config.get("track_appservice_user_ips", False) def default_config(cls, **kwargs): return """\ # A list of application service config file to use app_service_config_files: [] + + # Whether or not to track application service IP addresses. Implicitly + # enables MAU tracking for application service users. + track_appservice_user_ips: False """ diff --git a/synapse/config/server.py b/synapse/config/server.py index 5ff9ac288d..4a5b902f8e 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -62,6 +62,11 @@ class ServerConfig(Config): # master, potentially causing inconsistency. self.enable_media_repo = config.get("enable_media_repo", True) + # whether to enable search. If disabled, new entries will not be inserted + # into the search tables and they will not be indexed. Users will receive + # errors when attempting to search for messages. + self.enable_search = config.get("enable_search", True) + self.filter_timeline_limit = config.get("filter_timeline_limit", -1) # Whether we should block invites sent to users on this server @@ -384,7 +389,12 @@ class ServerConfig(Config): # mau_limit_reserved_threepids: # - medium: 'email' # address: 'reserved_user@example.com' - + # + # Room searching + # + # If disabled, new messages will not be indexed for searching and users + # will receive errors when searching for messages. Defaults to enabled. + # enable_search: true """ % locals() def read_arguments(self, args): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index d2beb275cf..015909bb26 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -217,7 +217,19 @@ class RegistrationHandler(BaseHandler): user_id = None token = None attempts += 1 + if not self.hs.config.user_consent_at_registration: + yield self._auto_join_rooms(user_id) + defer.returnValue((user_id, token)) + + @defer.inlineCallbacks + def _auto_join_rooms(self, user_id): + """Automatically joins users to auto join rooms - creating the room in the first place + if the user is the first to be created. + + Args: + user_id(str): The user to join + """ # auto-join the user to any rooms we're supposed to dump them into fake_requester = create_requester(user_id) @@ -226,7 +238,6 @@ class RegistrationHandler(BaseHandler): if self.hs.config.autocreate_auto_join_rooms: count = yield self.store.count_all_users() should_auto_create_rooms = count == 1 - for r in self.hs.config.auto_join_rooms: try: if should_auto_create_rooms: @@ -256,7 +267,15 @@ class RegistrationHandler(BaseHandler): except Exception as e: logger.error("Failed to join new user to %r: %r", r, e) - defer.returnValue((user_id, token)) + @defer.inlineCallbacks + def post_consent_actions(self, user_id): + """A series of registration actions that can only be carried out once consent + has been granted + + Args: + user_id (str): The user to join + """ + yield self._auto_join_rooms(user_id) @defer.inlineCallbacks def appservice_register(self, user_localpart, as_token): diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 80e7b15de8..ec936bbb4e 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -50,6 +50,9 @@ class SearchHandler(BaseHandler): dict to be returned to the client with results of search """ + if not self.hs.config.enable_search: + raise SynapseError(400, "Search is disabled on this homeserver") + batch_group = None batch_group_key = None batch_token = None diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 0010699d31..f6b4a85e40 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -27,7 +27,7 @@ from twisted.web.client import PartialDownloadError from synapse.api.errors import Codes, LoginError, SynapseError from synapse.http.server import finish_request -from synapse.http.servlet import parse_json_object_from_request +from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.types import UserID from synapse.util.msisdn import phone_number_to_msisdn @@ -83,6 +83,7 @@ class LoginRestServlet(ClientV1RestServlet): PATTERNS = client_path_patterns("/login$") SAML2_TYPE = "m.login.saml2" CAS_TYPE = "m.login.cas" + SSO_TYPE = "m.login.sso" TOKEN_TYPE = "m.login.token" JWT_TYPE = "m.login.jwt" @@ -105,6 +106,10 @@ class LoginRestServlet(ClientV1RestServlet): if self.saml2_enabled: flows.append({"type": LoginRestServlet.SAML2_TYPE}) if self.cas_enabled: + flows.append({"type": LoginRestServlet.SSO_TYPE}) + + # we advertise CAS for backwards compat, though MSC1721 renamed it + # to SSO. flows.append({"type": LoginRestServlet.CAS_TYPE}) # While its valid for us to advertise this login type generally, @@ -384,11 +389,11 @@ class SAML2RestServlet(ClientV1RestServlet): defer.returnValue((200, {"status": "not_authenticated"})) -class CasRedirectServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/login/cas/redirect", releases=()) +class CasRedirectServlet(RestServlet): + PATTERNS = client_path_patterns("/login/(cas|sso)/redirect") def __init__(self, hs): - super(CasRedirectServlet, self).__init__(hs) + super(CasRedirectServlet, self).__init__() self.cas_server_url = hs.config.cas_server_url.encode('ascii') self.cas_service_url = hs.config.cas_service_url.encode('ascii') diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 9382b1f124..c654f9b5f0 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -42,7 +42,7 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_PUT(self, request): - spec = _rule_spec_from_path(request.postpath) + spec = _rule_spec_from_path([x.decode('utf8') for x in request.postpath]) try: priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: @@ -103,7 +103,7 @@ class PushRuleRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_DELETE(self, request): - spec = _rule_spec_from_path(request.postpath) + spec = _rule_spec_from_path([x.decode('utf8') for x in request.postpath]) requester = yield self.auth.get_user_by_req(request) user_id = requester.user.to_string() @@ -134,7 +134,7 @@ class PushRuleRestServlet(ClientV1RestServlet): rules = format_push_rules_for_user(requester.user, rules) - path = request.postpath[1:] + path = [x.decode('utf8') for x in request.postpath][1:] if path == []: # we're a reference impl: pedantry is our job. @@ -142,11 +142,10 @@ class PushRuleRestServlet(ClientV1RestServlet): PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR ) - if path[0] == b'': + if path[0] == '': defer.returnValue((200, rules)) - elif path[0] == b'global': - path = [x.decode('ascii') for x in path[1:]] - result = _filter_ruleset_with_path(rules['global'], path) + elif path[0] == 'global': + result = _filter_ruleset_with_path(rules['global'], path[1:]) defer.returnValue((200, result)) else: raise UnrecognizedRequestError() @@ -190,12 +189,24 @@ class PushRuleRestServlet(ClientV1RestServlet): def _rule_spec_from_path(path): + """Turn a sequence of path components into a rule spec + + Args: + path (sequence[unicode]): the URL path components. + + Returns: + dict: rule spec dict, containing scope/template/rule_id entries, + and possibly attr. + + Raises: + UnrecognizedRequestError if the path components cannot be parsed. + """ if len(path) < 2: raise UnrecognizedRequestError() - if path[0] != b'pushrules': + if path[0] != 'pushrules': raise UnrecognizedRequestError() - scope = path[1].decode('ascii') + scope = path[1] path = path[2:] if scope != 'global': raise UnrecognizedRequestError() @@ -203,13 +214,13 @@ def _rule_spec_from_path(path): if len(path) == 0: raise UnrecognizedRequestError() - template = path[0].decode('ascii') + template = path[0] path = path[1:] if len(path) == 0 or len(path[0]) == 0: raise UnrecognizedRequestError() - rule_id = path[0].decode('ascii') + rule_id = path[0] spec = { 'scope': scope, @@ -220,7 +231,7 @@ def _rule_spec_from_path(path): path = path[1:] if len(path) > 0 and len(path[0]) > 0: - spec['attr'] = path[0].decode('ascii') + spec['attr'] = path[0] return spec diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index b84f0260f2..4c07ae7f45 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -142,7 +142,7 @@ class PushersRemoveRestServlet(RestServlet): To allow pusher to be delete by clicking a link (ie. GET request) """ PATTERNS = client_path_patterns("/pushers/remove$") - SUCCESS_HTML = "You have been unsubscribed" + SUCCESS_HTML = b"You have been unsubscribed" def __init__(self, hs): super(PushersRemoveRestServlet, self).__init__() diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 0515715f7c..aec0c6b075 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -457,6 +457,7 @@ class RegisterRestServlet(RestServlet): yield self.store.user_set_consent_version( registered_user_id, self.hs.config.user_consent_version, ) + yield self.registration_handler.post_consent_actions(registered_user_id) defer.returnValue((200, return_dict)) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index ad525b22e1..80611cfe84 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -89,6 +89,7 @@ class ConsentResource(Resource): self.hs = hs self.store = hs.get_datastore() + self.registration_handler = hs.get_handlers().registration_handler # this is required by the request_handler wrapper self.clock = hs.get_clock() @@ -199,6 +200,7 @@ class ConsentResource(Resource): if e.code != 404: raise raise NotFoundError("Unknown user") + yield self.registration_handler.post_consent_actions(qualified_user_id) try: self._render_template(request, "success.html") diff --git a/synapse/state/v1.py b/synapse/state/v1.py index 70a981f4a2..19e091ce3b 100644 --- a/synapse/state/v1.py +++ b/synapse/state/v1.py @@ -298,6 +298,8 @@ def _resolve_normal_events(events, auth_events): def _ordered_events(events): def key_func(e): - return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() + # we have to use utf-8 rather than ascii here because it turns out we allow + # people to send us events with non-ascii event IDs :/ + return -int(e.depth), hashlib.sha1(e.event_id.encode('utf-8')).hexdigest() return sorted(events, key=key_func) diff --git a/synapse/static/client/login/index.html b/synapse/static/client/login/index.html index 96c8723cab..bcb6bc6bb7 100644 --- a/synapse/static/client/login/index.html +++ b/synapse/static/client/login/index.html @@ -12,35 +12,30 @@

Log in with one of the following methods

-
-
-