Merge branch 'develop' into matrix-org-hotfixes

changelog.d/4113.bugfix

@@ -0,0 +1 @@
+Fix e2e key backup with more than 9 backup versions

changelog.d/4163.bugfix

@@ -0,0 +1 @@
+Generating the user consent URI no longer fails on Python 3.

changelog.d/4165.misc

@@ -0,0 +1 @@
+Drop incoming events from federation for unknown rooms

changelog.d/4184.feature

@@ -0,0 +1 @@
+Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled.

scripts-dev/federation_client.py

@@ -154,10 +154,15 @@ def request_json(method, origin_name, origin_key, destination, path, content):
     s = requests.Session()
     s.mount("matrix://", MatrixConnectionAdapter())
 
+    headers = {"Host": destination, "Authorization": authorization_headers[0]}
+
+    if method == "POST":
+        headers["Content-Type"] = "application/json"
+
     result = s.request(
         method=method,
         url=dest,
-        headers={"Host": destination, "Authorization": authorization_headers[0]},
+        headers=headers,
         verify=False,
         data=content,
     )
@@ -203,7 +208,7 @@ def main():
     parser.add_argument(
         "-X",
         "--method",
-        help="HTTP method to use for the request. Defaults to GET if --data is"
+        help="HTTP method to use for the request. Defaults to GET if --body is"
         "unspecified, POST if it is.",
     )
 

synapse/federation/federation_server.py

@@ -162,8 +162,30 @@ class FederationServer(FederationBase):
                 p["age_ts"] = request_time - int(p["age"])
                 del p["age"]
 
+            # We try and pull out an event ID so that if later checks fail we
+            # can log something sensible. We don't mandate an event ID here in
+            # case future event formats get rid of the key.
+            possible_event_id = p.get("event_id", "<Unknown>")
+
+            # Now we get the room ID so that we can check that we know the
+            # version of the room.
+            room_id = p.get("room_id")
+            if not room_id:
+                logger.info(
+                    "Ignoring PDU as does not have a room_id. Event ID: %s",
+                    possible_event_id,
+                )
+                continue
+
+            try:
+                # In future we will actually use the room version to parse the
+                # PDU into an event.
+                yield self.store.get_room_version(room_id)
+            except NotFoundError:
+                logger.info("Ignoring PDU for unknown room_id: %s", room_id)
+                continue
+
             event = event_from_pdu_json(p)
-            room_id = event.room_id
             pdus_by_room.setdefault(room_id, []).append(event)
 
         pdu_results = {}

synapse/handlers/federation.py

@@ -202,27 +202,22 @@ class FederationHandler(BaseHandler):
             self.room_queues[room_id].append((pdu, origin))
             return
 
-        # If we're no longer in the room just ditch the event entirely. This
+        # If we're not in the room just ditch the event entirely. This is
-        # is probably an old server that has come back and thinks we're still
+        # probably an old server that has come back and thinks we're still in
-        # in the room (or we've been rejoined to the room by a state reset).
+        # the room (or we've been rejoined to the room by a state reset).
         #
-        # If we were never in the room then maybe our database got vaped and
+        # Note that if we were never in the room then we would have already
-        # we should check if we *are* in fact in the room. If we are then we
+        # dropped the event, since we wouldn't know the room version.
-        # can magically rejoin the room.
         is_in_room = yield self.auth.check_host_in_room(
             room_id,
             self.server_name
         )
         if not is_in_room:
-            was_in_room = yield self.store.was_host_joined(
+            logger.info(
-                pdu.room_id, self.server_name,
+                "[%s %s] Ignoring PDU from %s as we're not in the room",
+                room_id, event_id, origin,
             )
-            if was_in_room:
+            defer.returnValue(None)
-                logger.info(
-                    "[%s %s] Ignoring PDU from %s as we've left the room",
-                    room_id, event_id, origin,
-                )
-                defer.returnValue(None)
 
         state = None
         auth_chain = []
@@ -557,86 +552,54 @@ class FederationHandler(BaseHandler):
             room_id, event_id, event,
         )
 
-        # FIXME (erikj): Awful hack to make the case where we are not currently
+        event_ids = set()
-        # in the room work
+        if state:
-        # If state and auth_chain are None, then we don't need to do this check
+            event_ids |= {e.event_id for e in state}
-        # as we already know we have enough state in the DB to handle this
+        if auth_chain:
-        # event.
+            event_ids |= {e.event_id for e in auth_chain}
-        if state and auth_chain and not event.internal_metadata.is_outlier():
+
-            is_in_room = yield self.auth.check_host_in_room(
+        seen_ids = yield self.store.have_seen_events(event_ids)
-                room_id,
+
-                self.server_name
+        if state and auth_chain is not None:
-            )
+            # If we have any state or auth_chain given to us by the replication
-        else:
+            # layer, then we should handle them (if we haven't before.)
-            is_in_room = True
+
+            event_infos = []
+
+            for e in itertools.chain(auth_chain, state):
+                if e.event_id in seen_ids:
+                    continue
+                e.internal_metadata.outlier = True
+                auth_ids = e.auth_event_ids()
+                auth = {
+                    (e.type, e.state_key): e for e in auth_chain
+                    if e.event_id in auth_ids or e.type == EventTypes.Create
+                }
+                event_infos.append({
+                    "event": e,
+                    "auth_events": auth,
+                })
+                seen_ids.add(e.event_id)
 
-        if not is_in_room:
             logger.info(
-                "[%s %s] Got event for room we're not in",
+                "[%s %s] persisting newly-received auth/state events %s",
-                room_id, event_id,
+                room_id, event_id, [e["event"].event_id for e in event_infos]
             )
+            yield self._handle_new_events(origin, event_infos)
 
-            try:
+        try:
-                yield self._persist_auth_tree(
+            context = yield self._handle_new_event(
-                    origin, auth_chain, state, event
+                origin,
-                )
+                event,
-            except AuthError as e:
+                state=state,
-                raise FederationError(
+            )
-                    "ERROR",
+        except AuthError as e:
-                    e.code,
+            raise FederationError(
-                    e.msg,
+                "ERROR",
-                    affected=event_id,
+                e.code,
-                )
+                e.msg,
-
+                affected=event.event_id,
-        else:
+            )
-            event_ids = set()
-            if state:
-                event_ids |= {e.event_id for e in state}
-            if auth_chain:
-                event_ids |= {e.event_id for e in auth_chain}
-
-            seen_ids = yield self.store.have_seen_events(event_ids)
-
-            if state and auth_chain is not None:
-                # If we have any state or auth_chain given to us by the replication
-                # layer, then we should handle them (if we haven't before.)
-
-                event_infos = []
-
-                for e in itertools.chain(auth_chain, state):
-                    if e.event_id in seen_ids:
-                        continue
-                    e.internal_metadata.outlier = True
-                    auth_ids = e.auth_event_ids()
-                    auth = {
-                        (e.type, e.state_key): e for e in auth_chain
-                        if e.event_id in auth_ids or e.type == EventTypes.Create
-                    }
-                    event_infos.append({
-                        "event": e,
-                        "auth_events": auth,
-                    })
-                    seen_ids.add(e.event_id)
-
-                logger.info(
-                    "[%s %s] persisting newly-received auth/state events %s",
-                    room_id, event_id, [e["event"].event_id for e in event_infos]
-                )
-                yield self._handle_new_events(origin, event_infos)
-
-            try:
-                context = yield self._handle_new_event(
-                    origin,
-                    event,
-                    state=state,
-                )
-            except AuthError as e:
-                raise FederationError(
-                    "ERROR",
-                    e.code,
-                    e.msg,
-                    affected=event.event_id,
-                )
 
         room = yield self.store.get_room(room_id)
 

synapse/http/servlet.py

@@ -121,16 +121,15 @@ def parse_string(request, name, default=None, required=False,
 
     Args:
         request: the twisted HTTP request.
-        name (bytes/unicode): the name of the query parameter.
+        name (bytes|unicode): the name of the query parameter.
-        default (bytes/unicode|None): value to use if the parameter is absent,
+        default (bytes|unicode|None): value to use if the parameter is absent,
             defaults to None. Must be bytes if encoding is None.
         required (bool): whether to raise a 400 SynapseError if the
             parameter is absent, defaults to False.
-        allowed_values (list[bytes/unicode]): List of allowed values for the
+        allowed_values (list[bytes|unicode]): List of allowed values for the
             string, or None if any value is allowed, defaults to None. Must be
             the same type as name, if given.
-        encoding: The encoding to decode the name to, and decode the string
+        encoding (str|None): The encoding to decode the string content with.
-            content with.
 
     Returns:
         bytes/unicode|None: A string value or the default. Unicode if encoding

synapse/rest/consent/consent_resource.py

@@ -142,10 +142,10 @@ class ConsentResource(Resource):
         userhmac = None
         has_consented = False
         public_version = username == ""
-        if not public_version or not self.hs.config.user_consent_at_registration:
+        if not public_version:
-            userhmac = parse_string(request, "h", required=True, encoding=None)
+            userhmac_bytes = parse_string(request, "h", required=True, encoding=None)
 
-            self._check_hash(username, userhmac)
+            self._check_hash(username, userhmac_bytes)
 
             if username.startswith('@'):
                 qualified_user_id = username
@@ -155,15 +155,18 @@ class ConsentResource(Resource):
             u = yield self.store.get_user_by_id(qualified_user_id)
             if u is None:
                 raise NotFoundError("Unknown user")
+
             has_consented = u["consent_version"] == version
+            userhmac = userhmac_bytes.decode("ascii")
 
         try:
             self._render_template(
                 request, "%s.html" % (version,),
                 user=username,
-                userhmac=userhmac.decode('ascii'),
+                userhmac=userhmac,
                 version=version,
-                has_consented=has_consented, public_version=public_version,
+                has_consented=has_consented,
+                public_version=public_version,
             )
         except TemplateNotFound:
             raise NotFoundError("Unknown policy version")

synapse/storage/e2e_room_keys.py

@@ -118,6 +118,11 @@ class EndToEndRoomKeyStore(SQLBaseStore):
             these room keys.
         """
 
+        try:
+            version = int(version)
+        except ValueError:
+            defer.returnValue({'rooms': {}})
+
         keyvalues = {
             "user_id": user_id,
             "version": version,
@@ -212,14 +217,23 @@ class EndToEndRoomKeyStore(SQLBaseStore):
         Raises:
             StoreError: with code 404 if there are no e2e_room_keys_versions present
         Returns:
-            A deferred dict giving the info metadata for this backup version
+            A deferred dict giving the info metadata for this backup version, with
+            fields including:
+                version(str)
+                algorithm(str)
+                auth_data(object): opaque dict supplied by the client
         """
 
         def _get_e2e_room_keys_version_info_txn(txn):
             if version is None:
                 this_version = self._get_current_version(txn, user_id)
             else:
-                this_version = version
+                try:
+                    this_version = int(version)
+                except ValueError:
+                    # Our versions are all ints so if we can't convert it to an integer,
+                    # it isn't there.
+                    raise StoreError(404, "No row found")
 
             result = self._simple_select_one_txn(
                 txn,
@@ -236,6 +250,7 @@ class EndToEndRoomKeyStore(SQLBaseStore):
                 ),
             )
             result["auth_data"] = json.loads(result["auth_data"])
+            result["version"] = str(result["version"])
             return result
 
         return self.runInteraction(

synapse/storage/schema/delta/52/e2e_room_keys.sql

@@ -0,0 +1,53 @@
+/* Copyright 2018 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Change version column to an integer so we can do MAX() sensibly
+ */
+CREATE TABLE e2e_room_keys_versions_new (
+    user_id TEXT NOT NULL,
+    version BIGINT NOT NULL,
+    algorithm TEXT NOT NULL,
+    auth_data TEXT NOT NULL,
+    deleted SMALLINT DEFAULT 0 NOT NULL
+);
+
+INSERT INTO e2e_room_keys_versions_new
+    SELECT user_id, CAST(version as BIGINT), algorithm, auth_data, deleted FROM e2e_room_keys_versions;
+
+DROP TABLE e2e_room_keys_versions;
+ALTER TABLE e2e_room_keys_versions_new RENAME TO e2e_room_keys_versions;
+
+CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version);
+
+/* Change e2e_rooms_keys to match
+ */
+CREATE TABLE e2e_room_keys_new (
+    user_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    session_id TEXT NOT NULL,
+    version BIGINT NOT NULL,
+    first_message_index INT,
+    forwarded_count INT,
+    is_verified BOOLEAN,
+    session_data TEXT NOT NULL
+);
+
+INSERT INTO e2e_room_keys_new
+    SELECT user_id, room_id, session_id, CAST(version as BIGINT), first_message_index, forwarded_count, is_verified, session_data FROM e2e_room_keys;
+
+DROP TABLE e2e_room_keys;
+ALTER TABLE e2e_room_keys_new RENAME TO e2e_room_keys;
+
+CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id);

tests/rest/client/test_consent.py

@@ -60,6 +60,13 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase):
         hs = self.setup_test_homeserver(config=config)
         return hs
 
+    def test_render_public_consent(self):
+        """You can observe the terms form without specifying a user"""
+        resource = consent_resource.ConsentResource(self.hs)
+        request, channel = self.make_request("GET", "/consent?v=1", shorthand=False)
+        render(request, resource, self.reactor)
+        self.assertEqual(channel.code, 200)
+
     def test_accept_consent(self):
         """
         A user can use the consent form to accept the terms.

tests/storage/test_end_to_end_keys.py

@@ -44,6 +44,21 @@ class EndToEndKeyStoreTestCase(tests.unittest.TestCase):
         dev = res["user"]["device"]
         self.assertDictContainsSubset({"keys": json, "device_display_name": None}, dev)
 
+    @defer.inlineCallbacks
+    def test_reupload_key(self):
+        now = 1470174257070
+        json = {"key": "value"}
+
+        yield self.store.store_device("user", "device", None)
+
+        changed = yield self.store.set_e2e_device_keys("user", "device", now, json)
+        self.assertTrue(changed)
+
+        # If we try to upload the same key then we should be told nothing
+        # changed
+        changed = yield self.store.set_e2e_device_keys("user", "device", now, json)
+        self.assertFalse(changed)
+
     @defer.inlineCallbacks
     def test_get_key_with_device_name(self):
         now = 1470174257070