From df3a661e4adb7676682a5e3c298a2dfda18b08a1 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 18 Jan 2019 10:04:47 +0000
Subject: [PATCH 01/35] Search for messages across predecessor rooms

Signed-off-by: Andrew Morgan <andrew@amorgan.xyz>
---
 synapse/api/filtering.py   |  3 ++
 synapse/handlers/search.py | 69 ++++++++++++++++++++++++++++++++++++++
 synapse/storage/state.py   |  1 +
 3 files changed, 73 insertions(+)

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 16ad654864..84000e6422 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -444,6 +444,9 @@ class Filter(object):
     def include_redundant_members(self):
         return self.filter_json.get("include_redundant_members", False)
 
+    def add_room_ids(self, room_ids):
+        self.rooms += room_ids
+
 
 def _matches_wildcard(actual_value, filter_value):
     if filter_value.endswith("*"):
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index ec936bbb4e..77e7e4e0fb 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -37,6 +37,54 @@ class SearchHandler(BaseHandler):
     def __init__(self, hs):
         super(SearchHandler, self).__init__(hs)
 
+    @defer.inlineCallbacks
+    def get_old_rooms_from_upgraded_room(self, room_id):
+        """Retrieves room IDs of old rooms in the history of an upgraded room.
+
+        We do so by checking the m.room.create event of the room for a
+        `predecessor` key. If it exists, we add the room ID to our return
+        list and then check that room for a m.room.create event and so on
+        until we can no longer find any more previous rooms.
+
+        The full list of all found rooms in then returned.
+
+        Args:
+            room_id (str): The ID of the room to search through.
+
+        Returns:
+            dict of past room IDs as strings
+        """
+
+        historical_room_ids = []
+
+        while True:
+            state_ids = yield self.store.get_current_state_ids(room_id)
+            create_id = state_ids.get((EventTypes.Create, ""))
+
+            # If we can't find the create event, assume we've hit a dead end
+            if not create_id:
+                break
+
+            # Retrieve the room's create event
+            create_event = yield self.store.get_event(create_id)
+
+            if not create_event:
+                break
+
+            # Check if a predecessor room is present
+            predecessor = create_event.content.get("predecessor", None)
+            if not predecessor:
+                break
+
+            # Add predecessor's room ID
+            historical_room_id = predecessor["room_id"]
+            historical_room_ids.append(historical_room_id)
+
+            # Scan through the old room for further predecessors
+            room_id = historical_room_id
+
+        defer.returnValue(historical_room_ids)
+
     @defer.inlineCallbacks
     def search(self, user, content, batch=None):
         """Performs a full text search for a user.
@@ -139,6 +187,27 @@ class SearchHandler(BaseHandler):
 
         room_ids = search_filter.filter_rooms(room_ids)
 
+        # If doing a subset of all rooms seearch, check if any of the rooms
+        # are from an upgraded room, and search their contents as well
+        # XXX: There is the possibility that we don't have a create event for
+        # the room in question, in which case we can't return all the results
+        # we want to.
+        # Ideally we would just return the results we can get now, and
+        # try to get more results from other servers in the background.
+        if search_filter.rooms:
+            historical_room_ids = []
+            for room_id in room_ids:
+                # Add any previous rooms to the search if they exist
+                ids = yield self.get_old_rooms_from_upgraded_room(room_id)
+                historical_room_ids += ids
+
+            # Add any found rooms to the list to search
+            for historical_room_id in historical_room_ids:
+                room_ids.add(historical_room_id)
+
+            # Prevent any historical events from being filtered
+            search_filter.add_room_ids(historical_room_ids)
+
         if batch_group == "room_id":
             room_ids.intersection_update({batch_group_key})
 
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index a134e9b3e8..49b3ff4a71 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -448,6 +448,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         Returns:
             deferred: dict of (type, state_key) -> event_id
         """
+
         def _get_current_state_ids_txn(txn):
             txn.execute(
                 """SELECT type, state_key, event_id FROM current_state_events

From cb80db894186c4c9a4991e0623530a038eb82543 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Fri, 18 Jan 2019 11:22:00 +0000
Subject: [PATCH 02/35] Add changelog

---
 changelog.d/4415.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4415.feature

diff --git a/changelog.d/4415.feature b/changelog.d/4415.feature
new file mode 100644
index 0000000000..1fb1d58f8f
--- /dev/null
+++ b/changelog.d/4415.feature
@@ -0,0 +1 @@
+Search now includes results from predecessor rooms after a room upgrade.
\ No newline at end of file

From c9bfb058d85f6205fada062c78a4d1eca119417c Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 22 Jan 2019 11:12:48 +0000
Subject: [PATCH 03/35] Fix a bug with single-room search searching all rooms

* Create a new method for getting predecessor rooms
* Remove formatting change
---
 synapse/api/filtering.py   | 15 ++++++++++++--
 synapse/handlers/search.py | 42 +++++++++-----------------------------
 synapse/storage/state.py   | 29 +++++++++++++++++++++++++-
 3 files changed, 51 insertions(+), 35 deletions(-)

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 84000e6422..0d8957175d 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -444,8 +444,19 @@ class Filter(object):
     def include_redundant_members(self):
         return self.filter_json.get("include_redundant_members", False)
 
-    def add_room_ids(self, room_ids):
-        self.rooms += room_ids
+    def with_room_ids(self, room_ids):
+        """Returns a new filter with the given room IDs appended.
+
+        Args:
+            room_ids (list): A list of room_ids.
+
+        Returns:
+            filter: A new filter including the given rooms and the old
+                    filter's rooms.
+        """
+        newFilter = self
+        newFilter.rooms += room_ids
+        return newFilter
 
 
 def _matches_wildcard(actual_value, filter_value):
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index 77e7e4e0fb..75c26fe065 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -49,39 +49,26 @@ class SearchHandler(BaseHandler):
         The full list of all found rooms in then returned.
 
         Args:
-            room_id (str): The ID of the room to search through.
+            room_id (str): id of the room to search through.
 
         Returns:
-            dict of past room IDs as strings
+            Deferred[iterable[str]]: predecessor room ids
         """
 
         historical_room_ids = []
 
         while True:
-            state_ids = yield self.store.get_current_state_ids(room_id)
-            create_id = state_ids.get((EventTypes.Create, ""))
+            predecessor = yield self.store.get_room_predecessor(room_id)
 
-            # If we can't find the create event, assume we've hit a dead end
-            if not create_id:
-                break
-
-            # Retrieve the room's create event
-            create_event = yield self.store.get_event(create_id)
-
-            if not create_event:
-                break
-
-            # Check if a predecessor room is present
-            predecessor = create_event.content.get("predecessor", None)
+            # If no predecessor, assume we've hit a dead end
             if not predecessor:
                 break
 
             # Add predecessor's room ID
-            historical_room_id = predecessor["room_id"]
-            historical_room_ids.append(historical_room_id)
+            historical_room_ids.append(predecessor["room_id"])
 
             # Scan through the old room for further predecessors
-            room_id = historical_room_id
+            room_id = predecessor["room_id"]
 
         defer.returnValue(historical_room_ids)
 
@@ -185,28 +172,19 @@ class SearchHandler(BaseHandler):
         )
         room_ids = set(r.room_id for r in rooms)
 
-        room_ids = search_filter.filter_rooms(room_ids)
-
         # If doing a subset of all rooms seearch, check if any of the rooms
         # are from an upgraded room, and search their contents as well
-        # XXX: There is the possibility that we don't have a create event for
-        # the room in question, in which case we can't return all the results
-        # we want to.
-        # Ideally we would just return the results we can get now, and
-        # try to get more results from other servers in the background.
         if search_filter.rooms:
             historical_room_ids = []
-            for room_id in room_ids:
+            for room_id in search_filter.rooms:
                 # Add any previous rooms to the search if they exist
                 ids = yield self.get_old_rooms_from_upgraded_room(room_id)
                 historical_room_ids += ids
 
-            # Add any found rooms to the list to search
-            for historical_room_id in historical_room_ids:
-                room_ids.add(historical_room_id)
-
             # Prevent any historical events from being filtered
-            search_filter.add_room_ids(historical_room_ids)
+            search_filter = search_filter.with_room_ids(historical_room_ids)
+
+        room_ids = search_filter.filter_rooms(room_ids)
 
         if batch_group == "room_id":
             room_ids.intersection_update({batch_group_key})
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 49b3ff4a71..b064671851 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -437,6 +437,34 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         create_event = yield self.get_event(create_id)
         defer.returnValue(create_event.content.get("room_version", "1"))
 
+    @defer.inlineCallbacks
+    def get_room_predecessor(self, room_id):
+        """Get the predecessor room of an upgraded room if one exists.
+        Otherwise return None.
+
+        Args:
+            room_id (str)
+
+        Returns:
+            Deferred[str]: predecessor room id
+        """
+
+        state_ids = yield self.get_current_state_ids(room_id)
+        create_id = state_ids.get((EventTypes.Create, ""))
+
+        # If we can't find the create event, assume we've hit a dead end
+        if not create_id:
+            return None
+
+        # Retrieve the room's create event
+        create_event = yield self.get_event(create_id)
+
+        if not create_event:
+            return None
+
+        # Return predecessor if present
+        return create_event.content.get("predecessor", None)
+
     @cached(max_entries=100000, iterable=True)
     def get_current_state_ids(self, room_id):
         """Get the current state event ids for a room based on the
@@ -448,7 +476,6 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         Returns:
             deferred: dict of (type, state_key) -> event_id
         """
-
         def _get_current_state_ids_txn(txn):
             txn.execute(
                 """SELECT type, state_key, event_id FROM current_state_events

From c433f6109145f0cf6c80dd07ee118b68a3a0cd4e Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 22 Jan 2019 12:06:36 +0000
Subject: [PATCH 04/35] Ensure new filter is actually created

---
 synapse/api/filtering.py | 2 +-
 synapse/storage/state.py | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index 0d8957175d..f3a056110f 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -454,7 +454,7 @@ class Filter(object):
             filter: A new filter including the given rooms and the old
                     filter's rooms.
         """
-        newFilter = self
+        newFilter = Filter(self.filter_json)
         newFilter.rooms += room_ids
         return newFilter
 
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index b064671851..981d1e3600 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -448,7 +448,6 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         Returns:
             Deferred[str]: predecessor room id
         """
-
         state_ids = yield self.get_current_state_ids(room_id)
         create_id = state_ids.get((EventTypes.Create, ""))
 

From 277e50462d1422ac8cfe2df7cd2213288f3d14c5 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Tue, 22 Jan 2019 12:40:26 +0000
Subject: [PATCH 05/35] Do not return in a deferred function

---
 synapse/storage/state.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 981d1e3600..fceb9744aa 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -453,16 +453,16 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
 
         # If we can't find the create event, assume we've hit a dead end
         if not create_id:
-            return None
+            defer.returnValue(None)
 
         # Retrieve the room's create event
         create_event = yield self.get_event(create_id)
 
         if not create_event:
-            return None
+            defer.returnValue(None)
 
         # Return predecessor if present
-        return create_event.content.get("predecessor", None)
+        defer.returnValue(create_event.content.get("predecessor", None))
 
     @cached(max_entries=100000, iterable=True)
     def get_current_state_ids(self, room_id):

From c5a296b10c53aa523facf16c3a4c918f93b8188c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 11:11:52 +0000
Subject: [PATCH 06/35] Add support for persisting event format versions

Currently we only have the one event format version defined, but this
adds the necessary infrastructure to persist and fetch the format
versions alongside the events.

We specify the format version rather than the room version as:

1. We don't necessarily know the room version, existing events may be
   either v1 or v2.
2. We'd need to be careful to prevent/handle correctly if different
   events in the same room reported to be of different versions, which
   sounds annoying.
---
 synapse/api/constants.py                      | 13 +++++++++++++
 synapse/events/__init__.py                    |  5 +++++
 synapse/storage/events.py                     |  1 +
 synapse/storage/events_worker.py              | 19 +++++++++++++++----
 .../schema/delta/53/event_format_version.sql  | 16 ++++++++++++++++
 5 files changed, 50 insertions(+), 4 deletions(-)
 create mode 100644 synapse/storage/schema/delta/53/event_format_version.sql

diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 46c4b4b9dc..51ee078bc3 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -120,6 +120,19 @@ KNOWN_ROOM_VERSIONS = {
     RoomVersions.STATE_V2_TEST,
 }
 
+
+class EventFormatVersions(object):
+    """This is an internal enum for tracking the version of the event format,
+    independently from the room version.
+    """
+    V1 = 1
+
+
+KNOWN_EVENT_FORMAT_VERSIONS = {
+    EventFormatVersions.V1,
+}
+
+
 ServerNoticeMsgType = "m.server_notice"
 ServerNoticeLimitReached = "m.server_notice.usage_limit_reached"
 
diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 84c75495d5..310075c2b1 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -18,6 +18,9 @@ from distutils.util import strtobool
 
 import six
 
+from synapse.api.constants import (
+    EventFormatVersions,
+)
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
@@ -179,6 +182,8 @@ class EventBase(object):
 
 
 class FrozenEvent(EventBase):
+    format_version = EventFormatVersions.V1  # All events of this type are V1
+
     def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None):
         event_dict = dict(event_dict)
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 79e0276de6..3e1915fb87 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -1268,6 +1268,7 @@ class EventsStore(StateGroupWorkerStore, EventFederationStore, EventsWorkerStore
                         event.internal_metadata.get_dict()
                     ),
                     "json": encode_json(event_dict(event)),
+                    "format_version": event.format_version,
                 }
                 for event, _ in events_and_contexts
             ],
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index a8326f5296..8dff91e5f8 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -21,11 +21,11 @@ from canonicaljson import json
 
 from twisted.internet import defer
 
+from synapse.api.constants import EventFormatVersions
 from synapse.api.errors import NotFoundError
 # these are only included to make the type annotations work
-from synapse.events import EventBase  # noqa: F401
-from synapse.events import FrozenEvent
 from synapse.events.snapshot import EventContext  # noqa: F401
+from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.logcontext import (
@@ -353,6 +353,7 @@ class EventsWorkerStore(SQLBaseStore):
                     self._get_event_from_row,
                     row["internal_metadata"], row["json"], row["redacts"],
                     rejected_reason=row["rejects"],
+                    format_version=row["format_version"],
                 )
                 for row in rows
             ],
@@ -377,6 +378,7 @@ class EventsWorkerStore(SQLBaseStore):
                 " e.event_id as event_id, "
                 " e.internal_metadata,"
                 " e.json,"
+                " e.format_version, "
                 " r.redacts as redacts,"
                 " rej.event_id as rejects "
                 " FROM event_json as e"
@@ -392,7 +394,7 @@ class EventsWorkerStore(SQLBaseStore):
 
     @defer.inlineCallbacks
     def _get_event_from_row(self, internal_metadata, js, redacted,
-                            rejected_reason=None):
+                            format_version, rejected_reason=None):
         with Measure(self._clock, "_get_event_from_row"):
             d = json.loads(js)
             internal_metadata = json.loads(internal_metadata)
@@ -405,8 +407,17 @@ class EventsWorkerStore(SQLBaseStore):
                     desc="_get_event_from_row_rejected_reason",
                 )
 
+            if format_version is None:
+                # This means that we stored the event before we had the concept
+                # of a event format version, so it must be a V1 event.
+                format_version = EventFormatVersions.V1
+
+            # TODO: When we implement new event formats we'll need to use a
+            # different event python type
+            assert format_version == EventFormatVersions.V1
+
             original_ev = FrozenEvent(
-                d,
+                event_dict=d,
                 internal_metadata_dict=internal_metadata,
                 rejected_reason=rejected_reason,
             )
diff --git a/synapse/storage/schema/delta/53/event_format_version.sql b/synapse/storage/schema/delta/53/event_format_version.sql
new file mode 100644
index 0000000000..1d977c2834
--- /dev/null
+++ b/synapse/storage/schema/delta/53/event_format_version.sql
@@ -0,0 +1,16 @@
+/* Copyright 2019 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ALTER TABLE event_json ADD COLUMN format_version INTEGER;

From 4cd50d983d5cd35e3f425dbd27723472e26e7c13 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 11:30:24 +0000
Subject: [PATCH 07/35] Newsfile

---
 changelog.d/4437.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4437.misc

diff --git a/changelog.d/4437.misc b/changelog.d/4437.misc
new file mode 100644
index 0000000000..3391722671
--- /dev/null
+++ b/changelog.d/4437.misc
@@ -0,0 +1 @@
+Add support for persisting event format versions in the database

From be1065af598e46224aee2eb7b8bcd7e723a72426 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 11:48:16 +0000
Subject: [PATCH 08/35] isort

---
 synapse/events/__init__.py       | 4 +---
 synapse/storage/events_worker.py | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 310075c2b1..9dd6940385 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -18,9 +18,7 @@ from distutils.util import strtobool
 
 import six
 
-from synapse.api.constants import (
-    EventFormatVersions,
-)
+from synapse.api.constants import EventFormatVersions
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 8dff91e5f8..599f892858 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -23,9 +23,9 @@ from twisted.internet import defer
 
 from synapse.api.constants import EventFormatVersions
 from synapse.api.errors import NotFoundError
+from synapse.events import FrozenEvent
 # these are only included to make the type annotations work
 from synapse.events.snapshot import EventContext  # noqa: F401
-from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.util.logcontext import (

From 67cd4dad81ed2932009472da2d13648ca11eab73 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 16:50:06 +0000
Subject: [PATCH 09/35] Implement MSC 1813 - Add room version to make APIs

We also implement `make_membership_event` converting the returned
room version to an event format version.
---
 synapse/events/__init__.py              | 17 ++++++++++++++++-
 synapse/federation/federation_client.py | 21 ++++++++++++++++-----
 synapse/federation/federation_server.py |  8 +++++++-
 synapse/handlers/federation.py          |  2 +-
 4 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 9dd6940385..e6f94e68af 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -18,7 +18,7 @@ from distutils.util import strtobool
 
 import six
 
-from synapse.api.constants import EventFormatVersions
+from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventFormatVersions
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
@@ -235,3 +235,18 @@ class FrozenEvent(EventBase):
             self.get("type", None),
             self.get("state_key", None),
         )
+
+
+def room_version_to_event_format(room_version):
+    """Converts a room version string to the event format
+
+    Args:
+        room_version (str)
+
+    Returns:
+        int
+    """
+    if room_version not in KNOWN_ROOM_VERSIONS:
+        raise
+
+    return EventFormatVersions.V1
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index d05ed91d64..0757ad12f4 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -25,14 +25,19 @@ from prometheus_client import Counter
 
 from twisted.internet import defer
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
+from synapse.api.constants import (
+    KNOWN_ROOM_VERSIONS,
+    EventTypes,
+    Membership,
+    RoomVersions,
+)
 from synapse.api.errors import (
     CodeMessageException,
     FederationDeniedError,
     HttpResponseException,
     SynapseError,
 )
-from synapse.events import builder
+from synapse.events import builder, room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -536,8 +541,9 @@ class FederationClient(FederationBase):
             params (dict[str, str|Iterable[str]]): Query parameters to include in the
                 request.
         Return:
-            Deferred: resolves to a tuple of (origin (str), event (object))
-            where origin is the remote homeserver which generated the event.
+            Deferred[tuple[str, dict, int]]: resolves to a tuple of
+            `(origin, event, event_format)` where origin is the remote
+            homeserver which generated the event.
 
             Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.
@@ -557,6 +563,11 @@ class FederationClient(FederationBase):
                 destination, room_id, user_id, membership, params,
             )
 
+            # Note: If not supplied, the room version may be either v1 or v2,
+            # however either way the event format version will be v1.
+            room_version = ret.get("room_version", RoomVersions.V1)
+            event_format = room_version_to_event_format(room_version)
+
             pdu_dict = ret.get("event", None)
             if not isinstance(pdu_dict, dict):
                 raise InvalidResponseError("Bad 'event' field in response")
@@ -574,7 +585,7 @@ class FederationClient(FederationBase):
             ev = builder.EventBuilder(pdu_dict)
 
             defer.returnValue(
-                (destination, ev)
+                (destination, ev, event_format)
             )
 
         return self._try_destination_list(
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 37d29e7027..17eccaaea0 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -400,8 +400,14 @@ class FederationServer(FederationBase):
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, room_id)
         pdu = yield self.handler.on_make_leave_request(room_id, user_id)
+
+        room_version = yield self.store.get_room_version(room_id)
+
         time_now = self._clock.time_msec()
-        defer.returnValue({"event": pdu.get_pdu_json(time_now)})
+        defer.returnValue({
+            "event": pdu.get_pdu_json(time_now),
+            "room_version": room_version,
+        })
 
     @defer.inlineCallbacks
     def on_send_leave_request(self, origin, content):
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a3bb864bb2..d1ba1450e9 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1336,7 +1336,7 @@ class FederationHandler(BaseHandler):
     @defer.inlineCallbacks
     def _make_and_verify_event(self, target_hosts, room_id, user_id, membership,
                                content={}, params=None):
-        origin, pdu = yield self.federation_client.make_membership_event(
+        origin, pdu, _ = yield self.federation_client.make_membership_event(
             target_hosts,
             room_id,
             user_id,

From 6a41d2a187ecef484a3aa67518ec9b4b0638c614 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 17:19:58 +0000
Subject: [PATCH 10/35] Add room_version param to get_pdu

When we add new event format we'll need to know the event format or room
version when parsing events.
---
 synapse/federation/federation_base.py   | 11 ++++--
 synapse/federation/federation_client.py | 46 +++++++++++++++++++++----
 synapse/federation/federation_server.py |  4 ++-
 synapse/handlers/federation.py          | 12 +++++--
 4 files changed, 60 insertions(+), 13 deletions(-)

diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index b7ad729c63..d749bfdd3a 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -43,8 +43,8 @@ class FederationBase(object):
         self._clock = hs.get_clock()
 
     @defer.inlineCallbacks
-    def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False,
-                                       include_none=False):
+    def _check_sigs_and_hash_and_fetch(self, origin, pdus, room_version,
+                                       outlier=False, include_none=False):
         """Takes a list of PDUs and checks the signatures and hashs of each
         one. If a PDU fails its signature check then we check if we have it in
         the database and if not then request if from the originating server of
@@ -56,8 +56,12 @@ class FederationBase(object):
         a new list.
 
         Args:
+            origin (str)
             pdu (list)
-            outlier (bool)
+            room_version (str)
+            outlier (bool): Whether the events are outliers or not
+            include_none (str): Whether to include None in the returned list
+                for events that have failed their checks
 
         Returns:
             Deferred : A list of PDUs that have valid signatures and hashes.
@@ -84,6 +88,7 @@ class FederationBase(object):
                     res = yield self.get_pdu(
                         destinations=[pdu.origin],
                         event_id=pdu.event_id,
+                        room_version=room_version,
                         outlier=outlier,
                         timeout=10000,
                     )
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index d05ed91d64..4e171f9b56 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -25,7 +25,12 @@ from prometheus_client import Counter
 
 from twisted.internet import defer
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventTypes, Membership
+from synapse.api.constants import (
+    KNOWN_ROOM_VERSIONS,
+    EventTypes,
+    Membership,
+    RoomVersions,
+)
 from synapse.api.errors import (
     CodeMessageException,
     FederationDeniedError,
@@ -202,7 +207,8 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    def get_pdu(self, destinations, event_id, outlier=False, timeout=None):
+    def get_pdu(self, destinations, event_id, room_version, outlier=False,
+                timeout=None):
         """Requests the PDU with given origin and ID from the remote home
         servers.
 
@@ -212,6 +218,7 @@ class FederationClient(FederationBase):
         Args:
             destinations (list): Which home servers to query
             event_id (str): event to fetch
+            room_version (str): version of the room
             outlier (bool): Indicates whether the PDU is an `outlier`, i.e. if
                 it's from an arbitary point in the context as opposed to part
                 of the current block of PDUs. Defaults to `False`
@@ -352,10 +359,13 @@ class FederationClient(FederationBase):
             ev.event_id for ev in itertools.chain(pdus, auth_chain)
         ])
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_pdus = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in pdus if p.event_id not in seen_events],
-            outlier=True
+            outlier=True,
+            room_version=room_version,
         )
         signed_pdus.extend(
             seen_events[p.event_id] for p in pdus if p.event_id in seen_events
@@ -364,7 +374,8 @@ class FederationClient(FederationBase):
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in auth_chain if p.event_id not in seen_events],
-            outlier=True
+            outlier=True,
+            room_version=room_version,
         )
         signed_auth.extend(
             seen_events[p.event_id] for p in auth_chain if p.event_id in seen_events
@@ -411,6 +422,8 @@ class FederationClient(FederationBase):
             random.shuffle(srvs)
             return srvs
 
+        room_version = yield self.store.get_room_version(room_id)
+
         batch_size = 20
         missing_events = list(missing_events)
         for i in range(0, len(missing_events), batch_size):
@@ -421,6 +434,7 @@ class FederationClient(FederationBase):
                     self.get_pdu,
                     destinations=random_server_list(),
                     event_id=e_id,
+                    room_version=room_version,
                 )
                 for e_id in batch
             ]
@@ -450,8 +464,11 @@ class FederationClient(FederationBase):
             for p in res["auth_chain"]
         ]
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
-            destination, auth_chain, outlier=True
+            destination, auth_chain,
+            outlier=True, room_version=room_version,
         )
 
         signed_auth.sort(key=lambda e: e.depth)
@@ -650,9 +667,20 @@ class FederationClient(FederationBase):
                 for p in itertools.chain(state, auth_chain)
             }
 
+            room_version = None
+            for e in state:
+                if (e.type, e.state_key) == (EventTypes.Create, ""):
+                    room_version = e.content.get("room_version", RoomVersions.V1)
+                    break
+
+            if room_version is None:
+                # We use this error has that is what
+                raise SynapseError(400, "No create event in state")
+
             valid_pdus = yield self._check_sigs_and_hash_and_fetch(
                 destination, list(pdus.values()),
                 outlier=True,
+                room_version=room_version,
             )
 
             valid_pdus_map = {
@@ -790,8 +818,10 @@ class FederationClient(FederationBase):
             for e in content["auth_chain"]
         ]
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
-            destination, auth_chain, outlier=True
+            destination, auth_chain, outlier=True, room_version=room_version,
         )
 
         signed_auth.sort(key=lambda e: e.depth)
@@ -838,8 +868,10 @@ class FederationClient(FederationBase):
                 for e in content.get("events", [])
             ]
 
+            room_version = yield self.store.get_room_version(room_id)
+
             signed_events = yield self._check_sigs_and_hash_and_fetch(
-                destination, events, outlier=False
+                destination, events, outlier=False, room_version=room_version,
             )
         except HttpResponseException as e:
             if not e.code == 400:
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 37d29e7027..cb729c69ea 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -457,8 +457,10 @@ class FederationServer(FederationBase):
                 for e in content["auth_chain"]
             ]
 
+            room_version = yield self.store.get_room_version(room_id)
+
             signed_auth = yield self._check_sigs_and_hash_and_fetch(
-                origin, auth_chain, outlier=True
+                origin, auth_chain, outlier=True, room_version=room_version,
             )
 
             ret = yield self.handler.on_query_auth(
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a3bb864bb2..a9dc4a4e4e 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -34,6 +34,7 @@ from synapse.api.constants import (
     EventTypes,
     Membership,
     RejectedReason,
+    RoomVersions,
 )
 from synapse.api.errors import (
     AuthError,
@@ -342,6 +343,8 @@ class FederationHandler(BaseHandler):
                             room_id, event_id, p,
                         )
 
+                        room_version = yield self.store.get_room_version(room_id)
+
                         with logcontext.nested_logging_context(p):
                             # note that if any of the missing prevs share missing state or
                             # auth events, the requests to fetch those events are deduped
@@ -355,7 +358,7 @@ class FederationHandler(BaseHandler):
                             # we want the state *after* p; get_state_for_room returns the
                             # state *before* p.
                             remote_event = yield self.federation_client.get_pdu(
-                                [origin], p, outlier=True,
+                                [origin], p, room_version, outlier=True,
                             )
 
                             if remote_event is None:
@@ -379,7 +382,6 @@ class FederationHandler(BaseHandler):
                             for x in remote_state:
                                 event_map[x.event_id] = x
 
-                    room_version = yield self.store.get_room_version(room_id)
                     state_map = yield resolve_events_with_store(
                         room_version, state_maps, event_map,
                         state_res_store=StateResolutionStore(self.store),
@@ -655,6 +657,8 @@ class FederationHandler(BaseHandler):
         if dest == self.server_name:
             raise SynapseError(400, "Can't backfill from self.")
 
+        room_version = yield self.store.get_room_version(room_id)
+
         events = yield self.federation_client.backfill(
             dest,
             room_id,
@@ -748,6 +752,7 @@ class FederationHandler(BaseHandler):
                             self.federation_client.get_pdu,
                             [dest],
                             event_id,
+                            room_version=room_version,
                             outlier=True,
                             timeout=10000,
                         )
@@ -1659,6 +1664,8 @@ class FederationHandler(BaseHandler):
                 create_event = e
                 break
 
+        room_version = create_event.content.get("room_version", RoomVersions.V1)
+
         missing_auth_events = set()
         for e in itertools.chain(auth_events, state, [event]):
             for e_id in e.auth_event_ids():
@@ -1669,6 +1676,7 @@ class FederationHandler(BaseHandler):
             m_ev = yield self.federation_client.get_pdu(
                 [origin],
                 e_id,
+                room_version=room_version,
                 outlier=True,
                 timeout=10000,
             )

From a4ef8d8dd57e05f5e3adb84210122781ac1cee2e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 17:24:44 +0000
Subject: [PATCH 11/35] Newsfile

---
 changelog.d/4448.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4448.misc

diff --git a/changelog.d/4448.misc b/changelog.d/4448.misc
new file mode 100644
index 0000000000..43f8963614
--- /dev/null
+++ b/changelog.d/4448.misc
@@ -0,0 +1 @@
+Add infrastructure to support different event formats

From 4a8b7156795443f540d08217dce14aa4d3af3c53 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 17:25:27 +0000
Subject: [PATCH 12/35] Newsfile

---
 changelog.d/4447.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4447.misc

diff --git a/changelog.d/4447.misc b/changelog.d/4447.misc
new file mode 100644
index 0000000000..43f8963614
--- /dev/null
+++ b/changelog.d/4447.misc
@@ -0,0 +1 @@
+Add infrastructure to support different event formats

From c5a125b24f33127b6a6e281e10b4535358d6cb92 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 17:26:06 +0000
Subject: [PATCH 13/35] Update newsfile

---
 changelog.d/4437.misc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/4437.misc b/changelog.d/4437.misc
index 3391722671..43f8963614 100644
--- a/changelog.d/4437.misc
+++ b/changelog.d/4437.misc
@@ -1 +1 @@
-Add support for persisting event format versions in the database
+Add infrastructure to support different event formats

From 886e5acc762b879b606773b511ff92345aef14c6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 16 Jan 2019 15:13:07 +0000
Subject: [PATCH 14/35] Store rejected remote invite events as outliers

Currently they're stored as non-outliers even though the server isn't in
the room, which can be problematic in places where the code assumes it
has the state for all non outlier events.

In particular, there is an edge case where persisting the leave event
triggers a state resolution, which requires looking up the room version
from state. Since the server doesn't have the state, this causes an
exception to be thrown.
---
 synapse/federation/federation_client.py | 10 ++++--
 synapse/handlers/federation.py          | 44 +++++++------------------
 synapse/storage/roommember.py           |  5 +--
 3 files changed, 21 insertions(+), 38 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index d05ed91d64..8fa726759e 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -32,7 +32,6 @@ from synapse.api.errors import (
     HttpResponseException,
     SynapseError,
 )
-from synapse.events import builder
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -66,6 +65,8 @@ class FederationClient(FederationBase):
         self.state = hs.get_state_handler()
         self.transport_layer = hs.get_federation_transport_client()
 
+        self.event_builder_factory = hs.get_event_builder_factory()
+
         self._get_pdu_cache = ExpiringCache(
             cache_name="get_pdu_cache",
             clock=self._clock,
@@ -571,7 +572,12 @@ class FederationClient(FederationBase):
             if "prev_state" not in pdu_dict:
                 pdu_dict["prev_state"] = []
 
-            ev = builder.EventBuilder(pdu_dict)
+            # Strip off the fields that we want to clobber.
+            pdu_dict.pop("origin", None)
+            pdu_dict.pop("origin_server_ts", None)
+            pdu_dict.pop("unsigned", None)
+
+            ev = self.event_builder_factory.new(pdu_dict)
 
             defer.returnValue(
                 (destination, ev)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a3bb864bb2..70be87cd3d 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -43,10 +43,7 @@ from synapse.api.errors import (
     StoreError,
     SynapseError,
 )
-from synapse.crypto.event_signing import (
-    add_hashes_and_signatures,
-    compute_event_signature,
-)
+from synapse.crypto.event_signing import compute_event_signature
 from synapse.events.validator import EventValidator
 from synapse.replication.http.federation import (
     ReplicationCleanRoomRestServlet,
@@ -58,7 +55,6 @@ from synapse.types import UserID, get_domain_from_id
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room
-from synapse.util.frozenutils import unfreeze
 from synapse.util.logutils import log_function
 from synapse.util.retryutils import NotRetryingDestination
 from synapse.visibility import filter_events_for_server
@@ -1083,7 +1079,9 @@ class FederationHandler(BaseHandler):
         handled_events = set()
 
         try:
-            event = self._sign_event(event)
+            self._sign_event(event)
+            event.internal_metadata.outlier = False
+
             # Try the host we successfully got a response to /make_join/
             # request first.
             try:
@@ -1289,13 +1287,7 @@ class FederationHandler(BaseHandler):
         event.internal_metadata.outlier = True
         event.internal_metadata.invite_from_remote = True
 
-        event.signatures.update(
-            compute_event_signature(
-                event,
-                self.hs.hostname,
-                self.hs.config.signing_key[0]
-            )
-        )
+        self._sign_event(event)
 
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
@@ -1313,7 +1305,7 @@ class FederationHandler(BaseHandler):
         # Mark as outlier as we don't have any state for this event; we're not
         # even in the room.
         event.internal_metadata.outlier = True
-        event = self._sign_event(event)
+        self._sign_event(event)
 
         # Try the host that we succesfully called /make_leave/ on first for
         # the /send_leave/ request.
@@ -1358,26 +1350,14 @@ class FederationHandler(BaseHandler):
         defer.returnValue((origin, event))
 
     def _sign_event(self, event):
-        event.internal_metadata.outlier = False
-
-        builder = self.event_builder_factory.new(
-            unfreeze(event.get_pdu_json())
+        event.signatures.update(
+            compute_event_signature(
+                event,
+                self.hs.hostname,
+                self.hs.config.signing_key[0]
+            )
         )
 
-        builder.event_id = self.event_builder_factory.create_event_id()
-        builder.origin = self.hs.hostname
-
-        if not hasattr(event, "signatures"):
-            builder.signatures = {}
-
-        add_hashes_and_signatures(
-            builder,
-            self.hs.hostname,
-            self.hs.config.signing_key[0],
-        )
-
-        return builder.build()
-
     @defer.inlineCallbacks
     @log_function
     def on_make_leave_request(self, room_id, user_id):
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 0707f9a86a..c7488f4259 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -591,10 +591,7 @@ class RoomMemberStore(RoomMemberWorkerStore):
             # i.e., its something that has just happened.
             # The only current event that can also be an outlier is if its an
             # invite that has come in across federation.
-            is_new_state = not backfilled and (
-                not event.internal_metadata.is_outlier()
-                or event.internal_metadata.is_invite_from_remote()
-            )
+            is_new_state = not backfilled
             is_mine = self.hs.is_mine_id(event.state_key)
             if is_new_state and is_mine:
                 if event.membership == Membership.INVITE:

From 183738f469b13b62f6c28e8dfe015371a8e938ff Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 16 Jan 2019 15:25:34 +0000
Subject: [PATCH 15/35] Newsfile

---
 changelog.d/4405.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4405.bugfix

diff --git a/changelog.d/4405.bugfix b/changelog.d/4405.bugfix
new file mode 100644
index 0000000000..974d799b88
--- /dev/null
+++ b/changelog.d/4405.bugfix
@@ -0,0 +1 @@
+Fix bug when rejecting remote invites

From 07f62da55ac8903f7ea224255b8defd122724ec4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 19:44:37 +0000
Subject: [PATCH 16/35] Remove unnecessary '_sign_event'

---
 synapse/federation/federation_client.py |  9 ++++++++-
 synapse/handlers/federation.py          | 14 --------------
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 8fa726759e..f4adcb556d 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -32,6 +32,7 @@ from synapse.api.errors import (
     HttpResponseException,
     SynapseError,
 )
+from synapse.crypto.event_signing import add_hashes_and_signatures
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -577,7 +578,13 @@ class FederationClient(FederationBase):
             pdu_dict.pop("origin_server_ts", None)
             pdu_dict.pop("unsigned", None)
 
-            ev = self.event_builder_factory.new(pdu_dict)
+            builder = self.event_builder_factory.new(pdu_dict)
+            add_hashes_and_signatures(
+                builder,
+                self.hs.hostname,
+                self.hs.config.signing_key[0]
+            )
+            ev = builder.build()
 
             defer.returnValue(
                 (destination, ev)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 70be87cd3d..9a14ba4517 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -43,7 +43,6 @@ from synapse.api.errors import (
     StoreError,
     SynapseError,
 )
-from synapse.crypto.event_signing import compute_event_signature
 from synapse.events.validator import EventValidator
 from synapse.replication.http.federation import (
     ReplicationCleanRoomRestServlet,
@@ -1079,7 +1078,6 @@ class FederationHandler(BaseHandler):
         handled_events = set()
 
         try:
-            self._sign_event(event)
             event.internal_metadata.outlier = False
 
             # Try the host we successfully got a response to /make_join/
@@ -1287,8 +1285,6 @@ class FederationHandler(BaseHandler):
         event.internal_metadata.outlier = True
         event.internal_metadata.invite_from_remote = True
 
-        self._sign_event(event)
-
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
 
@@ -1305,7 +1301,6 @@ class FederationHandler(BaseHandler):
         # Mark as outlier as we don't have any state for this event; we're not
         # even in the room.
         event.internal_metadata.outlier = True
-        self._sign_event(event)
 
         # Try the host that we succesfully called /make_leave/ on first for
         # the /send_leave/ request.
@@ -1349,15 +1344,6 @@ class FederationHandler(BaseHandler):
         assert(event.room_id == room_id)
         defer.returnValue((origin, event))
 
-    def _sign_event(self, event):
-        event.signatures.update(
-            compute_event_signature(
-                event,
-                self.hs.hostname,
-                self.hs.config.signing_key[0]
-            )
-        )
-
     @defer.inlineCallbacks
     @log_function
     def on_make_leave_request(self, room_id, user_id):

From 7c288c22500e2045d36a29c38d2671fad6484e30 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 20:05:44 +0000
Subject: [PATCH 17/35] Clarify the invite flows

---
 synapse/events/__init__.py     |  8 ++++++--
 synapse/handlers/federation.py | 12 +++++++++++-
 synapse/storage/roommember.py  | 11 +++++++----
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 84c75495d5..5030636c7e 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -41,8 +41,12 @@ class _EventInternalMetadata(object):
     def is_outlier(self):
         return getattr(self, "outlier", False)
 
-    def is_invite_from_remote(self):
-        return getattr(self, "invite_from_remote", False)
+    def is_new_remote_event(self):
+        """Whether this is a new remote event, like an invite or an invite
+        rejection. This is needed as those events are marked as outliers, but
+        they still need to be processed.
+        """
+        return getattr(self, "new_remote_event", False)
 
     def get_send_on_behalf_of(self):
         """Whether this server should send the event on behalf of another server.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 9a14ba4517..e017cab777 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -43,6 +43,7 @@ from synapse.api.errors import (
     StoreError,
     SynapseError,
 )
+from synapse.crypto.event_signing import compute_event_signature
 from synapse.events.validator import EventValidator
 from synapse.replication.http.federation import (
     ReplicationCleanRoomRestServlet,
@@ -1283,7 +1284,15 @@ class FederationHandler(BaseHandler):
             )
 
         event.internal_metadata.outlier = True
-        event.internal_metadata.invite_from_remote = True
+        event.internal_metadata.new_remote_event = True
+
+        event.signatures.update(
+            compute_event_signature(
+                event,
+                self.hs.hostname,
+                self.hs.config.signing_key[0]
+            )
+        )
 
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
@@ -1301,6 +1310,7 @@ class FederationHandler(BaseHandler):
         # Mark as outlier as we don't have any state for this event; we're not
         # even in the room.
         event.internal_metadata.outlier = True
+        event.internal_metadata.new_remote_event = True
 
         # Try the host that we succesfully called /make_leave/ on first for
         # the /send_leave/ request.
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index c7488f4259..40b13de80b 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -588,10 +588,13 @@ class RoomMemberStore(RoomMemberWorkerStore):
             )
 
             # We update the local_invites table only if the event is "current",
-            # i.e., its something that has just happened.
-            # The only current event that can also be an outlier is if its an
-            # invite that has come in across federation.
-            is_new_state = not backfilled
+            # i.e., its something that has just happened. If the event is an
+            # outlier it is only current if its a "new remote event", like a
+            # remote invite or a rejection of a remote invite.
+            is_new_state = not backfilled and (
+                not event.internal_metadata.is_outlier()
+                or event.internal_metadata.is_new_remote_event()
+            )
             is_mine = self.hs.is_mine_id(event.state_key)
             if is_new_state and is_mine:
                 if event.membership == Membership.INVITE:

From f431ff3fb82cb3da549f92d8aef704bbf58387b6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 20:21:33 +0000
Subject: [PATCH 18/35] Require event format version to parse or create events

---
 synapse/events/__init__.py              | 24 ++++++++-
 synapse/events/builder.py               | 51 +++++++++++++++++-
 synapse/federation/federation_base.py   |  9 ++--
 synapse/federation/federation_client.py | 64 +++++++++++++---------
 synapse/federation/federation_server.py | 33 +++++++-----
 synapse/federation/transport/server.py  |  4 +-
 synapse/handlers/federation.py          | 72 +++++++++++++++----------
 synapse/handlers/message.py             | 10 +++-
 8 files changed, 193 insertions(+), 74 deletions(-)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 01db729847..e74ea33a7c 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -18,7 +18,11 @@ from distutils.util import strtobool
 
 import six
 
-from synapse.api.constants import KNOWN_ROOM_VERSIONS, EventFormatVersions
+from synapse.api.constants import (
+    KNOWN_EVENT_FORMAT_VERSIONS,
+    KNOWN_ROOM_VERSIONS,
+    EventFormatVersions,
+)
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
@@ -254,3 +258,21 @@ def room_version_to_event_format(room_version):
         raise
 
     return EventFormatVersions.V1
+
+
+def event_type_from_format_version(format_version):
+    """Returns the python type to use to construct an Event object for the
+    given event format version.
+
+    Args:
+        format_version (int): The event format version
+
+    Returns:
+        type: A type that can be initialized as per the initializer of
+        `FrozenEvent`
+    """
+    if format_version not in KNOWN_EVENT_FORMAT_VERSIONS:
+        raise Exception(
+            "No event format %r" % (format_version,)
+        )
+    return FrozenEvent
diff --git a/synapse/events/builder.py b/synapse/events/builder.py
index e662eaef10..7e63371095 100644
--- a/synapse/events/builder.py
+++ b/synapse/events/builder.py
@@ -15,12 +15,39 @@
 
 import copy
 
+from synapse.api.constants import RoomVersions
 from synapse.types import EventID
 from synapse.util.stringutils import random_string
 
 from . import EventBase, FrozenEvent, _event_dict_property
 
 
+def get_event_builder(room_version, key_values={}, internal_metadata_dict={}):
+    """Generate an event builder appropriate for the given room version
+
+    Args:
+        room_version (str): Version of the room that we're creating an
+            event builder for
+        key_values (dict): Fields used as the basis of the new event
+        internal_metadata_dict (dict): Used to create the `_EventInternalMetadata`
+            object.
+
+    Returns:
+        EventBuilder
+    """
+    if room_version in {
+        RoomVersions.V1,
+        RoomVersions.V2,
+        RoomVersions.VDH_TEST,
+        RoomVersions.STATE_V2_TEST,
+    }:
+        return EventBuilder(key_values, internal_metadata_dict)
+    else:
+        raise Exception(
+            "No event format defined for version %r" % (room_version,)
+        )
+
+
 class EventBuilder(EventBase):
     def __init__(self, key_values={}, internal_metadata_dict={}):
         signatures = copy.deepcopy(key_values.pop("signatures", {}))
@@ -58,7 +85,29 @@ class EventBuilderFactory(object):
 
         return e_id.to_string()
 
-    def new(self, key_values={}):
+    def new(self, room_version, key_values={}):
+        """Generate an event builder appropriate for the given room version
+
+        Args:
+            room_version (str): Version of the room that we're creating an
+                event builder for
+            key_values (dict): Fields used as the basis of the new event
+
+        Returns:
+            EventBuilder
+        """
+
+        # There's currently only the one event version defined
+        if room_version not in {
+            RoomVersions.V1,
+            RoomVersions.V2,
+            RoomVersions.VDH_TEST,
+            RoomVersions.STATE_V2_TEST,
+        }:
+            raise Exception(
+                "No event format defined for version %r" % (room_version,)
+            )
+
         key_values["event_id"] = self.create_event_id()
 
         time_now = int(self.clock.time_msec())
diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index d749bfdd3a..5c31e5f85f 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -23,7 +23,7 @@ from twisted.internet.defer import DeferredList
 from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
 from synapse.api.errors import Codes, SynapseError
 from synapse.crypto.event_signing import check_event_content_hash
-from synapse.events import FrozenEvent
+from synapse.events import event_type_from_format_version
 from synapse.events.utils import prune_event
 from synapse.http.servlet import assert_params_in_dict
 from synapse.types import get_domain_from_id
@@ -302,11 +302,12 @@ def _is_invite_via_3pid(event):
     )
 
 
-def event_from_pdu_json(pdu_json, outlier=False):
+def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
     """Construct a FrozenEvent from an event json received over federation
 
     Args:
         pdu_json (object): pdu as received over federation
+        event_format_version (int): The event format version
         outlier (bool): True to mark this event as an outlier
 
     Returns:
@@ -330,8 +331,8 @@ def event_from_pdu_json(pdu_json, outlier=False):
     elif depth > MAX_DEPTH:
         raise SynapseError(400, "Depth too large", Codes.BAD_JSON)
 
-    event = FrozenEvent(
-        pdu_json
+    event = event_type_from_format_version(event_format_version)(
+        pdu_json,
     )
 
     event.internal_metadata.outlier = outlier
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 9ba3e1c42f..5bfdbe8126 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -37,8 +37,8 @@ from synapse.api.errors import (
     HttpResponseException,
     SynapseError,
 )
-from synapse.events import room_version_to_event_format
 from synapse.crypto.event_signing import add_hashes_and_signatures
+from synapse.events import room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -170,13 +170,13 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    def backfill(self, dest, context, limit, extremities):
+    def backfill(self, dest, room_id, limit, extremities):
         """Requests some more historic PDUs for the given context from the
         given destination server.
 
         Args:
             dest (str): The remote home server to ask.
-            context (str): The context to backfill.
+            room_id (str): The room_id to backfill.
             limit (int): The maximum number of PDUs to return.
             extremities (list): List of PDU id and origins of the first pdus
                 we have seen from the context
@@ -191,12 +191,15 @@ class FederationClient(FederationBase):
             return
 
         transaction_data = yield self.transport_layer.backfill(
-            dest, context, extremities, limit)
+            dest, room_id, extremities, limit)
 
         logger.debug("backfill transaction_data=%s", repr(transaction_data))
 
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
         pdus = [
-            event_from_pdu_json(p, outlier=False)
+            event_from_pdu_json(p, format_ver, outlier=False)
             for p in transaction_data["pdus"]
         ]
 
@@ -240,6 +243,8 @@ class FederationClient(FederationBase):
 
         pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {})
 
+        format_ver = room_version_to_event_format(room_version)
+
         signed_pdu = None
         for destination in destinations:
             now = self._clock.time_msec()
@@ -255,7 +260,7 @@ class FederationClient(FederationBase):
                 logger.debug("transaction_data %r", transaction_data)
 
                 pdu_list = [
-                    event_from_pdu_json(p, outlier=outlier)
+                    event_from_pdu_json(p, format_ver, outlier=outlier)
                     for p in transaction_data["pdus"]
                 ]
 
@@ -349,12 +354,16 @@ class FederationClient(FederationBase):
             destination, room_id, event_id=event_id,
         )
 
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
         pdus = [
-            event_from_pdu_json(p, outlier=True) for p in result["pdus"]
+            event_from_pdu_json(p, format_ver, outlier=True)
+            for p in result["pdus"]
         ]
 
         auth_chain = [
-            event_from_pdu_json(p, outlier=True)
+            event_from_pdu_json(p, format_ver, outlier=True)
             for p in result.get("auth_chain", [])
         ]
 
@@ -362,8 +371,6 @@ class FederationClient(FederationBase):
             ev.event_id for ev in itertools.chain(pdus, auth_chain)
         ])
 
-        room_version = yield self.store.get_room_version(room_id)
-
         signed_pdus = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in pdus if p.event_id not in seen_events],
@@ -462,13 +469,14 @@ class FederationClient(FederationBase):
             destination, room_id, event_id,
         )
 
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
         auth_chain = [
-            event_from_pdu_json(p, outlier=True)
+            event_from_pdu_json(p, format_ver, outlier=True)
             for p in res["auth_chain"]
         ]
 
-        room_version = yield self.store.get_room_version(room_id)
-
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination, auth_chain,
             outlier=True, room_version=room_version,
@@ -556,7 +564,7 @@ class FederationClient(FederationBase):
             params (dict[str, str|Iterable[str]]): Query parameters to include in the
                 request.
         Return:
-            Deferred[tuple[str, dict, int]]: resolves to a tuple of
+            Deferred[tuple[str, FrozenEvent, int]]: resolves to a tuple of
             `(origin, event, event_format)` where origin is the remote
             homeserver which generated the event.
 
@@ -602,7 +610,7 @@ class FederationClient(FederationBase):
             pdu_dict.pop("origin_server_ts", None)
             pdu_dict.pop("unsigned", None)
 
-            builder = self.event_builder_factory.new(pdu_dict)
+            builder = self.event_builder_factory.new(room_version, pdu_dict)
             add_hashes_and_signatures(
                 builder,
                 self.hs.hostname,
@@ -618,7 +626,7 @@ class FederationClient(FederationBase):
             "make_" + membership, destinations, send_request,
         )
 
-    def send_join(self, destinations, pdu):
+    def send_join(self, destinations, pdu, event_format_version):
         """Sends a join event to one of a list of homeservers.
 
         Doing so will cause the remote server to add the event to the graph,
@@ -628,6 +636,7 @@ class FederationClient(FederationBase):
             destinations (str): Candidate homeservers which are probably
                 participating in the room.
             pdu (BaseEvent): event to be sent
+            event_format_version (int): The event format version
 
         Return:
             Deferred: resolves to a dict with members ``origin`` (a string
@@ -673,12 +682,12 @@ class FederationClient(FederationBase):
             logger.debug("Got content: %s", content)
 
             state = [
-                event_from_pdu_json(p, outlier=True)
+                event_from_pdu_json(p, event_format_version, outlier=True)
                 for p in content.get("state", [])
             ]
 
             auth_chain = [
-                event_from_pdu_json(p, outlier=True)
+                event_from_pdu_json(p, event_format_version, outlier=True)
                 for p in content.get("auth_chain", [])
             ]
 
@@ -755,7 +764,10 @@ class FederationClient(FederationBase):
 
         logger.debug("Got response to send_invite: %s", pdu_dict)
 
-        pdu = event_from_pdu_json(pdu_dict)
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
+        pdu = event_from_pdu_json(pdu_dict, format_ver)
 
         # Check signatures are correct.
         pdu = yield self._check_sigs_and_hash(pdu)
@@ -833,13 +845,14 @@ class FederationClient(FederationBase):
             content=send_content,
         )
 
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+
         auth_chain = [
-            event_from_pdu_json(e)
+            event_from_pdu_json(e, format_ver)
             for e in content["auth_chain"]
         ]
 
-        room_version = yield self.store.get_room_version(room_id)
-
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination, auth_chain, outlier=True, room_version=room_version,
         )
@@ -883,13 +896,14 @@ class FederationClient(FederationBase):
                 timeout=timeout,
             )
 
+            room_version = yield self.store.get_room_version(room_id)
+            format_ver = room_version_to_event_format(room_version)
+
             events = [
-                event_from_pdu_json(e)
+                event_from_pdu_json(e, format_ver)
                 for e in content.get("events", [])
             ]
 
-            room_version = yield self.store.get_room_version(room_id)
-
             signed_events = yield self._check_sigs_and_hash_and_fetch(
                 destination, events, outlier=False, room_version=room_version,
             )
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index dde166e295..4aa04b9588 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -34,6 +34,7 @@ from synapse.api.errors import (
     SynapseError,
 )
 from synapse.crypto.event_signing import compute_event_signature
+from synapse.events import room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.persistence import TransactionActions
 from synapse.federation.units import Edu, Transaction
@@ -178,14 +179,13 @@ class FederationServer(FederationBase):
                 continue
 
             try:
-                # In future we will actually use the room version to parse the
-                # PDU into an event.
-                yield self.store.get_room_version(room_id)
+                room_version = yield self.store.get_room_version(room_id)
+                format_ver = room_version_to_event_format(room_version)
             except NotFoundError:
                 logger.info("Ignoring PDU for unknown room_id: %s", room_id)
                 continue
 
-            event = event_from_pdu_json(p)
+            event = event_from_pdu_json(p, format_ver)
             pdus_by_room.setdefault(room_id, []).append(event)
 
         pdu_results = {}
@@ -370,7 +370,9 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, content, room_version):
-        pdu = event_from_pdu_json(content)
+        format_ver = room_version_to_event_format(room_version)
+
+        pdu = event_from_pdu_json(content, format_ver)
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
         ret_pdu = yield self.handler.on_invite_request(origin, pdu)
@@ -378,9 +380,12 @@ class FederationServer(FederationBase):
         defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
 
     @defer.inlineCallbacks
-    def on_send_join_request(self, origin, content):
+    def on_send_join_request(self, origin, content, room_id):
         logger.debug("on_send_join_request: content: %s", content)
-        pdu = event_from_pdu_json(content)
+
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+        pdu = event_from_pdu_json(content, format_ver)
 
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
@@ -410,9 +415,12 @@ class FederationServer(FederationBase):
         })
 
     @defer.inlineCallbacks
-    def on_send_leave_request(self, origin, content):
+    def on_send_leave_request(self, origin, content, room_id):
         logger.debug("on_send_leave_request: content: %s", content)
-        pdu = event_from_pdu_json(content)
+
+        room_version = yield self.store.get_room_version(room_id)
+        format_ver = room_version_to_event_format(room_version)
+        pdu = event_from_pdu_json(content, format_ver)
 
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
@@ -458,13 +466,14 @@ class FederationServer(FederationBase):
             origin_host, _ = parse_server_name(origin)
             yield self.check_server_matches_acl(origin_host, room_id)
 
+            room_version = yield self.store.get_room_version(room_id)
+            format_ver = room_version_to_event_format(room_version)
+
             auth_chain = [
-                event_from_pdu_json(e)
+                event_from_pdu_json(e, format_ver)
                 for e in content["auth_chain"]
             ]
 
-            room_version = yield self.store.get_room_version(room_id)
-
             signed_auth = yield self._check_sigs_and_hash_and_fetch(
                 origin, auth_chain, outlier=True, room_version=room_version,
             )
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 4557a9e66e..67ae0212c3 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -469,7 +469,7 @@ class FederationSendLeaveServlet(BaseFederationServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, origin, content, query, room_id, event_id):
-        content = yield self.handler.on_send_leave_request(origin, content)
+        content = yield self.handler.on_send_leave_request(origin, content, room_id)
         defer.returnValue((200, content))
 
 
@@ -487,7 +487,7 @@ class FederationSendJoinServlet(BaseFederationServlet):
     def on_PUT(self, origin, content, query, context, event_id):
         # TODO(paul): assert that context/event_id parsed from path actually
         #   match those given in content
-        content = yield self.handler.on_send_join_request(origin, content)
+        content = yield self.handler.on_send_join_request(origin, content, context)
         defer.returnValue((200, content))
 
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index d4d945030e..1d1d058db2 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1061,7 +1061,7 @@ class FederationHandler(BaseHandler):
         """
         logger.debug("Joining %s to %s", joinee, room_id)
 
-        origin, event = yield self._make_and_verify_event(
+        origin, event, event_format_version = yield self._make_and_verify_event(
             target_hosts,
             room_id,
             joinee,
@@ -1093,7 +1093,9 @@ class FederationHandler(BaseHandler):
                 target_hosts.insert(0, origin)
             except ValueError:
                 pass
-            ret = yield self.federation_client.send_join(target_hosts, event)
+            ret = yield self.federation_client.send_join(
+                target_hosts, event, event_format_version,
+            )
 
             origin = ret["origin"]
             state = ret["state"]
@@ -1166,13 +1168,18 @@ class FederationHandler(BaseHandler):
         """
         event_content = {"membership": Membership.JOIN}
 
-        builder = self.event_builder_factory.new({
-            "type": EventTypes.Member,
-            "content": event_content,
-            "room_id": room_id,
-            "sender": user_id,
-            "state_key": user_id,
-        })
+        room_version = yield self.store.get_room_version(room_id)
+
+        builder = self.event_builder_factory.new(
+            room_version,
+            {
+                "type": EventTypes.Member,
+                "content": event_content,
+                "room_id": room_id,
+                "sender": user_id,
+                "state_key": user_id,
+            }
+        )
 
         try:
             event, context = yield self.event_creation_handler.create_new_client_event(
@@ -1306,7 +1313,7 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
-        origin, event = yield self._make_and_verify_event(
+        origin, event, event_format_version = yield self._make_and_verify_event(
             target_hosts,
             room_id,
             user_id,
@@ -1338,7 +1345,7 @@ class FederationHandler(BaseHandler):
     @defer.inlineCallbacks
     def _make_and_verify_event(self, target_hosts, room_id, user_id, membership,
                                content={}, params=None):
-        origin, pdu, _ = yield self.federation_client.make_membership_event(
+        origin, event, format_ver = yield self.federation_client.make_membership_event(
             target_hosts,
             room_id,
             user_id,
@@ -1347,9 +1354,7 @@ class FederationHandler(BaseHandler):
             params=params,
         )
 
-        logger.debug("Got response to make_%s: %s", membership, pdu)
-
-        event = pdu
+        logger.debug("Got response to make_%s: %s", membership, event)
 
         # We should assert some things.
         # FIXME: Do this in a nicer way
@@ -1357,7 +1362,7 @@ class FederationHandler(BaseHandler):
         assert(event.user_id == user_id)
         assert(event.state_key == user_id)
         assert(event.room_id == room_id)
-        defer.returnValue((origin, event))
+        defer.returnValue((origin, event, format_ver))
 
     @defer.inlineCallbacks
     @log_function
@@ -1366,13 +1371,17 @@ class FederationHandler(BaseHandler):
         leave event for the room and return that. We do *not* persist or
         process it until the other server has signed it and sent it back.
         """
-        builder = self.event_builder_factory.new({
-            "type": EventTypes.Member,
-            "content": {"membership": Membership.LEAVE},
-            "room_id": room_id,
-            "sender": user_id,
-            "state_key": user_id,
-        })
+        room_version = yield self.store.get_room_version(room_id)
+        builder = self.event_builder_factory.new(
+            room_version,
+            {
+                "type": EventTypes.Member,
+                "content": {"membership": Membership.LEAVE},
+                "room_id": room_id,
+                "sender": user_id,
+                "state_key": user_id,
+            }
+        )
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
@@ -2263,14 +2272,16 @@ class FederationHandler(BaseHandler):
         }
 
         if (yield self.auth.check_host_in_room(room_id, self.hs.hostname)):
-            builder = self.event_builder_factory.new(event_dict)
+            room_version = yield self.store.get_room_version(room_id)
+            builder = self.event_builder_factory.new(room_version, event_dict)
+
             EventValidator().validate_new(builder)
             event, context = yield self.event_creation_handler.create_new_client_event(
                 builder=builder
             )
 
             event, context = yield self.add_display_name_to_third_party_invite(
-                event_dict, event, context
+                room_version, event_dict, event, context
             )
 
             try:
@@ -2301,14 +2312,18 @@ class FederationHandler(BaseHandler):
         Returns:
             Deferred: resolves (to None)
         """
-        builder = self.event_builder_factory.new(event_dict)
+        room_version = yield self.store.get_room_version(room_id)
+
+        # NB: event_dict has a particular specced format we might need to fudge
+        # if we change event formats too much.
+        builder = self.event_builder_factory.new(room_version, event_dict)
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
         )
 
         event, context = yield self.add_display_name_to_third_party_invite(
-            event_dict, event, context
+            room_version, event_dict, event, context
         )
 
         try:
@@ -2328,7 +2343,8 @@ class FederationHandler(BaseHandler):
         yield member_handler.send_membership_event(None, event, context)
 
     @defer.inlineCallbacks
-    def add_display_name_to_third_party_invite(self, event_dict, event, context):
+    def add_display_name_to_third_party_invite(self, room_version, event_dict,
+                                               event, context):
         key = (
             EventTypes.ThirdPartyInvite,
             event.content["third_party_invite"]["signed"]["token"]
@@ -2352,7 +2368,7 @@ class FederationHandler(BaseHandler):
             # auth checks. If we need the invite and don't have it then the
             # auth check code will explode appropriately.
 
-        builder = self.event_builder_factory.new(event_dict)
+        builder = self.event_builder_factory.new(room_version, event_dict)
         EventValidator().validate_new(builder)
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index a7cd779b02..7aaa4fba33 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -278,7 +278,15 @@ class EventCreationHandler(object):
         """
         yield self.auth.check_auth_blocking(requester.user.to_string())
 
-        builder = self.event_builder_factory.new(event_dict)
+        if event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "":
+            room_version = event_dict["content"]["room_version"]
+        else:
+            try:
+                room_version = yield self.store.get_room_version(event_dict["room_id"])
+            except NotFoundError:
+                raise AuthError(403, "Unknown room")
+
+        builder = self.event_builder_factory.new(room_version, event_dict)
 
         self.validator.validate_new(builder)
 

From 073173277cc249163cd4df2872abf928cf3cc5c7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 23 Jan 2019 20:27:31 +0000
Subject: [PATCH 19/35] Newsfile

---
 changelog.d/4451.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/4451.misc

diff --git a/changelog.d/4451.misc b/changelog.d/4451.misc
new file mode 100644
index 0000000000..43f8963614
--- /dev/null
+++ b/changelog.d/4451.misc
@@ -0,0 +1 @@
+Add infrastructure to support different event formats

From e79ba9eb34697cb6205c50282882b294c0acafa7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 09:28:16 +0000
Subject: [PATCH 20/35] Fix tests

---
 tests/storage/test_redaction.py  | 5 ++++-
 tests/storage/test_roommember.py | 3 ++-
 tests/storage/test_state.py      | 3 ++-
 tests/test_visibility.py         | 4 ++++
 tests/utils.py                   | 3 ++-
 5 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 02bf975fbf..3957561b1e 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -18,7 +18,7 @@ from mock import Mock
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, Membership
+from synapse.api.constants import EventTypes, Membership, RoomVersions
 from synapse.types import RoomID, UserID
 
 from tests import unittest
@@ -52,6 +52,7 @@ class RedactionTestCase(unittest.TestCase):
         content = {"membership": membership}
         content.update(extra_content)
         builder = self.event_builder_factory.new(
+            RoomVersions.V1,
             {
                 "type": EventTypes.Member,
                 "sender": user.to_string(),
@@ -74,6 +75,7 @@ class RedactionTestCase(unittest.TestCase):
         self.depth += 1
 
         builder = self.event_builder_factory.new(
+            RoomVersions.V1,
             {
                 "type": EventTypes.Message,
                 "sender": user.to_string(),
@@ -94,6 +96,7 @@ class RedactionTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def inject_redaction(self, room, event_id, user, reason):
         builder = self.event_builder_factory.new(
+            RoomVersions.V1,
             {
                 "type": EventTypes.Redaction,
                 "sender": user.to_string(),
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index 978c66133d..7fa2f4fd70 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -18,7 +18,7 @@ from mock import Mock
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, Membership
+from synapse.api.constants import EventTypes, Membership, RoomVersions
 from synapse.types import RoomID, UserID
 
 from tests import unittest
@@ -50,6 +50,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def inject_room_member(self, room, user, membership, replaces_state=None):
         builder = self.event_builder_factory.new(
+            RoomVersions.V1,
             {
                 "type": EventTypes.Member,
                 "sender": user.to_string(),
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index 086a39d834..a1f99134dc 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -17,7 +17,7 @@ import logging
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, Membership
+from synapse.api.constants import EventTypes, Membership, RoomVersions
 from synapse.storage.state import StateFilter
 from synapse.types import RoomID, UserID
 
@@ -52,6 +52,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
     @defer.inlineCallbacks
     def inject_state_event(self, room, sender, typ, state_key, content):
         builder = self.event_builder_factory.new(
+            RoomVersions.V1,
             {
                 "type": typ,
                 "sender": sender.to_string(),
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 2eea3b098b..82d63ce00e 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -17,6 +17,7 @@ import logging
 from twisted.internet import defer
 from twisted.internet.defer import succeed
 
+from synapse.api.constants import RoomVersions
 from synapse.events import FrozenEvent
 from synapse.visibility import filter_events_for_server
 
@@ -124,6 +125,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
     def inject_visibility(self, user_id, visibility):
         content = {"history_visibility": visibility}
         builder = self.event_builder_factory.new(
+            RoomVersions.V1,
             {
                 "type": "m.room.history_visibility",
                 "sender": user_id,
@@ -144,6 +146,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
         content = {"membership": membership}
         content.update(extra_content)
         builder = self.event_builder_factory.new(
+            RoomVersions.V1,
             {
                 "type": "m.room.member",
                 "sender": user_id,
@@ -165,6 +168,7 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
         if content is None:
             content = {"body": "testytest"}
         builder = self.event_builder_factory.new(
+            RoomVersions.V1,
             {
                 "type": "m.room.message",
                 "sender": user_id,
diff --git a/tests/utils.py b/tests/utils.py
index 08d6faa0a6..1aa8f98094 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -26,7 +26,7 @@ from six.moves.urllib import parse as urlparse
 
 from twisted.internet import defer, reactor
 
-from synapse.api.constants import EventTypes
+from synapse.api.constants import EventTypes, RoomVersions
 from synapse.api.errors import CodeMessageException, cs_error
 from synapse.config.server import ServerConfig
 from synapse.federation.transport import server
@@ -622,6 +622,7 @@ def create_room(hs, room_id, creator_id):
     event_creation_handler = hs.get_event_creation_handler()
 
     builder = event_builder_factory.new(
+        RoomVersions.V1,
         {
             "type": EventTypes.Create,
             "state_key": "",

From e8c9f1539716b9593de837343da5dc4fa9bc10b0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 11:14:07 +0000
Subject: [PATCH 21/35] Replace missed usages of FrozenEvent

---
 synapse/replication/http/federation.py | 8 ++++++--
 synapse/replication/http/send_event.py | 8 ++++++--
 synapse/storage/events_worker.py       | 8 ++------
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index 64a79da162..2e16c69666 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -17,7 +17,7 @@ import logging
 
 from twisted.internet import defer
 
-from synapse.events import FrozenEvent
+from synapse.events import event_type_from_format_version
 from synapse.events.snapshot import EventContext
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
@@ -70,6 +70,7 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint):
 
             event_payloads.append({
                 "event": event.get_pdu_json(),
+                "event_format_version": event.format_version,
                 "internal_metadata": event.internal_metadata.get_dict(),
                 "rejected_reason": event.rejected_reason,
                 "context": serialized_context,
@@ -94,9 +95,12 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint):
             event_and_contexts = []
             for event_payload in event_payloads:
                 event_dict = event_payload["event"]
+                format_ver = content["event_format_version"]
                 internal_metadata = event_payload["internal_metadata"]
                 rejected_reason = event_payload["rejected_reason"]
-                event = FrozenEvent(event_dict, internal_metadata, rejected_reason)
+
+                EventType = event_type_from_format_version(format_ver)
+                event = EventType(event_dict, internal_metadata, rejected_reason)
 
                 context = yield EventContext.deserialize(
                     self.store, event_payload["context"],
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index 5b52c91650..3635015eda 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -17,7 +17,7 @@ import logging
 
 from twisted.internet import defer
 
-from synapse.events import FrozenEvent
+from synapse.events import event_type_from_format_version
 from synapse.events.snapshot import EventContext
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
@@ -74,6 +74,7 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint):
 
         payload = {
             "event": event.get_pdu_json(),
+            "event_format_version": event.format_version,
             "internal_metadata": event.internal_metadata.get_dict(),
             "rejected_reason": event.rejected_reason,
             "context": serialized_context,
@@ -90,9 +91,12 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint):
             content = parse_json_object_from_request(request)
 
             event_dict = content["event"]
+            format_ver = content["event_format_version"]
             internal_metadata = content["internal_metadata"]
             rejected_reason = content["rejected_reason"]
-            event = FrozenEvent(event_dict, internal_metadata, rejected_reason)
+
+            EventType = event_type_from_format_version(format_ver)
+            event = EventType(event_dict, internal_metadata, rejected_reason)
 
             requester = Requester.deserialize(self.store, content["requester"])
             context = yield EventContext.deserialize(self.store, content["context"])
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 599f892858..0a0ca58fc4 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -23,7 +23,7 @@ from twisted.internet import defer
 
 from synapse.api.constants import EventFormatVersions
 from synapse.api.errors import NotFoundError
-from synapse.events import FrozenEvent
+from synapse.events import FrozenEvent, event_type_from_format_version  # noqa: F401
 # these are only included to make the type annotations work
 from synapse.events.snapshot import EventContext  # noqa: F401
 from synapse.events.utils import prune_event
@@ -412,11 +412,7 @@ class EventsWorkerStore(SQLBaseStore):
                 # of a event format version, so it must be a V1 event.
                 format_version = EventFormatVersions.V1
 
-            # TODO: When we implement new event formats we'll need to use a
-            # different event python type
-            assert format_version == EventFormatVersions.V1
-
-            original_ev = FrozenEvent(
+            original_ev = event_type_from_format_version(format_version)(
                 event_dict=d,
                 internal_metadata_dict=internal_metadata,
                 rejected_reason=rejected_reason,

From 8ea509a9357d53f71e7bef09aae59f53b9f2317e Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 24 Jan 2019 17:21:35 +0000
Subject: [PATCH 22/35] Update synapse/api/filtering.py

Co-Authored-By: anoadragon453 <1342360+anoadragon453@users.noreply.github.com>
---
 synapse/api/filtering.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py
index f3a056110f..3906475403 100644
--- a/synapse/api/filtering.py
+++ b/synapse/api/filtering.py
@@ -448,7 +448,7 @@ class Filter(object):
         """Returns a new filter with the given room IDs appended.
 
         Args:
-            room_ids (list): A list of room_ids.
+            room_ids (iterable[unicode]): The room_ids to add
 
         Returns:
             filter: A new filter including the given rooms and the old

From 03c85335d1d386c0523af3b6bf992f83bfb905d7 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 24 Jan 2019 17:22:09 +0000
Subject: [PATCH 23/35] Apply suggestions from code review

Co-Authored-By: anoadragon453 <1342360+anoadragon453@users.noreply.github.com>
---
 synapse/handlers/search.py | 2 +-
 synapse/storage/state.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index 75c26fe065..49c439313e 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -52,7 +52,7 @@ class SearchHandler(BaseHandler):
             room_id (str): id of the room to search through.
 
         Returns:
-            Deferred[iterable[str]]: predecessor room ids
+            Deferred[iterable[unicode]]: predecessor room ids
         """
 
         historical_room_ids = []
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index fceb9744aa..0a0691cd00 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -446,7 +446,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
             room_id (str)
 
         Returns:
-            Deferred[str]: predecessor room id
+            Deferred[unicode|None]: predecessor room id
         """
         state_ids = yield self.get_current_state_ids(room_id)
         create_id = state_ids.get((EventTypes.Create, ""))

From e1781b043b4a73e1b22142b6e09c07ddd782ae57 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <andrew@amorgan.xyz>
Date: Thu, 24 Jan 2019 17:23:39 +0000
Subject: [PATCH 24/35] Remove redundant create event None check

---
 synapse/storage/state.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index fceb9744aa..c02130d9d6 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -458,9 +458,6 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         # Retrieve the room's create event
         create_event = yield self.get_event(create_id)
 
-        if not create_event:
-            defer.returnValue(None)
-
         # Return predecessor if present
         defer.returnValue(create_event.content.get("predecessor", None))
 

From b8082a54451bb4db30e3b2a4d19dc8cb23330eb7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 17:33:19 +0000
Subject: [PATCH 25/35] Use term 'out of band membership' instead

---
 synapse/events/__init__.py     | 9 +++++----
 synapse/handlers/federation.py | 4 ++--
 synapse/storage/roommember.py  | 6 +++---
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 5030636c7e..48289cad06 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -41,12 +41,13 @@ class _EventInternalMetadata(object):
     def is_outlier(self):
         return getattr(self, "outlier", False)
 
-    def is_new_remote_event(self):
-        """Whether this is a new remote event, like an invite or an invite
+    def is_out_of_band_membership(self):
+        """Whether this is an out of band membership, like an invite or an invite
         rejection. This is needed as those events are marked as outliers, but
-        they still need to be processed.
+        they still need to be processed as if they're new events (e.g. updating
+        invite state in the database, relaying to clients, etc).
         """
-        return getattr(self, "new_remote_event", False)
+        return getattr(self, "out_of_band_membership", False)
 
     def get_send_on_behalf_of(self):
         """Whether this server should send the event on behalf of another server.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index e017cab777..242719b7ce 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1284,7 +1284,7 @@ class FederationHandler(BaseHandler):
             )
 
         event.internal_metadata.outlier = True
-        event.internal_metadata.new_remote_event = True
+        event.internal_metadata.out_of_band_membership = True
 
         event.signatures.update(
             compute_event_signature(
@@ -1310,7 +1310,7 @@ class FederationHandler(BaseHandler):
         # Mark as outlier as we don't have any state for this event; we're not
         # even in the room.
         event.internal_metadata.outlier = True
-        event.internal_metadata.new_remote_event = True
+        event.internal_metadata.out_of_band_membership = True
 
         # Try the host that we succesfully called /make_leave/ on first for
         # the /send_leave/ request.
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 40b13de80b..592c1bcd33 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -589,11 +589,11 @@ class RoomMemberStore(RoomMemberWorkerStore):
 
             # We update the local_invites table only if the event is "current",
             # i.e., its something that has just happened. If the event is an
-            # outlier it is only current if its a "new remote event", like a
-            # remote invite or a rejection of a remote invite.
+            # outlier it is only current if its an "out of band membership",
+            # like a remote invite or a rejection of a remote invite.
             is_new_state = not backfilled and (
                 not event.internal_metadata.is_outlier()
-                or event.internal_metadata.is_new_remote_event()
+                or event.internal_metadata.is_out_of_band_membership()
             )
             is_mine = self.hs.is_mine_id(event.state_key)
             if is_new_state and is_mine:

From 9139b87be420fcfce22e70a7c35ba52b2ea32f3a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 18:04:02 +0000
Subject: [PATCH 26/35] Remove unecessary setting of outlier bit

---
 synapse/handlers/federation.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 242719b7ce..d53b716ffb 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1079,8 +1079,6 @@ class FederationHandler(BaseHandler):
         handled_events = set()
 
         try:
-            event.internal_metadata.outlier = False
-
             # Try the host we successfully got a response to /make_join/
             # request first.
             try:

From 5ee1f997a8e7177077e2c5f0750e28725a452791 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 18:08:08 +0000
Subject: [PATCH 27/35] Update make_membership_event docs

---
 synapse/federation/federation_client.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index f4adcb556d..df7d18700c 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -524,6 +524,8 @@ class FederationClient(FederationBase):
         Does so by asking one of the already participating servers to create an
         event with proper context.
 
+        Returns a fully signed and hashed event.
+
         Note that this does not append any events to any graphs.
 
         Args:
@@ -538,8 +540,9 @@ class FederationClient(FederationBase):
             params (dict[str, str|Iterable[str]]): Query parameters to include in the
                 request.
         Return:
-            Deferred: resolves to a tuple of (origin (str), event (object))
-            where origin is the remote homeserver which generated the event.
+            Deferred[tuple[str, FrozenEvent]]: resolves to a tuple of `origin`
+            and event where origin is the remote homeserver which generated
+            the event.
 
             Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.

From 26f44164c82c89c08f7ad930f365dbf69759e317 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 18:28:00 +0000
Subject: [PATCH 28/35] Review comments

---
 synapse/events/__init__.py              | 3 ++-
 synapse/federation/federation_client.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index e6f94e68af..154a7c0198 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -247,6 +247,7 @@ def room_version_to_event_format(room_version):
         int
     """
     if room_version not in KNOWN_ROOM_VERSIONS:
-        raise
+        # We should have already checked version, so this should not happen
+        raise RuntimeError("Unrecognized room version %s" % (room_version,))
 
     return EventFormatVersions.V1
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 0757ad12f4..c43f21a0ba 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -543,7 +543,8 @@ class FederationClient(FederationBase):
         Return:
             Deferred[tuple[str, dict, int]]: resolves to a tuple of
             `(origin, event, event_format)` where origin is the remote
-            homeserver which generated the event.
+            homeserver which generated the event, and event_format is one of
+            `synapse.api.constants.EventFormatVersions`.
 
             Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.

From d148c43050f7a85523a743ff6069683c644a517d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 24 Jan 2019 18:31:23 +0000
Subject: [PATCH 29/35] Review comments

---
 synapse/federation/federation_client.py | 3 ++-
 synapse/handlers/federation.py          | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 4e171f9b56..7fb5736142 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -674,7 +674,8 @@ class FederationClient(FederationBase):
                     break
 
             if room_version is None:
-                # We use this error has that is what
+                # If the state doesn't have a create event then the room is
+                # invalid, and it would fail auth checks anyway.
                 raise SynapseError(400, "No create event in state")
 
             valid_pdus = yield self._check_sigs_and_hash_and_fetch(
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a9dc4a4e4e..5280d88a50 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1664,6 +1664,11 @@ class FederationHandler(BaseHandler):
                 create_event = e
                 break
 
+        if create_event is None:
+            # If the state doesn't have a create event then the room is
+            # invalid, and it would fail auth checks anyway.
+            raise SynapseError(400, "No create event in state")
+
         room_version = create_event.content.get("room_version", RoomVersions.V1)
 
         missing_auth_events = set()

From be6a7e47fa75d72466a356c254376b8eb0707bb2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erikj@jki.re>
Date: Fri, 25 Jan 2019 10:23:51 +0000
Subject: [PATCH 30/35] Revert "Require event format version to parse or create
 events"

---
 changelog.d/4447.misc                   |  1 -
 changelog.d/4451.misc                   |  1 -
 synapse/events/__init__.py              | 39 +------------
 synapse/events/builder.py               | 51 +----------------
 synapse/federation/federation_base.py   |  9 ++-
 synapse/federation/federation_client.py | 74 +++++++++----------------
 synapse/federation/federation_server.py | 41 +++++---------
 synapse/federation/transport/server.py  |  4 +-
 synapse/handlers/federation.py          | 72 ++++++++++--------------
 synapse/handlers/message.py             | 10 +---
 synapse/replication/http/federation.py  |  8 +--
 synapse/replication/http/send_event.py  |  8 +--
 synapse/storage/events_worker.py        |  8 ++-
 tests/storage/test_redaction.py         |  5 +-
 tests/storage/test_roommember.py        |  3 +-
 tests/storage/test_state.py             |  3 +-
 tests/test_visibility.py                |  4 --
 tests/utils.py                          |  3 +-
 18 files changed, 91 insertions(+), 253 deletions(-)
 delete mode 100644 changelog.d/4447.misc
 delete mode 100644 changelog.d/4451.misc

diff --git a/changelog.d/4447.misc b/changelog.d/4447.misc
deleted file mode 100644
index 43f8963614..0000000000
--- a/changelog.d/4447.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add infrastructure to support different event formats
diff --git a/changelog.d/4451.misc b/changelog.d/4451.misc
deleted file mode 100644
index 43f8963614..0000000000
--- a/changelog.d/4451.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add infrastructure to support different event formats
diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index c3e6caf597..888296933b 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -18,11 +18,7 @@ from distutils.util import strtobool
 
 import six
 
-from synapse.api.constants import (
-    KNOWN_EVENT_FORMAT_VERSIONS,
-    KNOWN_ROOM_VERSIONS,
-    EventFormatVersions,
-)
+from synapse.api.constants import EventFormatVersions
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 
@@ -244,36 +240,3 @@ class FrozenEvent(EventBase):
             self.get("type", None),
             self.get("state_key", None),
         )
-
-
-def room_version_to_event_format(room_version):
-    """Converts a room version string to the event format
-
-    Args:
-        room_version (str)
-
-    Returns:
-        int
-    """
-    if room_version not in KNOWN_ROOM_VERSIONS:
-        raise
-
-    return EventFormatVersions.V1
-
-
-def event_type_from_format_version(format_version):
-    """Returns the python type to use to construct an Event object for the
-    given event format version.
-
-    Args:
-        format_version (int): The event format version
-
-    Returns:
-        type: A type that can be initialized as per the initializer of
-        `FrozenEvent`
-    """
-    if format_version not in KNOWN_EVENT_FORMAT_VERSIONS:
-        raise Exception(
-            "No event format %r" % (format_version,)
-        )
-    return FrozenEvent
diff --git a/synapse/events/builder.py b/synapse/events/builder.py
index 7e63371095..e662eaef10 100644
--- a/synapse/events/builder.py
+++ b/synapse/events/builder.py
@@ -15,39 +15,12 @@
 
 import copy
 
-from synapse.api.constants import RoomVersions
 from synapse.types import EventID
 from synapse.util.stringutils import random_string
 
 from . import EventBase, FrozenEvent, _event_dict_property
 
 
-def get_event_builder(room_version, key_values={}, internal_metadata_dict={}):
-    """Generate an event builder appropriate for the given room version
-
-    Args:
-        room_version (str): Version of the room that we're creating an
-            event builder for
-        key_values (dict): Fields used as the basis of the new event
-        internal_metadata_dict (dict): Used to create the `_EventInternalMetadata`
-            object.
-
-    Returns:
-        EventBuilder
-    """
-    if room_version in {
-        RoomVersions.V1,
-        RoomVersions.V2,
-        RoomVersions.VDH_TEST,
-        RoomVersions.STATE_V2_TEST,
-    }:
-        return EventBuilder(key_values, internal_metadata_dict)
-    else:
-        raise Exception(
-            "No event format defined for version %r" % (room_version,)
-        )
-
-
 class EventBuilder(EventBase):
     def __init__(self, key_values={}, internal_metadata_dict={}):
         signatures = copy.deepcopy(key_values.pop("signatures", {}))
@@ -85,29 +58,7 @@ class EventBuilderFactory(object):
 
         return e_id.to_string()
 
-    def new(self, room_version, key_values={}):
-        """Generate an event builder appropriate for the given room version
-
-        Args:
-            room_version (str): Version of the room that we're creating an
-                event builder for
-            key_values (dict): Fields used as the basis of the new event
-
-        Returns:
-            EventBuilder
-        """
-
-        # There's currently only the one event version defined
-        if room_version not in {
-            RoomVersions.V1,
-            RoomVersions.V2,
-            RoomVersions.VDH_TEST,
-            RoomVersions.STATE_V2_TEST,
-        }:
-            raise Exception(
-                "No event format defined for version %r" % (room_version,)
-            )
-
+    def new(self, key_values={}):
         key_values["event_id"] = self.create_event_id()
 
         time_now = int(self.clock.time_msec())
diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py
index 5c31e5f85f..d749bfdd3a 100644
--- a/synapse/federation/federation_base.py
+++ b/synapse/federation/federation_base.py
@@ -23,7 +23,7 @@ from twisted.internet.defer import DeferredList
 from synapse.api.constants import MAX_DEPTH, EventTypes, Membership
 from synapse.api.errors import Codes, SynapseError
 from synapse.crypto.event_signing import check_event_content_hash
-from synapse.events import event_type_from_format_version
+from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 from synapse.http.servlet import assert_params_in_dict
 from synapse.types import get_domain_from_id
@@ -302,12 +302,11 @@ def _is_invite_via_3pid(event):
     )
 
 
-def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
+def event_from_pdu_json(pdu_json, outlier=False):
     """Construct a FrozenEvent from an event json received over federation
 
     Args:
         pdu_json (object): pdu as received over federation
-        event_format_version (int): The event format version
         outlier (bool): True to mark this event as an outlier
 
     Returns:
@@ -331,8 +330,8 @@ def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
     elif depth > MAX_DEPTH:
         raise SynapseError(400, "Depth too large", Codes.BAD_JSON)
 
-    event = event_type_from_format_version(event_format_version)(
-        pdu_json,
+    event = FrozenEvent(
+        pdu_json
     )
 
     event.internal_metadata.outlier = outlier
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 4b25f891ca..777deabdf7 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -38,7 +38,6 @@ from synapse.api.errors import (
     SynapseError,
 )
 from synapse.crypto.event_signing import add_hashes_and_signatures
-from synapse.events import room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.util import logcontext, unwrapFirstError
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -170,13 +169,13 @@ class FederationClient(FederationBase):
 
     @defer.inlineCallbacks
     @log_function
-    def backfill(self, dest, room_id, limit, extremities):
+    def backfill(self, dest, context, limit, extremities):
         """Requests some more historic PDUs for the given context from the
         given destination server.
 
         Args:
             dest (str): The remote home server to ask.
-            room_id (str): The room_id to backfill.
+            context (str): The context to backfill.
             limit (int): The maximum number of PDUs to return.
             extremities (list): List of PDU id and origins of the first pdus
                 we have seen from the context
@@ -191,15 +190,12 @@ class FederationClient(FederationBase):
             return
 
         transaction_data = yield self.transport_layer.backfill(
-            dest, room_id, extremities, limit)
+            dest, context, extremities, limit)
 
         logger.debug("backfill transaction_data=%s", repr(transaction_data))
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
         pdus = [
-            event_from_pdu_json(p, format_ver, outlier=False)
+            event_from_pdu_json(p, outlier=False)
             for p in transaction_data["pdus"]
         ]
 
@@ -243,8 +239,6 @@ class FederationClient(FederationBase):
 
         pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {})
 
-        format_ver = room_version_to_event_format(room_version)
-
         signed_pdu = None
         for destination in destinations:
             now = self._clock.time_msec()
@@ -260,7 +254,7 @@ class FederationClient(FederationBase):
                 logger.debug("transaction_data %r", transaction_data)
 
                 pdu_list = [
-                    event_from_pdu_json(p, format_ver, outlier=outlier)
+                    event_from_pdu_json(p, outlier=outlier)
                     for p in transaction_data["pdus"]
                 ]
 
@@ -354,16 +348,12 @@ class FederationClient(FederationBase):
             destination, room_id, event_id=event_id,
         )
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
         pdus = [
-            event_from_pdu_json(p, format_ver, outlier=True)
-            for p in result["pdus"]
+            event_from_pdu_json(p, outlier=True) for p in result["pdus"]
         ]
 
         auth_chain = [
-            event_from_pdu_json(p, format_ver, outlier=True)
+            event_from_pdu_json(p, outlier=True)
             for p in result.get("auth_chain", [])
         ]
 
@@ -371,6 +361,8 @@ class FederationClient(FederationBase):
             ev.event_id for ev in itertools.chain(pdus, auth_chain)
         ])
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_pdus = yield self._check_sigs_and_hash_and_fetch(
             destination,
             [p for p in pdus if p.event_id not in seen_events],
@@ -469,14 +461,13 @@ class FederationClient(FederationBase):
             destination, room_id, event_id,
         )
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
         auth_chain = [
-            event_from_pdu_json(p, format_ver, outlier=True)
+            event_from_pdu_json(p, outlier=True)
             for p in res["auth_chain"]
         ]
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination, auth_chain,
             outlier=True, room_version=room_version,
@@ -566,9 +557,9 @@ class FederationClient(FederationBase):
             params (dict[str, str|Iterable[str]]): Query parameters to include in the
                 request.
         Return:
-            Deferred[tuple[str, FrozenEvent, int]]: resolves to a tuple of
-            `(origin, event, event_format)` where origin is the remote
-            homeserver which generated the event.
+            Deferred[tuple[str, FrozenEvent]]: resolves to a tuple of `origin`
+            and event where origin is the remote homeserver which generated
+            the event.
 
             Fails with a ``SynapseError`` if the chosen remote server
             returns a 300/400 code.
@@ -588,11 +579,6 @@ class FederationClient(FederationBase):
                 destination, room_id, user_id, membership, params,
             )
 
-            # Note: If not supplied, the room version may be either v1 or v2,
-            # however either way the event format version will be v1.
-            room_version = ret.get("room_version", RoomVersions.V1)
-            event_format = room_version_to_event_format(room_version)
-
             pdu_dict = ret.get("event", None)
             if not isinstance(pdu_dict, dict):
                 raise InvalidResponseError("Bad 'event' field in response")
@@ -612,7 +598,7 @@ class FederationClient(FederationBase):
             pdu_dict.pop("origin_server_ts", None)
             pdu_dict.pop("unsigned", None)
 
-            builder = self.event_builder_factory.new(room_version, pdu_dict)
+            builder = self.event_builder_factory.new(pdu_dict)
             add_hashes_and_signatures(
                 builder,
                 self.hs.hostname,
@@ -621,14 +607,14 @@ class FederationClient(FederationBase):
             ev = builder.build()
 
             defer.returnValue(
-                (destination, ev, event_format)
+                (destination, ev)
             )
 
         return self._try_destination_list(
             "make_" + membership, destinations, send_request,
         )
 
-    def send_join(self, destinations, pdu, event_format_version):
+    def send_join(self, destinations, pdu):
         """Sends a join event to one of a list of homeservers.
 
         Doing so will cause the remote server to add the event to the graph,
@@ -638,7 +624,6 @@ class FederationClient(FederationBase):
             destinations (str): Candidate homeservers which are probably
                 participating in the room.
             pdu (BaseEvent): event to be sent
-            event_format_version (int): The event format version
 
         Return:
             Deferred: resolves to a dict with members ``origin`` (a string
@@ -684,12 +669,12 @@ class FederationClient(FederationBase):
             logger.debug("Got content: %s", content)
 
             state = [
-                event_from_pdu_json(p, event_format_version, outlier=True)
+                event_from_pdu_json(p, outlier=True)
                 for p in content.get("state", [])
             ]
 
             auth_chain = [
-                event_from_pdu_json(p, event_format_version, outlier=True)
+                event_from_pdu_json(p, outlier=True)
                 for p in content.get("auth_chain", [])
             ]
 
@@ -767,10 +752,7 @@ class FederationClient(FederationBase):
 
         logger.debug("Got response to send_invite: %s", pdu_dict)
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
-        pdu = event_from_pdu_json(pdu_dict, format_ver)
+        pdu = event_from_pdu_json(pdu_dict)
 
         # Check signatures are correct.
         pdu = yield self._check_sigs_and_hash(pdu)
@@ -848,14 +830,13 @@ class FederationClient(FederationBase):
             content=send_content,
         )
 
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-
         auth_chain = [
-            event_from_pdu_json(e, format_ver)
+            event_from_pdu_json(e)
             for e in content["auth_chain"]
         ]
 
+        room_version = yield self.store.get_room_version(room_id)
+
         signed_auth = yield self._check_sigs_and_hash_and_fetch(
             destination, auth_chain, outlier=True, room_version=room_version,
         )
@@ -899,14 +880,13 @@ class FederationClient(FederationBase):
                 timeout=timeout,
             )
 
-            room_version = yield self.store.get_room_version(room_id)
-            format_ver = room_version_to_event_format(room_version)
-
             events = [
-                event_from_pdu_json(e, format_ver)
+                event_from_pdu_json(e)
                 for e in content.get("events", [])
             ]
 
+            room_version = yield self.store.get_room_version(room_id)
+
             signed_events = yield self._check_sigs_and_hash_and_fetch(
                 destination, events, outlier=False, room_version=room_version,
             )
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index 4aa04b9588..cb729c69ea 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -34,7 +34,6 @@ from synapse.api.errors import (
     SynapseError,
 )
 from synapse.crypto.event_signing import compute_event_signature
-from synapse.events import room_version_to_event_format
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.persistence import TransactionActions
 from synapse.federation.units import Edu, Transaction
@@ -179,13 +178,14 @@ class FederationServer(FederationBase):
                 continue
 
             try:
-                room_version = yield self.store.get_room_version(room_id)
-                format_ver = room_version_to_event_format(room_version)
+                # In future we will actually use the room version to parse the
+                # PDU into an event.
+                yield self.store.get_room_version(room_id)
             except NotFoundError:
                 logger.info("Ignoring PDU for unknown room_id: %s", room_id)
                 continue
 
-            event = event_from_pdu_json(p, format_ver)
+            event = event_from_pdu_json(p)
             pdus_by_room.setdefault(room_id, []).append(event)
 
         pdu_results = {}
@@ -370,9 +370,7 @@ class FederationServer(FederationBase):
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, content, room_version):
-        format_ver = room_version_to_event_format(room_version)
-
-        pdu = event_from_pdu_json(content, format_ver)
+        pdu = event_from_pdu_json(content)
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
         ret_pdu = yield self.handler.on_invite_request(origin, pdu)
@@ -380,12 +378,9 @@ class FederationServer(FederationBase):
         defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
 
     @defer.inlineCallbacks
-    def on_send_join_request(self, origin, content, room_id):
+    def on_send_join_request(self, origin, content):
         logger.debug("on_send_join_request: content: %s", content)
-
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-        pdu = event_from_pdu_json(content, format_ver)
+        pdu = event_from_pdu_json(content)
 
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
@@ -405,22 +400,13 @@ class FederationServer(FederationBase):
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, room_id)
         pdu = yield self.handler.on_make_leave_request(room_id, user_id)
-
-        room_version = yield self.store.get_room_version(room_id)
-
         time_now = self._clock.time_msec()
-        defer.returnValue({
-            "event": pdu.get_pdu_json(time_now),
-            "room_version": room_version,
-        })
+        defer.returnValue({"event": pdu.get_pdu_json(time_now)})
 
     @defer.inlineCallbacks
-    def on_send_leave_request(self, origin, content, room_id):
+    def on_send_leave_request(self, origin, content):
         logger.debug("on_send_leave_request: content: %s", content)
-
-        room_version = yield self.store.get_room_version(room_id)
-        format_ver = room_version_to_event_format(room_version)
-        pdu = event_from_pdu_json(content, format_ver)
+        pdu = event_from_pdu_json(content)
 
         origin_host, _ = parse_server_name(origin)
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
@@ -466,14 +452,13 @@ class FederationServer(FederationBase):
             origin_host, _ = parse_server_name(origin)
             yield self.check_server_matches_acl(origin_host, room_id)
 
-            room_version = yield self.store.get_room_version(room_id)
-            format_ver = room_version_to_event_format(room_version)
-
             auth_chain = [
-                event_from_pdu_json(e, format_ver)
+                event_from_pdu_json(e)
                 for e in content["auth_chain"]
             ]
 
+            room_version = yield self.store.get_room_version(room_id)
+
             signed_auth = yield self._check_sigs_and_hash_and_fetch(
                 origin, auth_chain, outlier=True, room_version=room_version,
             )
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 67ae0212c3..4557a9e66e 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -469,7 +469,7 @@ class FederationSendLeaveServlet(BaseFederationServlet):
 
     @defer.inlineCallbacks
     def on_PUT(self, origin, content, query, room_id, event_id):
-        content = yield self.handler.on_send_leave_request(origin, content, room_id)
+        content = yield self.handler.on_send_leave_request(origin, content)
         defer.returnValue((200, content))
 
 
@@ -487,7 +487,7 @@ class FederationSendJoinServlet(BaseFederationServlet):
     def on_PUT(self, origin, content, query, context, event_id):
         # TODO(paul): assert that context/event_id parsed from path actually
         #   match those given in content
-        content = yield self.handler.on_send_join_request(origin, content, context)
+        content = yield self.handler.on_send_join_request(origin, content)
         defer.returnValue((200, content))
 
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index a4b771049c..453d393ce1 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1061,7 +1061,7 @@ class FederationHandler(BaseHandler):
         """
         logger.debug("Joining %s to %s", joinee, room_id)
 
-        origin, event, event_format_version = yield self._make_and_verify_event(
+        origin, event = yield self._make_and_verify_event(
             target_hosts,
             room_id,
             joinee,
@@ -1091,9 +1091,7 @@ class FederationHandler(BaseHandler):
                 target_hosts.insert(0, origin)
             except ValueError:
                 pass
-            ret = yield self.federation_client.send_join(
-                target_hosts, event, event_format_version,
-            )
+            ret = yield self.federation_client.send_join(target_hosts, event)
 
             origin = ret["origin"]
             state = ret["state"]
@@ -1166,18 +1164,13 @@ class FederationHandler(BaseHandler):
         """
         event_content = {"membership": Membership.JOIN}
 
-        room_version = yield self.store.get_room_version(room_id)
-
-        builder = self.event_builder_factory.new(
-            room_version,
-            {
-                "type": EventTypes.Member,
-                "content": event_content,
-                "room_id": room_id,
-                "sender": user_id,
-                "state_key": user_id,
-            }
-        )
+        builder = self.event_builder_factory.new({
+            "type": EventTypes.Member,
+            "content": event_content,
+            "room_id": room_id,
+            "sender": user_id,
+            "state_key": user_id,
+        })
 
         try:
             event, context = yield self.event_creation_handler.create_new_client_event(
@@ -1311,7 +1304,7 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
-        origin, event, event_format_version = yield self._make_and_verify_event(
+        origin, event = yield self._make_and_verify_event(
             target_hosts,
             room_id,
             user_id,
@@ -1343,7 +1336,7 @@ class FederationHandler(BaseHandler):
     @defer.inlineCallbacks
     def _make_and_verify_event(self, target_hosts, room_id, user_id, membership,
                                content={}, params=None):
-        origin, event, format_ver = yield self.federation_client.make_membership_event(
+        origin, pdu = yield self.federation_client.make_membership_event(
             target_hosts,
             room_id,
             user_id,
@@ -1352,7 +1345,9 @@ class FederationHandler(BaseHandler):
             params=params,
         )
 
-        logger.debug("Got response to make_%s: %s", membership, event)
+        logger.debug("Got response to make_%s: %s", membership, pdu)
+
+        event = pdu
 
         # We should assert some things.
         # FIXME: Do this in a nicer way
@@ -1360,7 +1355,7 @@ class FederationHandler(BaseHandler):
         assert(event.user_id == user_id)
         assert(event.state_key == user_id)
         assert(event.room_id == room_id)
-        defer.returnValue((origin, event, format_ver))
+        defer.returnValue((origin, event))
 
     @defer.inlineCallbacks
     @log_function
@@ -1369,17 +1364,13 @@ class FederationHandler(BaseHandler):
         leave event for the room and return that. We do *not* persist or
         process it until the other server has signed it and sent it back.
         """
-        room_version = yield self.store.get_room_version(room_id)
-        builder = self.event_builder_factory.new(
-            room_version,
-            {
-                "type": EventTypes.Member,
-                "content": {"membership": Membership.LEAVE},
-                "room_id": room_id,
-                "sender": user_id,
-                "state_key": user_id,
-            }
-        )
+        builder = self.event_builder_factory.new({
+            "type": EventTypes.Member,
+            "content": {"membership": Membership.LEAVE},
+            "room_id": room_id,
+            "sender": user_id,
+            "state_key": user_id,
+        })
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
@@ -2275,16 +2266,14 @@ class FederationHandler(BaseHandler):
         }
 
         if (yield self.auth.check_host_in_room(room_id, self.hs.hostname)):
-            room_version = yield self.store.get_room_version(room_id)
-            builder = self.event_builder_factory.new(room_version, event_dict)
-
+            builder = self.event_builder_factory.new(event_dict)
             EventValidator().validate_new(builder)
             event, context = yield self.event_creation_handler.create_new_client_event(
                 builder=builder
             )
 
             event, context = yield self.add_display_name_to_third_party_invite(
-                room_version, event_dict, event, context
+                event_dict, event, context
             )
 
             try:
@@ -2315,18 +2304,14 @@ class FederationHandler(BaseHandler):
         Returns:
             Deferred: resolves (to None)
         """
-        room_version = yield self.store.get_room_version(room_id)
-
-        # NB: event_dict has a particular specced format we might need to fudge
-        # if we change event formats too much.
-        builder = self.event_builder_factory.new(room_version, event_dict)
+        builder = self.event_builder_factory.new(event_dict)
 
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
         )
 
         event, context = yield self.add_display_name_to_third_party_invite(
-            room_version, event_dict, event, context
+            event_dict, event, context
         )
 
         try:
@@ -2346,8 +2331,7 @@ class FederationHandler(BaseHandler):
         yield member_handler.send_membership_event(None, event, context)
 
     @defer.inlineCallbacks
-    def add_display_name_to_third_party_invite(self, room_version, event_dict,
-                                               event, context):
+    def add_display_name_to_third_party_invite(self, event_dict, event, context):
         key = (
             EventTypes.ThirdPartyInvite,
             event.content["third_party_invite"]["signed"]["token"]
@@ -2371,7 +2355,7 @@ class FederationHandler(BaseHandler):
             # auth checks. If we need the invite and don't have it then the
             # auth check code will explode appropriately.
 
-        builder = self.event_builder_factory.new(room_version, event_dict)
+        builder = self.event_builder_factory.new(event_dict)
         EventValidator().validate_new(builder)
         event, context = yield self.event_creation_handler.create_new_client_event(
             builder=builder,
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 7aaa4fba33..a7cd779b02 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -278,15 +278,7 @@ class EventCreationHandler(object):
         """
         yield self.auth.check_auth_blocking(requester.user.to_string())
 
-        if event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "":
-            room_version = event_dict["content"]["room_version"]
-        else:
-            try:
-                room_version = yield self.store.get_room_version(event_dict["room_id"])
-            except NotFoundError:
-                raise AuthError(403, "Unknown room")
-
-        builder = self.event_builder_factory.new(room_version, event_dict)
+        builder = self.event_builder_factory.new(event_dict)
 
         self.validator.validate_new(builder)
 
diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py
index 2e16c69666..64a79da162 100644
--- a/synapse/replication/http/federation.py
+++ b/synapse/replication/http/federation.py
@@ -17,7 +17,7 @@ import logging
 
 from twisted.internet import defer
 
-from synapse.events import event_type_from_format_version
+from synapse.events import FrozenEvent
 from synapse.events.snapshot import EventContext
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
@@ -70,7 +70,6 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint):
 
             event_payloads.append({
                 "event": event.get_pdu_json(),
-                "event_format_version": event.format_version,
                 "internal_metadata": event.internal_metadata.get_dict(),
                 "rejected_reason": event.rejected_reason,
                 "context": serialized_context,
@@ -95,12 +94,9 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint):
             event_and_contexts = []
             for event_payload in event_payloads:
                 event_dict = event_payload["event"]
-                format_ver = content["event_format_version"]
                 internal_metadata = event_payload["internal_metadata"]
                 rejected_reason = event_payload["rejected_reason"]
-
-                EventType = event_type_from_format_version(format_ver)
-                event = EventType(event_dict, internal_metadata, rejected_reason)
+                event = FrozenEvent(event_dict, internal_metadata, rejected_reason)
 
                 context = yield EventContext.deserialize(
                     self.store, event_payload["context"],
diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py
index 3635015eda..5b52c91650 100644
--- a/synapse/replication/http/send_event.py
+++ b/synapse/replication/http/send_event.py
@@ -17,7 +17,7 @@ import logging
 
 from twisted.internet import defer
 
-from synapse.events import event_type_from_format_version
+from synapse.events import FrozenEvent
 from synapse.events.snapshot import EventContext
 from synapse.http.servlet import parse_json_object_from_request
 from synapse.replication.http._base import ReplicationEndpoint
@@ -74,7 +74,6 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint):
 
         payload = {
             "event": event.get_pdu_json(),
-            "event_format_version": event.format_version,
             "internal_metadata": event.internal_metadata.get_dict(),
             "rejected_reason": event.rejected_reason,
             "context": serialized_context,
@@ -91,12 +90,9 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint):
             content = parse_json_object_from_request(request)
 
             event_dict = content["event"]
-            format_ver = content["event_format_version"]
             internal_metadata = content["internal_metadata"]
             rejected_reason = content["rejected_reason"]
-
-            EventType = event_type_from_format_version(format_ver)
-            event = EventType(event_dict, internal_metadata, rejected_reason)
+            event = FrozenEvent(event_dict, internal_metadata, rejected_reason)
 
             requester = Requester.deserialize(self.store, content["requester"])
             context = yield EventContext.deserialize(self.store, content["context"])
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 0a0ca58fc4..599f892858 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -23,7 +23,7 @@ from twisted.internet import defer
 
 from synapse.api.constants import EventFormatVersions
 from synapse.api.errors import NotFoundError
-from synapse.events import FrozenEvent, event_type_from_format_version  # noqa: F401
+from synapse.events import FrozenEvent
 # these are only included to make the type annotations work
 from synapse.events.snapshot import EventContext  # noqa: F401
 from synapse.events.utils import prune_event
@@ -412,7 +412,11 @@ class EventsWorkerStore(SQLBaseStore):
                 # of a event format version, so it must be a V1 event.
                 format_version = EventFormatVersions.V1
 
-            original_ev = event_type_from_format_version(format_version)(
+            # TODO: When we implement new event formats we'll need to use a
+            # different event python type
+            assert format_version == EventFormatVersions.V1
+
+            original_ev = FrozenEvent(
                 event_dict=d,
                 internal_metadata_dict=internal_metadata,
                 rejected_reason=rejected_reason,
diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py
index 3957561b1e..02bf975fbf 100644
--- a/tests/storage/test_redaction.py
+++ b/tests/storage/test_redaction.py
@@ -18,7 +18,7 @@ from mock import Mock
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, Membership, RoomVersions
+from synapse.api.constants import EventTypes, Membership
 from synapse.types import RoomID, UserID
 
 from tests import unittest
@@ -52,7 +52,6 @@ class RedactionTestCase(unittest.TestCase):
         content = {"membership": membership}
         content.update(extra_content)
         builder = self.event_builder_factory.new(
-            RoomVersions.V1,
             {
                 "type": EventTypes.Member,
                 "sender": user.to_string(),
@@ -75,7 +74,6 @@ class RedactionTestCase(unittest.TestCase):
         self.depth += 1
 
         builder = self.event_builder_factory.new(
-            RoomVersions.V1,
             {
                 "type": EventTypes.Message,
                 "sender": user.to_string(),
@@ -96,7 +94,6 @@ class RedactionTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def inject_redaction(self, room, event_id, user, reason):
         builder = self.event_builder_factory.new(
-            RoomVersions.V1,
             {
                 "type": EventTypes.Redaction,
                 "sender": user.to_string(),
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index 7fa2f4fd70..978c66133d 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -18,7 +18,7 @@ from mock import Mock
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, Membership, RoomVersions
+from synapse.api.constants import EventTypes, Membership
 from synapse.types import RoomID, UserID
 
 from tests import unittest
@@ -50,7 +50,6 @@ class RoomMemberStoreTestCase(unittest.TestCase):
     @defer.inlineCallbacks
     def inject_room_member(self, room, user, membership, replaces_state=None):
         builder = self.event_builder_factory.new(
-            RoomVersions.V1,
             {
                 "type": EventTypes.Member,
                 "sender": user.to_string(),
diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py
index a1f99134dc..086a39d834 100644
--- a/tests/storage/test_state.py
+++ b/tests/storage/test_state.py
@@ -17,7 +17,7 @@ import logging
 
 from twisted.internet import defer
 
-from synapse.api.constants import EventTypes, Membership, RoomVersions
+from synapse.api.constants import EventTypes, Membership
 from synapse.storage.state import StateFilter
 from synapse.types import RoomID, UserID
 
@@ -52,7 +52,6 @@ class StateStoreTestCase(tests.unittest.TestCase):
     @defer.inlineCallbacks
     def inject_state_event(self, room, sender, typ, state_key, content):
         builder = self.event_builder_factory.new(
-            RoomVersions.V1,
             {
                 "type": typ,
                 "sender": sender.to_string(),
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
index 82d63ce00e..2eea3b098b 100644
--- a/tests/test_visibility.py
+++ b/tests/test_visibility.py
@@ -17,7 +17,6 @@ import logging
 from twisted.internet import defer
 from twisted.internet.defer import succeed
 
-from synapse.api.constants import RoomVersions
 from synapse.events import FrozenEvent
 from synapse.visibility import filter_events_for_server
 
@@ -125,7 +124,6 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
     def inject_visibility(self, user_id, visibility):
         content = {"history_visibility": visibility}
         builder = self.event_builder_factory.new(
-            RoomVersions.V1,
             {
                 "type": "m.room.history_visibility",
                 "sender": user_id,
@@ -146,7 +144,6 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
         content = {"membership": membership}
         content.update(extra_content)
         builder = self.event_builder_factory.new(
-            RoomVersions.V1,
             {
                 "type": "m.room.member",
                 "sender": user_id,
@@ -168,7 +165,6 @@ class FilterEventsForServerTestCase(tests.unittest.TestCase):
         if content is None:
             content = {"body": "testytest"}
         builder = self.event_builder_factory.new(
-            RoomVersions.V1,
             {
                 "type": "m.room.message",
                 "sender": user_id,
diff --git a/tests/utils.py b/tests/utils.py
index 2dfcb70a93..df73c539c3 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -26,7 +26,7 @@ from six.moves.urllib import parse as urlparse
 
 from twisted.internet import defer, reactor
 
-from synapse.api.constants import EventTypes, RoomVersions
+from synapse.api.constants import EventTypes
 from synapse.api.errors import CodeMessageException, cs_error
 from synapse.config.server import ServerConfig
 from synapse.federation.transport import server
@@ -624,7 +624,6 @@ def create_room(hs, room_id, creator_id):
     event_creation_handler = hs.get_event_creation_handler()
 
     builder = event_builder_factory.new(
-        RoomVersions.V1,
         {
             "type": EventTypes.Create,
             "state_key": "",

From 19530671362021b2ef53f4926c278e346406997b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 25 Jan 2019 10:46:49 +0000
Subject: [PATCH 31/35] Disable native upserts for sqlite, as they don't work

---
 synapse/storage/engines/sqlite.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py
index c64d73ff21..206919fcd5 100644
--- a/synapse/storage/engines/sqlite.py
+++ b/synapse/storage/engines/sqlite.py
@@ -34,10 +34,14 @@ class Sqlite3Engine(object):
     @property
     def can_native_upsert(self):
         """
-        Do we support native UPSERTs? This requires SQLite3 3.24+, plus some
-        more work we haven't done yet to tell what was inserted vs updated.
+        Do we support native UPSERTs?
         """
-        return sqlite_version_info >= (3, 24, 0)
+        # SQLite3 3.24+ supports them, but empirically the unit tests don't work
+        # when its enabled.
+        # FIXME: Figure out what is wrong so we can re-enable native upserts
+
+        # return sqlite_version_info >= (3, 24, 0)
+        return False
 
     def check_database(self, txn):
         pass

From 8dcfa6e75cb1c8e5afc62c1d964e1c752dd1917b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 25 Jan 2019 10:48:00 +0000
Subject: [PATCH 32/35] Newsfile

---
 changelog.d/4306.misc | 2 +-
 changelog.d/4471.misc | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/4471.misc

diff --git a/changelog.d/4306.misc b/changelog.d/4306.misc
index 58130b6190..7f48b02fbf 100644
--- a/changelog.d/4306.misc
+++ b/changelog.d/4306.misc
@@ -1 +1 @@
-Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+.
+Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+.
diff --git a/changelog.d/4471.misc b/changelog.d/4471.misc
new file mode 100644
index 0000000000..7f48b02fbf
--- /dev/null
+++ b/changelog.d/4471.misc
@@ -0,0 +1 @@
+Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+.

From 0b3fd1401fdebb944729cf46d6de9c3bff482933 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Fri, 25 Jan 2019 11:25:02 +0000
Subject: [PATCH 33/35] Don't require sqlite3 when using postgres (#4466)

---
 changelog.d/4466.misc             | 1 +
 synapse/storage/engines/sqlite.py | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/4466.misc

diff --git a/changelog.d/4466.misc b/changelog.d/4466.misc
new file mode 100644
index 0000000000..58130b6190
--- /dev/null
+++ b/changelog.d/4466.misc
@@ -0,0 +1 @@
+Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+.
diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py
index c64d73ff21..059ab81055 100644
--- a/synapse/storage/engines/sqlite.py
+++ b/synapse/storage/engines/sqlite.py
@@ -15,7 +15,6 @@
 
 import struct
 import threading
-from sqlite3 import sqlite_version_info
 
 from synapse.storage.prepare_database import prepare_database
 
@@ -37,7 +36,7 @@ class Sqlite3Engine(object):
         Do we support native UPSERTs? This requires SQLite3 3.24+, plus some
         more work we haven't done yet to tell what was inserted vs updated.
         """
-        return sqlite_version_info >= (3, 24, 0)
+        return self.module.sqlite_version_info >= (3, 24, 0)
 
     def check_database(self, txn):
         pass

From 8520bc3109c2de6175391497941f3fc0b74c08e5 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Fri, 25 Jan 2019 12:38:16 +0000
Subject: [PATCH 34/35] Fix Host header sent by MatrixFederationAgent (#4468)

Move the Host header logic down here so that (a) it is used if we reuse the
agent elsewhere, and (b) we can mess about with it with .well-known.
---
 changelog.d/4468.misc                            |  1 +
 .../http/federation/matrix_federation_agent.py   | 10 ++++++++++
 synapse/http/matrixfederationclient.py           |  1 -
 .../federation/test_matrix_federation_agent.py   | 16 ++++++++++++++++
 tests/http/test_fedclient.py                     |  2 +-
 5 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/4468.misc

diff --git a/changelog.d/4468.misc b/changelog.d/4468.misc
new file mode 100644
index 0000000000..9a51434755
--- /dev/null
+++ b/changelog.d/4468.misc
@@ -0,0 +1 @@
+Move SRV logic into the Agent layer
diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py
index 0ec28c6696..1788e9a34a 100644
--- a/synapse/http/federation/matrix_federation_agent.py
+++ b/synapse/http/federation/matrix_federation_agent.py
@@ -19,6 +19,7 @@ from zope.interface import implementer
 from twisted.internet import defer
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.web.client import URI, Agent, HTTPConnectionPool
+from twisted.web.http_headers import Headers
 from twisted.web.iweb import IAgent
 
 from synapse.http.endpoint import parse_server_name
@@ -109,6 +110,15 @@ class MatrixFederationAgent(object):
             else:
                 target = pick_server_from_list(server_list)
 
+        # make sure that the Host header is set correctly
+        if headers is None:
+            headers = Headers()
+        else:
+            headers = headers.copy()
+
+        if not headers.hasHeader(b'host'):
+            headers.addRawHeader(b'host', server_name_bytes)
+
         class EndpointFactory(object):
             @staticmethod
             def endpointForURI(_uri):
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 980e912348..bb2e64ed80 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -255,7 +255,6 @@ class MatrixFederationHttpClient(object):
 
         headers_dict = {
             b"User-Agent": [self.version_string_bytes],
-            b"Host": [destination_bytes],
         }
 
         with limiter:
diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py
index b32d7566a5..261afb5f41 100644
--- a/tests/http/federation/test_matrix_federation_agent.py
+++ b/tests/http/federation/test_matrix_federation_agent.py
@@ -131,6 +131,10 @@ class MatrixFederationAgentTests(TestCase):
         request = http_server.requests[0]
         self.assertEqual(request.method, b'GET')
         self.assertEqual(request.path, b'/foo/bar')
+        self.assertEqual(
+            request.requestHeaders.getRawHeaders(b'host'),
+            [b'testserv:8448']
+        )
         content = request.content.read()
         self.assertEqual(content, b'')
 
@@ -195,6 +199,10 @@ class MatrixFederationAgentTests(TestCase):
         request = http_server.requests[0]
         self.assertEqual(request.method, b'GET')
         self.assertEqual(request.path, b'/foo/bar')
+        self.assertEqual(
+            request.requestHeaders.getRawHeaders(b'host'),
+            [b'1.2.3.4'],
+        )
 
         # finish the request
         request.finish()
@@ -235,6 +243,10 @@ class MatrixFederationAgentTests(TestCase):
         request = http_server.requests[0]
         self.assertEqual(request.method, b'GET')
         self.assertEqual(request.path, b'/foo/bar')
+        self.assertEqual(
+            request.requestHeaders.getRawHeaders(b'host'),
+            [b'testserv'],
+        )
 
         # finish the request
         request.finish()
@@ -276,6 +288,10 @@ class MatrixFederationAgentTests(TestCase):
         request = http_server.requests[0]
         self.assertEqual(request.method, b'GET')
         self.assertEqual(request.path, b'/foo/bar')
+        self.assertEqual(
+            request.requestHeaders.getRawHeaders(b'host'),
+            [b'testserv'],
+        )
 
         # finish the request
         request.finish()
diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py
index d37f8f9981..018c77ebcd 100644
--- a/tests/http/test_fedclient.py
+++ b/tests/http/test_fedclient.py
@@ -49,7 +49,6 @@ class FederationClientTests(HomeserverTestCase):
         return hs
 
     def prepare(self, reactor, clock, homeserver):
-
         self.cl = MatrixFederationHttpClient(self.hs)
         self.reactor.lookups["testserv"] = "1.2.3.4"
 
@@ -95,6 +94,7 @@ class FederationClientTests(HomeserverTestCase):
 
         # that should have made it send the request to the transport
         self.assertRegex(transport.value(), b"^GET /foo/bar")
+        self.assertRegex(transport.value(), b"Host: testserv:8008")
 
         # Deferred is still without a result
         self.assertNoResult(test_d)

From 4a3f1388328f8181c33b74a6a777ab2073d400d8 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Fri, 25 Jan 2019 13:57:52 +0000
Subject: [PATCH 35/35] Fix quoting for allowed_local_3pids example config
 (#4476)

If you use double-quotes here, you have to escape your backslashes. It's much
easier with single-quotes.

(Note that the existing double-backslashes are already interpreted by python's
""" parsing.)
---
 changelog.d/4476.misc          | 1 +
 synapse/config/registration.py | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/4476.misc

diff --git a/changelog.d/4476.misc b/changelog.d/4476.misc
new file mode 100644
index 0000000000..a070e10c7c
--- /dev/null
+++ b/changelog.d/4476.misc
@@ -0,0 +1 @@
+Fix quoting for allowed_local_3pids example config
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index fe520d6855..d808a989f3 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -84,11 +84,11 @@ class RegistrationConfig(Config):
         #
         # allowed_local_3pids:
         #     - medium: email
-        #       pattern: ".*@matrix\\.org"
+        #       pattern: '.*@matrix\\.org'
         #     - medium: email
-        #       pattern: ".*@vector\\.im"
+        #       pattern: '.*@vector\\.im'
         #     - medium: msisdn
-        #       pattern: "\\+44"
+        #       pattern: '\\+44'
 
         # If set, allows registration by anyone who also has the shared
         # secret, even if registration is otherwise disabled.