From 8d33adfbbbd01a9a85e862ad2bd21ae45230e710 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 2 Mar 2015 18:23:55 +0000
Subject: [PATCH 01/14] SYN-67: Begin changing the way we handle schema
 versioning

---
 synapse/storage/__init__.py | 152 ++++++++++++++++++++++++++----------
 1 file changed, 110 insertions(+), 42 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index d6ec446bd2..a08c74fac1 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -45,8 +45,10 @@ from syutil.jsonutil import encode_canonical_json
 from synapse.crypto.event_signing import compute_event_reference_hash
 
 
+import imp
 import logging
 import os
+import sqlite3
 
 
 logger = logging.getLogger(__name__)
@@ -610,49 +612,115 @@ class UpgradeDatabaseException(PrepareDatabaseException):
 
 
 def prepare_database(db_conn):
-    """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we
-    don't have to worry about overwriting existing content.
+    """Prepares a database for usage. Will either create all necessary tables
+    or upgrade from an older schema version.
     """
-    c = db_conn.cursor()
-    c.execute("PRAGMA user_version")
-    row = c.fetchone()
-
-    if row and row[0]:
-        user_version = row[0]
-
-        if user_version > SCHEMA_VERSION:
-            raise ValueError(
-                "Cannot use this database as it is too " +
-                "new for the server to understand"
-            )
-        elif user_version < SCHEMA_VERSION:
-            logger.info(
-                "Upgrading database from version %d",
-                user_version
-            )
-
-            # Run every version since after the current version.
-            for v in range(user_version + 1, SCHEMA_VERSION + 1):
-                if v in (10, 14,):
-                    raise UpgradeDatabaseException(
-                        "No delta for version 10"
-                    )
-                sql_script = read_schema("delta/v%d" % (v,))
-                c.executescript(sql_script)
-
-            db_conn.commit()
-        else:
-            logger.info("Database is at version %r", user_version)
+    cur = db_conn.cursor()
+    version_info = get_schema_state(cur)
 
+    if version_info:
+        user_version, delta_files = version_info
+        _upgrade_existing_database(cur, user_version, delta_files)
     else:
-        sql_script = "BEGIN TRANSACTION;\n"
-        for sql_loc in SCHEMAS:
-            logger.debug("Applying schema %r", sql_loc)
-            sql_script += read_schema(sql_loc)
-            sql_script += "\n"
-        sql_script += "COMMIT TRANSACTION;"
-        c.executescript(sql_script)
-        db_conn.commit()
-        c.execute("PRAGMA user_version = %d" % SCHEMA_VERSION)
+        _setup_new_database(cur)
 
-    c.close()
+    cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,))
+    db_conn.commit()
+
+    cur.close()
+
+
+def _setup_new_database(cur):
+    sql_script = "BEGIN TRANSACTION;\n"
+    for sql_loc in SCHEMAS:
+        logger.debug("Applying schema %r", sql_loc)
+        sql_script += read_schema(sql_loc)
+        sql_script += "\n"
+    sql_script += "COMMIT TRANSACTION;"
+    cur.executescript(sql_script)
+
+
+def _upgrade_existing_database(cur, user_version, delta_files):
+    """Upgrades an existing database.
+
+    Delta files can either be SQL stored in *.sql files, or python modules
+    in *.py.
+
+    There can be multiple delta files per version. Synapse will keep track of
+    which delta files have been applied, and will apply any that haven't been
+    even if there has been no version bump. This is useful for development
+    where orthogonal schema changes may happen on separate branches.
+    """
+
+    if user_version > SCHEMA_VERSION:
+        raise ValueError(
+            "Cannot use this database as it is too " +
+            "new for the server to understand"
+        )
+
+    for v in range(user_version, SCHEMA_VERSION + 1):
+        delta_dir = os.path.join(dir_path, "schema", "delta", v)
+        directory_entries = os.listdir(delta_dir)
+
+        for file_name in directory_entries:
+            relative_path = os.path.join(v, file_name)
+            if relative_path in delta_files:
+                continue
+
+            absolute_path = os.path.join(
+                dir_path, "schema", "delta", relative_path,
+            )
+            root_name, ext = os.path.splitext(file_name)
+            if ext == ".py":
+                module_name = "synapse.storage.schema.v%d_%s" % (
+                    v, root_name
+                )
+                with open(absolute_path) as schema_file:
+                    module = imp.load_source(
+                        module_name, absolute_path, schema_file
+                    )
+                module.run_upgrade(cur)
+            elif ext == ".sql":
+                with open(absolute_path) as schema_file:
+                    delta_schema = schema_file.read()
+                cur.executescript(delta_schema)
+            else:
+                # Not a valid delta file.
+                logger.warn(
+                    "Found directory entry that did not end in .py or"
+                    " .sql: %s",
+                    relative_path,
+                )
+                continue
+
+            # Mark as done.
+            cur.execute(
+                "INSERT INTO schema_version (version, file)"
+                " VALUES (?,?)",
+                (v, relative_path)
+            )
+
+
+def get_schema_state(txn):
+    sql = (
+        "SELECT MAX(version), file FROM schema_version"
+        " WHERE version = (SELECT MAX(version) FROM schema_version)"
+    )
+
+    try:
+        txn.execute(sql)
+        res = txn.fetchall()
+
+        if res:
+            current_verison = max(r[0] for r in res)
+            applied_delta = [r[1] for r in res]
+
+            return current_verison, applied_delta
+    except sqlite3.OperationalError:
+        txn.execute("PRAGMA user_version")
+        row = txn.fetchone()
+        if row and row[0]:
+            # FIXME: We need to create schema_version table!
+            return row[0], []
+
+    return None

From 84a4367657f5c97849f629bebb1e411532042290 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 4 Mar 2015 01:27:47 +0100
Subject: [PATCH 02/14] WIP vertobridge AS

---
 contrib/vertobot/bridge.pl | 489 +++++++++++++++++++++++++++++++++++++
 1 file changed, 489 insertions(+)
 create mode 100755 contrib/vertobot/bridge.pl

diff --git a/contrib/vertobot/bridge.pl b/contrib/vertobot/bridge.pl
new file mode 100755
index 0000000000..e1a07f6659
--- /dev/null
+++ b/contrib/vertobot/bridge.pl
@@ -0,0 +1,489 @@
+#!/usr/bin/env perl 
+
+use strict;
+use warnings;
+use 5.010; # //
+use IO::Socket::SSL qw(SSL_VERIFY_NONE);
+use IO::Async::Loop;
+use Net::Async::WebSocket::Client;
+use Net::Async::HTTP;
+use Net::Async::HTTP::Server;
+use JSON;
+use YAML;
+use Data::UUID;
+use Getopt::Long;
+use Data::Dumper;
+use URI::Encode qw(uri_encode uri_decode);
+    
+binmode STDOUT, ":encoding(UTF-8)";
+binmode STDERR, ":encoding(UTF-8)";
+
+my $msisdn_to_matrix = {
+    '447417892400' => '@matthew:matrix.org',
+};
+
+my $matrix_to_msisdn = {};
+foreach (keys %$msisdn_to_matrix) {
+    $matrix_to_msisdn->{$msisdn_to_matrix->{$_}} = $_;
+}
+
+
+my $loop = IO::Async::Loop->new;
+# Net::Async::HTTP + SSL + IO::Poll doesn't play well. See
+#   https://rt.cpan.org/Ticket/Display.html?id=93107
+# ref $loop eq "IO::Async::Loop::Poll" and
+#     warn "Using SSL with IO::Poll causes known memory-leaks!!\n";
+
+GetOptions(
+   'C|config=s' => \my $CONFIG,
+   'eval-from=s' => \my $EVAL_FROM,
+) or exit 1;
+
+if( defined $EVAL_FROM ) {
+    # An emergency 'eval() this file' hack
+    $SIG{HUP} = sub {
+        my $code = do {
+            open my $fh, "<", $EVAL_FROM or warn( "Cannot read - $!" ), return;
+            local $/; <$fh>
+        };
+
+        eval $code or warn "Cannot eval() - $@";
+    };
+}
+
+defined $CONFIG or die "Must supply --config\n";
+
+my %CONFIG = %{ YAML::LoadFile( $CONFIG ) };
+
+my %MATRIX_CONFIG = %{ $CONFIG{matrix} };
+# No harm in always applying this
+$MATRIX_CONFIG{SSL_verify_mode} = SSL_VERIFY_NONE;
+
+my $bridgestate = {};
+my $roomid_by_callid = {};
+    
+my $sessid = lc new Data::UUID->create_str();    
+my $as_token = $CONFIG{"matrix-bot"}->{as_token};
+my $hs_domain = $CONFIG{"matrix-bot"}->{domain};
+
+my $http = Net::Async::HTTP->new();
+$loop->add( $http );
+
+sub create_virtual_user
+{
+    my ($localpart) = @_;
+    my ( $response ) = $http->do_request(
+        method => "POST",
+        uri => URI->new(
+            $CONFIG{"matrix"}->{server}.
+                "/_matrix/client/api/v1/register?".
+                "access_token=$as_token&user_id=$localpart"
+        ),
+        content_type => "application/json",
+        content => <<EOT
+{
+    "type": "m.login.application_service",
+    "user": "$localpart"
+}
+EOT
+    )->get;    
+    warn $response->as_string if ($response->code != 200);
+}
+    
+my $http_server =  Net::Async::HTTP::Server->new(
+    on_request => sub {
+        my $self = shift;
+        my ( $req ) = @_;
+
+        my $response;
+        my $path = uri_decode($req->path);
+        warn("request: $path");
+        if ($path =~ m#/users/\@(\+.*)#) {
+            # when queried about virtual users, auto-create them in the HS
+            my $localpart = $1;
+            create_virtual_user($localpart);
+            $response = HTTP::Response->new( 200 );
+            $response->add_content('{}');
+            $response->content_type( "application/json" );
+        }
+        elsif ($path =~ m#/transactions/(.*)#) {
+            my $event = JSON->new->decode($req->body);
+            print Dumper($event);
+
+            my $room_id = $event->{room_id};
+            my %dp = %{$CONFIG{'verto-dialog-params'}};
+            $dp{callID} = $bridgestate->{$room_id}->{callid};
+
+            if ($event->{type} eq 'm.room.membership') {
+                my $membership = $event->{content}->{membership};
+                my $state_key = $event->{state_key};
+                my $room_id = $event->{state_id};
+                
+                if ($membership eq 'invite') {
+                    # autojoin invites
+                    my ( $response ) = $http->do_request(
+                        method => "POST",
+                        uri => URI->new(
+                            $CONFIG{"matrix"}->{server}.
+                                "/_matrix/client/api/v1/rooms/$room_id/join?".
+                                "access_token=$as_token&user_id=$state_key"
+                        ),
+                        content_type => "application/json",
+                        content => "{}",
+                    )->get;
+                    warn $response->as_string if ($response->code != 200);
+                }
+            }
+            elsif ($event->{type} eq 'm.call.invite') {
+                my $room_id = $event->{room_id};
+                $bridgestate->{$room_id}->{matrix_callid} = $event->{content}->{call_id};
+                $bridgestate->{$room_id}->{callid} = lc new Data::UUID->create_str();
+                $bridgestate->{$room_id}->{sessid} = $sessid;                
+                # $bridgestate->{$room_id}->{offer} = $event->{content}->{offer}->{sdp};
+                my $offer = $event->{content}->{offer}->{sdp};
+                # $bridgestate->{$room_id}->{gathered_candidates} = 0;
+                $roomid_by_callid->{ $bridgestate->{$room_id}->{callid} } = $room_id;
+                # no trickle ICE in verto apparently
+
+                my $f = send_verto_json_request("verto.invite", {
+                    "sdp" => $offer,
+                    "dialogParams" => \%dp,
+                    "sessid" => $bridgestate->{$room_id}->{sessid},
+                });
+                $self->adopt_future($f);
+            }
+            # elsif ($event->{type} eq 'm.call.candidates') {
+            #     # XXX: this could fire for both matrix->verto and verto->matrix calls
+            #     # and races as it collects candidates. much better to just turn off
+            #     # candidate gathering in the webclient entirely for now
+            #     
+            #     my $room_id = $event->{room_id};
+            #     # XXX: compare call IDs
+            #     if (!$bridgestate->{$room_id}->{gathered_candidates}) {
+            #         $bridgestate->{$room_id}->{gathered_candidates} = 1;
+            #         my $offer = $bridgestate->{$room_id}->{offer};
+            #         my $candidate_block = "";
+            #         foreach (@{$event->{content}->{candidates}}) {
+            #             $candidate_block .= "a=" . $_->{candidate} . "\r\n";
+            #         }
+            #         # XXX: collate using the right m= line - for now assume audio call
+            #         $offer =~ s/(a=rtcp.*[\r\n]+)/$1$candidate_block/;
+            #     
+            #         my $f = send_verto_json_request("verto.invite", {
+            #             "sdp" => $offer,
+            #             "dialogParams" => \%dp,
+            #             "sessid" => $bridgestate->{$room_id}->{sessid},
+            #         });
+            #         $self->adopt_future($f);
+            #     }
+            #     else {
+            #         # ignore them, as no trickle ICE, although we might as well
+            #         # batch them up
+            #         # foreach (@{$event->{content}->{candidates}}) {
+            #         #     push @{$bridgestate->{$room_id}->{candidates}}, $_;
+            #         # }
+            #     }
+            # }
+            elsif ($event->{type} eq 'm.call.answer') {
+                # grab the answer and relay it to verto as a verto.answer
+                my $room_id = $event->{room_id};
+                
+                my $answer = $event->{content}->{answer}->{sdp};
+                my $f = send_verto_json_request("verto.answer", {
+                    "sdp" => $answer,
+                    "dialogParams" => \%dp,
+                    "sessid" => $bridgestate->{$room_id}->{sessid},
+                });
+                $self->adopt_future($f);
+            }
+            elsif ($event->{type} eq 'm.call.hangup') {
+                my $room_id = $event->{room_id};
+                if ($bridgestate->{$room_id}->{matrix_callid} eq $event->{content}->{call_id}) {
+                    my $f = send_verto_json_request("verto.bye", {
+                        "dialogParams" => \%dp,
+                        "sessid" => $bridgestate->{$room_id}->{sessid},
+                    });
+                    $self->adopt_future($f);
+                }
+                else {
+                    warn "Ignoring unrecognised callid: ".$event->{content}->{call_id};
+                }
+            }
+            else {
+                warn "Unhandled event: $event->{type}";
+            }
+            
+            $response = HTTP::Response->new( 200 );
+            $response->add_content('{}');
+            $response->content_type( "application/json" );            
+        }
+        else {
+            warn "Unhandled path: $path";
+            $response = HTTP::Response->new( 404 );
+        }
+
+        $req->respond( $response );
+    },
+);
+$loop->add( $http_server );
+
+$http_server->listen(
+    addr => { family => "inet", socktype => "stream", port => 8009 },
+    on_listen_error => sub { die "Cannot listen - $_[-1]\n" },
+);
+
+my $bot_verto = Net::Async::WebSocket::Client->new(
+    on_frame => sub {
+          my ( $self, $frame ) = @_;
+          warn "[Verto] receiving $frame";
+          on_verto_json($frame);
+    },
+);
+$loop->add( $bot_verto );
+
+my $verto_connecting = $loop->new_future;
+$bot_verto->connect(
+    %{ $CONFIG{"verto-bot"} },
+    on_connected => sub {
+        warn("[Verto] connected to websocket");
+        if (not $verto_connecting->is_done) {
+            $verto_connecting->done($bot_verto);
+
+            send_verto_json_request("login", {
+                'login' => $CONFIG{'verto-dialog-params'}{'login'},
+                'passwd' => $CONFIG{'verto-config'}{'passwd'},
+                'sessid' => $sessid,
+            });
+        }
+    },
+    on_connect_error => sub { die "Cannot connect to verto - $_[-1]" },
+    on_resolve_error => sub { die "Cannot resolve to verto - $_[-1]" },        
+);
+
+# die Dumper($verto_connecting);
+
+my $as_url = $CONFIG{"matrix-bot"}->{as_url};
+
+Future->needs_all(
+    $http->do_request(
+        method => "POST",
+        uri => URI->new( $CONFIG{"matrix"}->{server}."/_matrix/appservice/v1/register" ),
+        content_type => "application/json",
+        content => <<EOT
+{
+    "as_token": "$as_token",
+    "url": "$as_url",
+    "namespaces": { "users": ["\@\\\\+.*"] }
+}
+EOT
+    ),
+    $verto_connecting,
+)->get;
+
+$loop->attach_signal(
+    PIPE => sub { warn "pipe\n" }
+);
+$loop->attach_signal(
+    INT => sub { $loop->stop },
+);
+$loop->attach_signal(
+    TERM => sub { $loop->stop },
+);
+
+eval {
+   $loop->run;
+} or my $e = $@;
+
+die $e if $e;
+
+exit 0;
+
+{    
+    my $json_id;
+    my $requests;
+
+    sub send_verto_json_request
+    {
+        $json_id ||= 1;
+        
+        my ($method, $params) = @_;
+        my $json = {
+            jsonrpc => "2.0",
+            method  => $method,
+            params  => $params,
+            id      => $json_id,
+        };
+        my $text = JSON->new->encode( $json );
+        warn "[Verto] sending $text";
+        $bot_verto->send_frame ( $text );
+        my $request = $loop->new_future;
+        $requests->{$json_id} = $request;
+        $json_id++;
+        return $request;
+    }
+    
+    sub send_verto_json_response
+    {
+        my ($result, $id) = @_;
+        my $json = {
+            jsonrpc => "2.0",
+            result  => $result,
+            id      => $id,
+        };
+        my $text = JSON->new->encode( $json );
+        warn "[Verto] sending $text";
+        $bot_verto->send_frame ( $text );
+    }
+    
+    sub on_verto_json
+    {
+        my $json = JSON->new->decode( $_[0] );
+        if ($json->{method}) {
+            if (($json->{method} eq 'verto.answer' && $json->{params}->{sdp}) ||
+                $json->{method} eq 'verto.media') {
+
+                my $caller = $json->{dialogParams}->{caller_id_number};
+                my $callee = $json->{dialogParams}->{destination_number};
+                my $caller_user = '@+' . $caller . ':' . $hs_domain;
+                my $callee_user = $msisdn_to_matrix->{$callee} || warn "unrecogised callee: $callee";                                
+                my $room_id = $roomid_by_callid->{$json->{params}->{callID}};
+
+                if ($json->{params}->{sdp}) {
+                    $http->do_request(
+                        method => "POST",
+                        uri => URI->new(
+                            $CONFIG{"matrix"}->{server}.
+                                "/_matrix/client/api/v1/send/m.call.answer?".
+                                "access_token=$as_token&user_id=$caller_user"
+                        ),
+                        content_type => "application/json",
+                        content => JSON->new->encode({
+                            call_id => $bridgestate->{$room_id}->{matrix_callid},
+                            version => 0,
+                            answer  => {
+                                sdp => $json->{params}->{sdp},
+                                type => "answer",
+                            },
+                        }),
+                    )->then( sub {
+                        send_verto_json_response( {
+                            method => $json->{method},
+                        }, $json->{id});
+                    })->get;
+                }
+            }
+            elsif ($json->{method} eq 'verto.invite') {
+                my $caller = $json->{dialogParams}->{caller_id_number};
+                my $callee = $json->{dialogParams}->{destination_number};
+                my $caller_user = '@+' . $caller . ':' . $hs_domain;
+                my $callee_user = $msisdn_to_matrix->{$callee} || warn "unrecogised callee: $callee";
+                    
+                my $alias = ($caller lt $callee) ? ($caller.'-'.$callee) : ($callee.'-'.$caller);
+                my $room_id;
+
+                # create a virtual user for the caller if needed.
+                create_virtual_user($caller);
+                
+                # create a room of form #peer-peer and invite the callee
+                $http->do_request(
+                    method => "POST",
+                    uri => URI->new(
+                        $CONFIG{"matrix"}->{server}.
+                            "/_matrix/client/api/v1/createRoom?".
+                            "access_token=$as_token&user_id=$caller_user"
+                    ),
+                    content_type => "application/json",
+                    content => JSON->new->encode({
+                        room_alias_name => $alias,
+                        invite => [ $callee_user ],
+                    }),
+                )->then( sub {
+                    my ( $response ) = @_;
+                    my $resp = JSON->new->decode($response->content);
+                    $room_id = $resp->{room_id};
+                    $roomid_by_callid->{$json->{params}->{callID}} = $room_id;
+                })->get;
+
+                # join it
+                my ($response) = $http->do_request(
+                    method => "POST",
+                    uri => URI->new(
+                        $CONFIG{"matrix"}->{server}.
+                            "/_matrix/client/api/v1/join/$room_id?".
+                            "access_token=$as_token&user_id=$caller_user"
+                    ),
+                    content_type => "application/json",
+                    content => '{}',
+                )->get;
+
+                $bridgestate->{$room_id}->{matrix_callid} = lc new Data::UUID->create_str();
+                $bridgestate->{$room_id}->{callid} = $json->{dialogParams}->{callID};
+                $bridgestate->{$room_id}->{sessid} = $sessid;
+
+                # put the m.call.invite in there
+                $http->do_request(
+                    method => "POST",
+                    uri => URI->new(
+                        $CONFIG{"matrix"}->{server}.
+                            "/_matrix/client/api/v1/send/m.call.invite?".
+                            "access_token=$as_token&user_id=$caller_user"
+                    ),
+                    content_type => "application/json",
+                    content => JSON->new->encode({
+                        call_id => $bridgestate->{$room_id}->{matrix_callid},
+                        version => 0,
+                        answer  => {
+                            sdp => $json->{params}->{sdp},
+                            type => "offer",
+                        },
+                    }),
+                )->then( sub {
+                    # acknowledge the verto
+                    send_verto_json_response( {
+                        method => $json->{method},
+                    }, $json->{id});
+                })->get;
+            }
+            elsif ($json->{method} eq 'verto.bye') {
+                my $caller = $json->{dialogParams}->{caller_id_number};
+                my $callee = $json->{dialogParams}->{destination_number};
+                my $caller_user = '@+' . $caller . ':' . $hs_domain;
+                my $callee_user = $msisdn_to_matrix->{$callee} || warn "unrecogised callee: $callee";                                
+                my $room_id = $roomid_by_callid->{$json->{params}->{callID}};
+                
+                # put the m.call.hangup into the room
+                $http->do_request(
+                    method => "POST",
+                    uri => URI->new(
+                        $CONFIG{"matrix"}->{server}.
+                            "/_matrix/client/api/v1/send/m.call.hangup?".
+                            "access_token=$as_token&user_id=$caller_user"
+                    ),
+                    content_type => "application/json",
+                    content => JSON->new->encode({
+                        call_id => $bridgestate->{$room_id}->{matrix_callid},
+                        version => 0,
+                    }),
+                )->then( sub {
+                    # acknowledge the verto
+                    send_verto_json_response( {
+                        method => $json->{method},
+                    }, $json->{id});
+                })->get;
+            }
+            else {
+                warn ("[Verto] unhandled method: " . $json->{method});
+                send_verto_json_response( {
+                    method => $json->{method},
+                }, $json->{id});
+            }
+        }
+        elsif ($json->{result}) {
+            $requests->{$json->{id}}->done($json->{result});
+        }
+        elsif ($json->{error}) {
+            $requests->{$json->{id}}->fail($json->{error}->{message}, $json->{error});
+        }
+    }
+}
+

From 82b34e813de4dadb8ec5bce068f7113e32e60ead Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 12:04:19 +0000
Subject: [PATCH 03/14] SYN-67: Finish up implementing new database schema
 management

---
 scripts/upgrade_appservice_db.py              |  54 -----
 synapse/app/homeserver.py                     |   5 +-
 synapse/storage/__init__.py                   | 199 ++++++++++++------
 .../schema/{ => current/11}/event_edges.sql   |   0
 .../{ => current/11}/event_signatures.sql     |   0
 .../storage/schema/{ => current/11}/im.sql    |   0
 .../storage/schema/{ => current/11}/keys.sql  |   0
 .../{ => current/11}/media_repository.sql     |   0
 .../schema/{ => current/11}/presence.sql      |   0
 .../schema/{ => current/11}/profiles.sql      |   0
 .../schema/{ => current/11}/redactions.sql    |   0
 .../schema/{ => current/11}/room_aliases.sql  |   0
 .../storage/schema/{ => current/11}/state.sql |   0
 .../schema/{ => current/11}/transactions.sql  |   0
 .../storage/schema/{ => current/11}/users.sql |   0
 synapse/storage/schema/delta/{ => 11}/v11.sql |   0
 synapse/storage/schema/delta/{ => 12}/v12.sql |   0
 synapse/storage/schema/delta/{ => 13}/v13.sql |   0
 .../schema/delta/14/upgrade_appservice_db.py  |  20 ++
 synapse/storage/schema/delta/v2.sql           | 168 ---------------
 synapse/storage/schema/delta/v3.sql           |  27 ---
 synapse/storage/schema/delta/v4.sql           |  26 ---
 synapse/storage/schema/delta/v5.sql           |  30 ---
 synapse/storage/schema/delta/v6.sql           |  31 ---
 synapse/storage/schema/delta/v8.sql           |  34 ---
 synapse/storage/schema/delta/v9.sql           |  79 -------
 synapse/storage/schema/filtering.sql          |  24 ---
 synapse/storage/schema/pusher.sql             |  46 ----
 synapse/storage/schema/rejections.sql         |  21 --
 ...cation_services.sql => schema_version.sql} |  25 +--
 30 files changed, 166 insertions(+), 623 deletions(-)
 delete mode 100644 scripts/upgrade_appservice_db.py
 rename synapse/storage/schema/{ => current/11}/event_edges.sql (100%)
 rename synapse/storage/schema/{ => current/11}/event_signatures.sql (100%)
 rename synapse/storage/schema/{ => current/11}/im.sql (100%)
 rename synapse/storage/schema/{ => current/11}/keys.sql (100%)
 rename synapse/storage/schema/{ => current/11}/media_repository.sql (100%)
 rename synapse/storage/schema/{ => current/11}/presence.sql (100%)
 rename synapse/storage/schema/{ => current/11}/profiles.sql (100%)
 rename synapse/storage/schema/{ => current/11}/redactions.sql (100%)
 rename synapse/storage/schema/{ => current/11}/room_aliases.sql (100%)
 rename synapse/storage/schema/{ => current/11}/state.sql (100%)
 rename synapse/storage/schema/{ => current/11}/transactions.sql (100%)
 rename synapse/storage/schema/{ => current/11}/users.sql (100%)
 rename synapse/storage/schema/delta/{ => 11}/v11.sql (100%)
 rename synapse/storage/schema/delta/{ => 12}/v12.sql (100%)
 rename synapse/storage/schema/delta/{ => 13}/v13.sql (100%)
 create mode 100644 synapse/storage/schema/delta/14/upgrade_appservice_db.py
 delete mode 100644 synapse/storage/schema/delta/v2.sql
 delete mode 100644 synapse/storage/schema/delta/v3.sql
 delete mode 100644 synapse/storage/schema/delta/v4.sql
 delete mode 100644 synapse/storage/schema/delta/v5.sql
 delete mode 100644 synapse/storage/schema/delta/v6.sql
 delete mode 100644 synapse/storage/schema/delta/v8.sql
 delete mode 100644 synapse/storage/schema/delta/v9.sql
 delete mode 100644 synapse/storage/schema/filtering.sql
 delete mode 100644 synapse/storage/schema/pusher.sql
 delete mode 100644 synapse/storage/schema/rejections.sql
 rename synapse/storage/schema/{application_services.sql => schema_version.sql} (51%)

diff --git a/scripts/upgrade_appservice_db.py b/scripts/upgrade_appservice_db.py
deleted file mode 100644
index ae1b91c64f..0000000000
--- a/scripts/upgrade_appservice_db.py
+++ /dev/null
@@ -1,54 +0,0 @@
-from synapse.storage import read_schema
-import argparse
-import json
-import sqlite3
-
-
-def do_other_deltas(cursor):
-    cursor.execute("PRAGMA user_version")
-    row = cursor.fetchone()
-
-    if row and row[0]:
-        user_version = row[0]
-        # Run every version since after the current version.
-        for v in range(user_version + 1, 10):
-            print "Running delta: %d" % (v,)
-            sql_script = read_schema("delta/v%d" % (v,))
-            cursor.executescript(sql_script)
-
-
-def update_app_service_table(cur):
-    cur.execute("SELECT id, regex FROM application_services_regex")
-    for row in cur.fetchall():
-        try:
-            print "checking %s..." % row[0]
-            json.loads(row[1])
-        except ValueError:
-            # row isn't in json, make it so.
-            string_regex = row[1]
-            new_regex = json.dumps({
-                "regex": string_regex,
-                "exclusive": True
-            })
-            cur.execute(
-                "UPDATE application_services_regex SET regex=? WHERE id=?",
-                (new_regex, row[0])
-            )
-
-
-def main(dbname):
-    con = sqlite3.connect(dbname)
-    cur = con.cursor()
-    do_other_deltas(cur)
-    update_app_service_table(cur)
-    cur.execute("PRAGMA user_version = 14")
-    cur.close()
-    con.commit()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("database")
-    args = parser.parse_args()
-
-    main(args.database)
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 5695d5aff8..b3ba7dfddc 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -17,7 +17,9 @@
 import sys
 sys.dont_write_bytecode = True
 
-from synapse.storage import prepare_database, UpgradeDatabaseException
+from synapse.storage import (
+    prepare_database, prepare_sqlite3_database, UpgradeDatabaseException,
+)
 
 from synapse.server import HomeServer
 
@@ -335,6 +337,7 @@ def setup():
 
     try:
         with sqlite3.connect(db_name) as db_conn:
+            prepare_sqlite3_database(db_conn)
             prepare_database(db_conn)
     except UpgradeDatabaseException:
         sys.stderr.write(
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index a08c74fac1..07ccc4e2ee 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -45,35 +45,16 @@ from syutil.jsonutil import encode_canonical_json
 from synapse.crypto.event_signing import compute_event_reference_hash
 
 
+import fnmatch
 import imp
 import logging
 import os
-import sqlite3
+import re
 
 
 logger = logging.getLogger(__name__)
 
 
-SCHEMAS = [
-    "transactions",
-    "users",
-    "profiles",
-    "presence",
-    "im",
-    "room_aliases",
-    "keys",
-    "redactions",
-    "state",
-    "event_edges",
-    "event_signatures",
-    "pusher",
-    "media_repository",
-    "application_services",
-    "filtering",
-    "rejections",
-]
-
-
 # Remember to update this number every time an incompatible change is made to
 # database schema files, so the users will be informed on server restarts.
 SCHEMA_VERSION = 14
@@ -578,28 +559,15 @@ class DataStore(RoomMemberStore, RoomStore,
         )
 
 
-def schema_path(schema):
-    """ Get a filesystem path for the named database schema
-
-    Args:
-        schema: Name of the database schema.
-    Returns:
-        A filesystem path pointing at a ".sql" file.
-
-    """
-    schemaPath = os.path.join(dir_path, "schema", schema + ".sql")
-    return schemaPath
-
-
-def read_schema(schema):
+def read_schema(path):
     """ Read the named database schema.
 
     Args:
-        schema: Name of the datbase schema.
+        path: Path of the database schema.
     Returns:
         A string containing the database schema.
     """
-    with open(schema_path(schema)) as schema_file:
+    with open(path) as schema_file:
         return schema_file.read()
 
 
@@ -616,11 +584,11 @@ def prepare_database(db_conn):
     or upgrade from an older schema version.
     """
     cur = db_conn.cursor()
-    version_info = get_schema_state(cur)
+    version_info = get_or_create_schema_state(cur)
 
     if version_info:
-        user_version, delta_files = version_info
-        _upgrade_existing_database(cur, user_version, delta_files)
+        user_version, delta_files, upgraded = version_info
+        _upgrade_existing_database(cur, user_version, delta_files, upgraded)
     else:
         _setup_new_database(cur)
 
@@ -631,16 +599,52 @@ def prepare_database(db_conn):
 
 
 def _setup_new_database(cur):
+    current_dir = os.path.join(dir_path, "schema", "current")
+    directory_entries = os.listdir(current_dir)
+
+    valid_dirs = []
+    pattern = re.compile(r"^\d+(\.sql)?$")
+    for filename in directory_entries:
+        match = pattern.match(filename)
+        abs_path = os.path.join(current_dir, filename)
+        if match and os.path.isdir(abs_path):
+            ver = int(match.group(0))
+            if ver < SCHEMA_VERSION:
+                valid_dirs.append((ver, abs_path))
+
+    if not valid_dirs:
+        raise RuntimeError("Could not find a suitable current.sql")
+
+    max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0])
+
+    logger.debug("Initialising schema v%d", max_current_ver)
+
+    directory_entries = os.listdir(sql_dir)
+
     sql_script = "BEGIN TRANSACTION;\n"
-    for sql_loc in SCHEMAS:
+    for filename in fnmatch.filter(directory_entries, "*.sql"):
+        sql_loc = os.path.join(sql_dir, filename)
         logger.debug("Applying schema %r", sql_loc)
         sql_script += read_schema(sql_loc)
         sql_script += "\n"
     sql_script += "COMMIT TRANSACTION;"
     cur.executescript(sql_script)
 
+    cur.execute(
+        "INSERT INTO schema_version (version, upgraded)"
+        " VALUES (?,?)",
+        (max_current_ver, False)
+    )
 
-def _upgrade_existing_database(cur, user_version, delta_files):
+    _upgrade_existing_database(
+        cur,
+        current_version=max_current_ver,
+        delta_files=[],
+        upgraded=False
+    )
+
+
+def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
     """Upgrades an existing database.
 
     Delta files can either be SQL stored in *.sql files, or python modules
@@ -650,20 +654,41 @@ def _upgrade_existing_database(cur, user_version, delta_files):
     which delta files have been applied, and will apply any that haven't been
     even if there has been no version bump. This is useful for development
     where orthogonal schema changes may happen on separate branches.
+
+    Args:
+        cur (Cursor)
+        current_version (int): The current version of the schema
+        delta_files (list): A list of deltas that have already been applied
+        upgraded (bool): Whether the current version was generated by having
+            applied deltas or from full schema file. If `True` the function
+            will never apply delta files for the given `current_version`, since
+            the current_version wasn't generated by applying those delta files.
     """
 
-    if user_version > SCHEMA_VERSION:
+    if current_version > SCHEMA_VERSION:
         raise ValueError(
             "Cannot use this database as it is too " +
             "new for the server to understand"
         )
 
-    for v in range(user_version, SCHEMA_VERSION + 1):
-        delta_dir = os.path.join(dir_path, "schema", "delta", v)
-        directory_entries = os.listdir(delta_dir)
+    start_ver = current_version
+    if not upgraded:
+        start_ver += 1
 
+    for v in range(start_ver, SCHEMA_VERSION + 1):
+        logger.debug("Upgrading schema to v%d", v)
+
+        delta_dir = os.path.join(dir_path, "schema", "delta", str(v))
+
+        try:
+            directory_entries = os.listdir(delta_dir)
+        except OSError:
+            logger.exception("Could not open delta dir for version %d", v)
+            raise
+
+        directory_entries.sort()
         for file_name in directory_entries:
-            relative_path = os.path.join(v, file_name)
+            relative_path = os.path.join(str(v), file_name)
             if relative_path in delta_files:
                 continue
 
@@ -672,17 +697,19 @@ def _upgrade_existing_database(cur, user_version, delta_files):
             )
             root_name, ext = os.path.splitext(file_name)
             if ext == ".py":
-                module_name = "synapse.storage.schema.v%d_%s" % (
+                module_name = "synapse.storage.v%d_%s" % (
                     v, root_name
                 )
                 with open(absolute_path) as schema_file:
                     module = imp.load_source(
                         module_name, absolute_path, schema_file
                     )
+                logger.debug("Running script %s", relative_path)
                 module.run_upgrade(cur)
             elif ext == ".sql":
                 with open(absolute_path) as schema_file:
                     delta_schema = schema_file.read()
+                logger.debug("Applying schema %s", relative_path)
                 cur.executescript(delta_schema)
             else:
                 # Not a valid delta file.
@@ -695,32 +722,70 @@ def _upgrade_existing_database(cur, user_version, delta_files):
 
             # Mark as done.
             cur.execute(
-                "INSERT INTO schema_version (version, file)"
+                "INSERT INTO schema_deltas (version, file)"
                 " VALUES (?,?)",
                 (v, relative_path)
             )
 
+            cur.execute(
+                "INSERT INTO schema_version (version, upgraded)"
+                " VALUES (?,?)",
+                (v, True)
+            )
 
-def get_schema_state(txn):
-    sql = (
-        "SELECT MAX(version), file FROM schema_version"
-        " WHERE version = (SELECT MAX(version) FROM schema_version)"
+
+def get_or_create_schema_state(txn):
+    schema_path = os.path.join(
+        dir_path, "schema", "schema_version.sql",
     )
+    create_schema = read_schema(schema_path)
+    txn.executescript(create_schema)
 
-    try:
-        txn.execute(sql)
+    txn.execute("SELECT version, upgraded FROM schema_version")
+    row = txn.fetchone()
+    current_version = int(row[0]) if row else None
+    upgraded = bool(row[1]) if row else None
+
+    if current_version:
+        txn.execute(
+            "SELECT file FROM schema_deltas WHERE version >= ?",
+            (current_version,)
+        )
         res = txn.fetchall()
-
-        if res:
-            current_verison = max(r[0] for r in res)
-            applied_delta = [r[1] for r in res]
-
-            return current_verison, applied_delta
-    except sqlite3.OperationalError:
-        txn.execute("PRAGMA user_version")
-        row = txn.fetchone()
-        if row and row[0]:
-            # FIXME: We need to create schema_version table!
-            return row[0], []
+        return current_version, txn.fetchall(), upgraded
 
     return None
+
+
+def prepare_sqlite3_database(db_conn):
+    """This function should be called before `prepare_database` on sqlite3
+    databases.
+
+    Since we changed the way we store the current schema version and handle
+    updates to schemas, we need a way to upgrade from the old method to the
+    new. This only affects sqlite databases since they were the only ones
+    supported at the time.
+    """
+    with db_conn:
+        schema_path = os.path.join(
+            dir_path, "schema", "schema_version.sql",
+        )
+        create_schema = read_schema(schema_path)
+        db_conn.executescript(create_schema)
+
+        c = db_conn.execute("SELECT * FROM schema_version")
+        rows = c.fetchall()
+        c.close()
+
+        if not rows:
+            c = db_conn.execute("PRAGMA user_version")
+            row = c.fetchone()
+            c.close()
+
+            if row and row[0]:
+                ver = row[0]
+                db_conn.execute(
+                    "INSERT INTO schema_version (version, upgraded)"
+                    " VALUES (?,?)",
+                    (row[0], False)
+                )
diff --git a/synapse/storage/schema/event_edges.sql b/synapse/storage/schema/current/11/event_edges.sql
similarity index 100%
rename from synapse/storage/schema/event_edges.sql
rename to synapse/storage/schema/current/11/event_edges.sql
diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/current/11/event_signatures.sql
similarity index 100%
rename from synapse/storage/schema/event_signatures.sql
rename to synapse/storage/schema/current/11/event_signatures.sql
diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/current/11/im.sql
similarity index 100%
rename from synapse/storage/schema/im.sql
rename to synapse/storage/schema/current/11/im.sql
diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/current/11/keys.sql
similarity index 100%
rename from synapse/storage/schema/keys.sql
rename to synapse/storage/schema/current/11/keys.sql
diff --git a/synapse/storage/schema/media_repository.sql b/synapse/storage/schema/current/11/media_repository.sql
similarity index 100%
rename from synapse/storage/schema/media_repository.sql
rename to synapse/storage/schema/current/11/media_repository.sql
diff --git a/synapse/storage/schema/presence.sql b/synapse/storage/schema/current/11/presence.sql
similarity index 100%
rename from synapse/storage/schema/presence.sql
rename to synapse/storage/schema/current/11/presence.sql
diff --git a/synapse/storage/schema/profiles.sql b/synapse/storage/schema/current/11/profiles.sql
similarity index 100%
rename from synapse/storage/schema/profiles.sql
rename to synapse/storage/schema/current/11/profiles.sql
diff --git a/synapse/storage/schema/redactions.sql b/synapse/storage/schema/current/11/redactions.sql
similarity index 100%
rename from synapse/storage/schema/redactions.sql
rename to synapse/storage/schema/current/11/redactions.sql
diff --git a/synapse/storage/schema/room_aliases.sql b/synapse/storage/schema/current/11/room_aliases.sql
similarity index 100%
rename from synapse/storage/schema/room_aliases.sql
rename to synapse/storage/schema/current/11/room_aliases.sql
diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/current/11/state.sql
similarity index 100%
rename from synapse/storage/schema/state.sql
rename to synapse/storage/schema/current/11/state.sql
diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/current/11/transactions.sql
similarity index 100%
rename from synapse/storage/schema/transactions.sql
rename to synapse/storage/schema/current/11/transactions.sql
diff --git a/synapse/storage/schema/users.sql b/synapse/storage/schema/current/11/users.sql
similarity index 100%
rename from synapse/storage/schema/users.sql
rename to synapse/storage/schema/current/11/users.sql
diff --git a/synapse/storage/schema/delta/v11.sql b/synapse/storage/schema/delta/11/v11.sql
similarity index 100%
rename from synapse/storage/schema/delta/v11.sql
rename to synapse/storage/schema/delta/11/v11.sql
diff --git a/synapse/storage/schema/delta/v12.sql b/synapse/storage/schema/delta/12/v12.sql
similarity index 100%
rename from synapse/storage/schema/delta/v12.sql
rename to synapse/storage/schema/delta/12/v12.sql
diff --git a/synapse/storage/schema/delta/v13.sql b/synapse/storage/schema/delta/13/v13.sql
similarity index 100%
rename from synapse/storage/schema/delta/v13.sql
rename to synapse/storage/schema/delta/13/v13.sql
diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
new file mode 100644
index 0000000000..55e43c41ab
--- /dev/null
+++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
@@ -0,0 +1,20 @@
+import json
+
+
+def run_upgrade(cur):
+    cur.execute("SELECT id, regex FROM application_services_regex")
+    for row in cur.fetchall():
+        try:
+            print "checking %s..." % row[0]
+            json.loads(row[1])
+        except ValueError:
+            # row isn't in json, make it so.
+            string_regex = row[1]
+            new_regex = json.dumps({
+                "regex": string_regex,
+                "exclusive": True
+            })
+            cur.execute(
+                "UPDATE application_services_regex SET regex=? WHERE id=?",
+                (new_regex, row[0])
+            )
diff --git a/synapse/storage/schema/delta/v2.sql b/synapse/storage/schema/delta/v2.sql
deleted file mode 100644
index f740f6dd5d..0000000000
--- a/synapse/storage/schema/delta/v2.sql
+++ /dev/null
@@ -1,168 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS events(
-    stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
-    topological_ordering INTEGER NOT NULL,
-    event_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    content TEXT NOT NULL,
-    unrecognized_keys TEXT,
-    processed BOOL NOT NULL,
-    outlier BOOL NOT NULL,
-    CONSTRAINT ev_uniq UNIQUE (event_id)
-);
-
-CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id);
-CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
-CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
-CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
-
-CREATE TABLE IF NOT EXISTS state_events(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    prev_state TEXT
-);
-
-CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id);
-CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
-CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
-CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
-
-
-CREATE TABLE IF NOT EXISTS current_state_events(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE
-);
-
-CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id);
-CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
-CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
-CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
-
-CREATE TABLE IF NOT EXISTS room_memberships(
-    event_id TEXT NOT NULL,
-    user_id TEXT NOT NULL,
-    sender TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    membership TEXT NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id);
-CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
-CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
-
-CREATE TABLE IF NOT EXISTS feedback(
-    event_id TEXT NOT NULL,
-    feedback_type TEXT,
-    target_event_id TEXT,
-    sender TEXT,
-    room_id TEXT
-);
-
-CREATE TABLE IF NOT EXISTS topics(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    topic TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS room_names(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    name TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS rooms(
-    room_id TEXT PRIMARY KEY NOT NULL,
-    is_public INTEGER,
-    creator TEXT
-);
-
-CREATE TABLE IF NOT EXISTS room_join_rules(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    join_rule TEXT NOT NULL
-);
-CREATE INDEX IF NOT EXISTS room_join_rules_event_id ON room_join_rules(event_id);
-CREATE INDEX IF NOT EXISTS room_join_rules_room_id ON room_join_rules(room_id);
-
-
-CREATE TABLE IF NOT EXISTS room_power_levels(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    user_id TEXT NOT NULL,
-    level INTEGER NOT NULL
-);
-CREATE INDEX IF NOT EXISTS room_power_levels_event_id ON room_power_levels(event_id);
-CREATE INDEX IF NOT EXISTS room_power_levels_room_id ON room_power_levels(room_id);
-CREATE INDEX IF NOT EXISTS room_power_levels_room_user ON room_power_levels(room_id, user_id);
-
-
-CREATE TABLE IF NOT EXISTS room_default_levels(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    level INTEGER NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS room_default_levels_event_id ON room_default_levels(event_id);
-CREATE INDEX IF NOT EXISTS room_default_levels_room_id ON room_default_levels(room_id);
-
-
-CREATE TABLE IF NOT EXISTS room_add_state_levels(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    level INTEGER NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS room_add_state_levels_event_id ON room_add_state_levels(event_id);
-CREATE INDEX IF NOT EXISTS room_add_state_levels_room_id ON room_add_state_levels(room_id);
-
-
-CREATE TABLE IF NOT EXISTS room_send_event_levels(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    level INTEGER NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS room_send_event_levels_event_id ON room_send_event_levels(event_id);
-CREATE INDEX IF NOT EXISTS room_send_event_levels_room_id ON room_send_event_levels(room_id);
-
-
-CREATE TABLE IF NOT EXISTS room_ops_levels(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    ban_level INTEGER,
-    kick_level INTEGER
-);
-
-CREATE INDEX IF NOT EXISTS room_ops_levels_event_id ON room_ops_levels(event_id);
-CREATE INDEX IF NOT EXISTS room_ops_levels_room_id ON room_ops_levels(room_id);
-
-
-CREATE TABLE IF NOT EXISTS room_hosts(
-    room_id TEXT NOT NULL,
-    host TEXT NOT NULL,
-    CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE
-);
-
-CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
-
-PRAGMA user_version = 2;
diff --git a/synapse/storage/schema/delta/v3.sql b/synapse/storage/schema/delta/v3.sql
deleted file mode 100644
index c67e38ff52..0000000000
--- a/synapse/storage/schema/delta/v3.sql
+++ /dev/null
@@ -1,27 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-CREATE INDEX IF NOT EXISTS room_aliases_alias ON room_aliases(room_alias);
-CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id);
-
-
-CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias);
-
-DELETE FROM room_aliases WHERE rowid NOT IN (SELECT max(rowid) FROM room_aliases GROUP BY room_alias, room_id);
-
-CREATE UNIQUE INDEX IF NOT EXISTS room_aliases_uniq ON room_aliases(room_alias, room_id);
-
-PRAGMA user_version = 3;
diff --git a/synapse/storage/schema/delta/v4.sql b/synapse/storage/schema/delta/v4.sql
deleted file mode 100644
index d3807b7686..0000000000
--- a/synapse/storage/schema/delta/v4.sql
+++ /dev/null
@@ -1,26 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS redactions (
-    event_id TEXT NOT NULL,
-    redacts TEXT NOT NULL,
-    CONSTRAINT ev_uniq UNIQUE (event_id)
-);
-
-CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
-CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
-
-ALTER TABLE room_ops_levels ADD COLUMN redact_level INTEGER;
-
-PRAGMA user_version = 4;
diff --git a/synapse/storage/schema/delta/v5.sql b/synapse/storage/schema/delta/v5.sql
deleted file mode 100644
index 0874a15431..0000000000
--- a/synapse/storage/schema/delta/v5.sql
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS user_ips (
-    user TEXT NOT NULL,
-    access_token TEXT NOT NULL,
-    device_id TEXT,
-    ip TEXT NOT NULL,
-    user_agent TEXT NOT NULL,
-    last_seen INTEGER NOT NULL,
-    CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE
-);
-
-CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
-
-ALTER TABLE users ADD COLUMN admin BOOL DEFAULT 0 NOT NULL;
-
-PRAGMA user_version = 5;
diff --git a/synapse/storage/schema/delta/v6.sql b/synapse/storage/schema/delta/v6.sql
deleted file mode 100644
index a9e0a4fe0d..0000000000
--- a/synapse/storage/schema/delta/v6.sql
+++ /dev/null
@@ -1,31 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS server_tls_certificates(
-  server_name TEXT, -- Server name.
-  fingerprint TEXT, -- Certificate fingerprint.
-  from_server TEXT, -- Which key server the certificate was fetched from.
-  ts_added_ms INTEGER, -- When the certifcate was added.
-  tls_certificate BLOB, -- DER encoded x509 certificate.
-  CONSTRAINT uniqueness UNIQUE (server_name, fingerprint)
-);
-
-CREATE TABLE IF NOT EXISTS server_signature_keys(
-  server_name TEXT, -- Server name.
-  key_id TEXT, -- Key version.
-  from_server TEXT, -- Which key server the key was fetched form.
-  ts_added_ms INTEGER, -- When the key was added.
-  verify_key BLOB, -- NACL verification key.
-  CONSTRAINT uniqueness UNIQUE (server_name, key_id)
-);
diff --git a/synapse/storage/schema/delta/v8.sql b/synapse/storage/schema/delta/v8.sql
deleted file mode 100644
index 1e9f8b18cb..0000000000
--- a/synapse/storage/schema/delta/v8.sql
+++ /dev/null
@@ -1,34 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- CREATE TABLE IF NOT EXISTS event_signatures_2 (
-    event_id TEXT,
-    signature_name TEXT,
-    key_id TEXT,
-    signature BLOB,
-    CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id)
-);
-
-INSERT INTO event_signatures_2 (event_id, signature_name, key_id, signature)
-SELECT event_id, signature_name, key_id, signature FROM event_signatures;
-
-DROP TABLE event_signatures;
-ALTER TABLE event_signatures_2 RENAME TO event_signatures;
-
-CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures (
-    event_id
-);
-
-PRAGMA user_version = 8;
\ No newline at end of file
diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql
deleted file mode 100644
index 455d51a70c..0000000000
--- a/synapse/storage/schema/delta/v9.sql
+++ /dev/null
@@ -1,79 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
--- To track destination health
-CREATE TABLE IF NOT EXISTS destinations(
-    destination TEXT PRIMARY KEY,
-    retry_last_ts INTEGER,
-    retry_interval INTEGER
-);
-
-
-CREATE TABLE IF NOT EXISTS local_media_repository (
-    media_id TEXT, -- The id used to refer to the media.
-    media_type TEXT, -- The MIME-type of the media.
-    media_length INTEGER, -- Length of the media in bytes.
-    created_ts INTEGER, -- When the content was uploaded in ms.
-    upload_name TEXT, -- The name the media was uploaded with.
-    user_id TEXT, -- The user who uploaded the file.
-    CONSTRAINT uniqueness UNIQUE (media_id)
-);
-
-CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
-    media_id TEXT, -- The id used to refer to the media.
-    thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
-    thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
-    thumbnail_method TEXT, -- The method used to make the thumbnail.
-    thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    CONSTRAINT uniqueness UNIQUE (
-        media_id, thumbnail_width, thumbnail_height, thumbnail_type
-    )
-);
-
-CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
-    ON local_media_repository_thumbnails (media_id);
-
-CREATE TABLE IF NOT EXISTS remote_media_cache (
-    media_origin TEXT, -- The remote HS the media came from.
-    media_id TEXT, -- The id used to refer to the media on that server.
-    media_type TEXT, -- The MIME-type of the media.
-    created_ts INTEGER, -- When the content was uploaded in ms.
-    upload_name TEXT, -- The name the media was uploaded with.
-    media_length INTEGER, -- Length of the media in bytes.
-    filesystem_id TEXT, -- The name used to store the media on disk.
-    CONSTRAINT uniqueness UNIQUE (media_origin, media_id)
-);
-
-CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
-    media_origin TEXT, -- The remote HS the media came from.
-    media_id TEXT, -- The id used to refer to the media.
-    thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
-    thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_method TEXT, -- The method used to make the thumbnail
-    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
-    thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    filesystem_id TEXT, -- The name used to store the media on disk.
-    CONSTRAINT uniqueness UNIQUE (
-        media_origin, media_id, thumbnail_width, thumbnail_height,
-        thumbnail_type, thumbnail_type
-    )
-);
-
-CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
-    ON local_media_repository_thumbnails (media_id);
-
-
-PRAGMA user_version = 9;
diff --git a/synapse/storage/schema/filtering.sql b/synapse/storage/schema/filtering.sql
deleted file mode 100644
index beb39ca201..0000000000
--- a/synapse/storage/schema/filtering.sql
+++ /dev/null
@@ -1,24 +0,0 @@
-/* Copyright 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS user_filters(
-  user_id TEXT,
-  filter_id INTEGER,
-  filter_json TEXT,
-  FOREIGN KEY(user_id) REFERENCES users(id)
-);
-
-CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
-  user_id, filter_id
-);
diff --git a/synapse/storage/schema/pusher.sql b/synapse/storage/schema/pusher.sql
deleted file mode 100644
index 3735b11547..0000000000
--- a/synapse/storage/schema/pusher.sql
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Copyright 2014 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--- Push notification endpoints that users have configured
-CREATE TABLE IF NOT EXISTS pushers (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_name TEXT NOT NULL,
-  profile_tag varchar(32) NOT NULL,
-  kind varchar(8) NOT NULL,
-  app_id varchar(64) NOT NULL,
-  app_display_name varchar(64) NOT NULL,
-  device_display_name varchar(128) NOT NULL,
-  pushkey blob NOT NULL,
-  ts BIGINT NOT NULL,
-  lang varchar(8),
-  data blob,
-  last_token TEXT,
-  last_success BIGINT,
-  failing_since BIGINT,
-  FOREIGN KEY(user_name) REFERENCES users(name),
-  UNIQUE (app_id, pushkey)
-);
-
-CREATE TABLE IF NOT EXISTS push_rules (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_name TEXT NOT NULL,
-  rule_id TEXT NOT NULL,
-  priority_class TINYINT NOT NULL,
-  priority INTEGER NOT NULL DEFAULT 0,
-  conditions TEXT NOT NULL,
-  actions TEXT NOT NULL,
-  UNIQUE(user_name, rule_id)
-);
-
-CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
diff --git a/synapse/storage/schema/rejections.sql b/synapse/storage/schema/rejections.sql
deleted file mode 100644
index bd2a8b1bb5..0000000000
--- a/synapse/storage/schema/rejections.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-/* Copyright 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS rejections(
-    event_id TEXT NOT NULL,
-    reason TEXT NOT NULL,
-    last_check TEXT NOT NULL,
-    CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE
-);
diff --git a/synapse/storage/schema/application_services.sql b/synapse/storage/schema/schema_version.sql
similarity index 51%
rename from synapse/storage/schema/application_services.sql
rename to synapse/storage/schema/schema_version.sql
index e491ad5aec..83a8c7b7ce 100644
--- a/synapse/storage/schema/application_services.sql
+++ b/synapse/storage/schema/schema_version.sql
@@ -13,22 +13,17 @@
  * limitations under the License.
  */
 
-CREATE TABLE IF NOT EXISTS application_services(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    url TEXT,
-    token TEXT,
-    hs_token TEXT,
-    sender TEXT,
-    UNIQUE(token) ON CONFLICT ROLLBACK
+CREATE TABLE IF NOT EXISTS schema_version(
+    Lock char(1) NOT NULL DEFAULT 'X',  -- Makes sure this table only has one row.
+    version INTEGER NOT NULL,
+    upgraded BOOL NOT NULL,  -- Whether we reached this version from an upgrade or an initial schema.
+    CONSTRAINT schema_version_lock CHECK (Lock='X') ON CONFLICT REPLACE
 );
 
-CREATE TABLE IF NOT EXISTS application_services_regex(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    as_id INTEGER NOT NULL,
-    namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
-    regex TEXT,
-    FOREIGN KEY(as_id) REFERENCES application_services(id)
+CREATE TABLE IF NOT EXISTS schema_deltas(
+    version INTEGER NOT NULL,
+    file TEXT NOT NULL,
+    CONSTRAINT schema_deltas_ver_file UNIQUE (version, file) ON CONFLICT IGNORE
 );
 
-
-
+CREATE INDEX IF NOT EXISTS schema_deltas_ver ON schema_deltas(version);

From 811355ccd0b1a22560ba0af57fd3ad1e07ad283f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 13:11:01 +0000
Subject: [PATCH 04/14] Add some docs and remove unused variables

---
 synapse/storage/__init__.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 07ccc4e2ee..565abe81d9 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -613,7 +613,7 @@ def _setup_new_database(cur):
                 valid_dirs.append((ver, abs_path))
 
     if not valid_dirs:
-        raise RuntimeError("Could not find a suitable current.sql")
+        raise PrepareDatabaseException("Could not find a suitable current.sql")
 
     max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0])
 
@@ -684,7 +684,9 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
             directory_entries = os.listdir(delta_dir)
         except OSError:
             logger.exception("Could not open delta dir for version %d", v)
-            raise
+            raise UpgradeDatabaseException(
+                "Could not open delta dir for version %d" % (v,)
+            )
 
         directory_entries.sort()
         for file_name in directory_entries:
@@ -697,18 +699,20 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
             )
             root_name, ext = os.path.splitext(file_name)
             if ext == ".py":
+                # This is a python upgrade module. We need to import into some
+                # package and then execute its `run_upgrade` function.
                 module_name = "synapse.storage.v%d_%s" % (
                     v, root_name
                 )
-                with open(absolute_path) as schema_file:
+                with open(absolute_path) as python_file:
                     module = imp.load_source(
-                        module_name, absolute_path, schema_file
+                        module_name, absolute_path, python_file
                     )
                 logger.debug("Running script %s", relative_path)
                 module.run_upgrade(cur)
             elif ext == ".sql":
-                with open(absolute_path) as schema_file:
-                    delta_schema = schema_file.read()
+                # A plain old .sql file, just read and execute it
+                delta_schema = read_schema(absolute_path)
                 logger.debug("Applying schema %s", relative_path)
                 cur.executescript(delta_schema)
             else:
@@ -751,7 +755,6 @@ def get_or_create_schema_state(txn):
             "SELECT file FROM schema_deltas WHERE version >= ?",
             (current_version,)
         )
-        res = txn.fetchall()
         return current_version, txn.fetchall(), upgraded
 
     return None
@@ -783,7 +786,6 @@ def prepare_sqlite3_database(db_conn):
             c.close()
 
             if row and row[0]:
-                ver = row[0]
                 db_conn.execute(
                     "INSERT INTO schema_version (version, upgraded)"
                     " VALUES (?,?)",

From c3530c3fb3f374bcb93097f23d311db979912d92 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 13:34:11 +0000
Subject: [PATCH 05/14] More docs. Rename 'schema/current' to
 'schema/full_schemas'

---
 synapse/storage/__init__.py                   |  30 ++++-
 .../storage/schema/current/11/event_edges.sql |  89 -------------
 .../schema/current/11/event_signatures.sql    |  65 ---------
 synapse/storage/schema/current/11/im.sql      | 125 ------------------
 synapse/storage/schema/current/11/keys.sql    |  31 -----
 .../schema/current/11/media_repository.sql    |  68 ----------
 .../storage/schema/current/11/presence.sql    |  38 ------
 .../storage/schema/current/11/profiles.sql    |  20 ---
 .../storage/schema/current/11/redactions.sql  |  22 ---
 .../schema/current/11/room_aliases.sql        |  27 ----
 synapse/storage/schema/current/11/state.sql   |  47 -------
 .../schema/current/11/transactions.sql        |  68 ----------
 synapse/storage/schema/current/11/users.sql   |  45 -------
 13 files changed, 23 insertions(+), 652 deletions(-)
 delete mode 100644 synapse/storage/schema/current/11/event_edges.sql
 delete mode 100644 synapse/storage/schema/current/11/event_signatures.sql
 delete mode 100644 synapse/storage/schema/current/11/im.sql
 delete mode 100644 synapse/storage/schema/current/11/keys.sql
 delete mode 100644 synapse/storage/schema/current/11/media_repository.sql
 delete mode 100644 synapse/storage/schema/current/11/presence.sql
 delete mode 100644 synapse/storage/schema/current/11/profiles.sql
 delete mode 100644 synapse/storage/schema/current/11/redactions.sql
 delete mode 100644 synapse/storage/schema/current/11/room_aliases.sql
 delete mode 100644 synapse/storage/schema/current/11/state.sql
 delete mode 100644 synapse/storage/schema/current/11/transactions.sql
 delete mode 100644 synapse/storage/schema/current/11/users.sql

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 565abe81d9..a22a0e2f39 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -55,8 +55,8 @@ import re
 logger = logging.getLogger(__name__)
 
 
-# Remember to update this number every time an incompatible change is made to
-# database schema files, so the users will be informed on server restarts.
+# Remember to update this number every time an change is made to database
+# schema files, so the users will be informed on server restarts.
 SCHEMA_VERSION = 14
 
 dir_path = os.path.abspath(os.path.dirname(__file__))
@@ -584,7 +584,7 @@ def prepare_database(db_conn):
     or upgrade from an older schema version.
     """
     cur = db_conn.cursor()
-    version_info = get_or_create_schema_state(cur)
+    version_info = _get_or_create_schema_state(cur)
 
     if version_info:
         user_version, delta_files, upgraded = version_info
@@ -599,7 +599,17 @@ def prepare_database(db_conn):
 
 
 def _setup_new_database(cur):
-    current_dir = os.path.join(dir_path, "schema", "current")
+    """Sets up the database by finding a base set of "full schemas" and then
+    applying any necessary deltas.
+
+    The "full_schemas" directory has subdirectories named after versions. This
+    function searches for the highest version less than or equal to
+    `SCHEMA_VERSION` and excutes all .sql files in that directory.
+
+    The function will then apply all deltas for all versions after the base
+    version.
+    """
+    current_dir = os.path.join(dir_path, "schema", "full_schemas")
     directory_entries = os.listdir(current_dir)
 
     valid_dirs = []
@@ -609,11 +619,15 @@ def _setup_new_database(cur):
         abs_path = os.path.join(current_dir, filename)
         if match and os.path.isdir(abs_path):
             ver = int(match.group(0))
-            if ver < SCHEMA_VERSION:
+            if ver <= SCHEMA_VERSION:
                 valid_dirs.append((ver, abs_path))
+        else:
+            logger.warn("Unexpected entry in 'full_schemas': %s", filename)
 
     if not valid_dirs:
-        raise PrepareDatabaseException("Could not find a suitable current.sql")
+        raise PrepareDatabaseException(
+            "Could not find a suitable base set of full schemas"
+        )
 
     max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0])
 
@@ -655,6 +669,8 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
     even if there has been no version bump. This is useful for development
     where orthogonal schema changes may happen on separate branches.
 
+    This is a no-op of current_version == SCHEMA_VERSION.
+
     Args:
         cur (Cursor)
         current_version (int): The current version of the schema
@@ -738,7 +754,7 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
             )
 
 
-def get_or_create_schema_state(txn):
+def _get_or_create_schema_state(txn):
     schema_path = os.path.join(
         dir_path, "schema", "schema_version.sql",
     )
diff --git a/synapse/storage/schema/current/11/event_edges.sql b/synapse/storage/schema/current/11/event_edges.sql
deleted file mode 100644
index 1e766d6db2..0000000000
--- a/synapse/storage/schema/current/11/event_edges.sql
+++ /dev/null
@@ -1,89 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS event_forward_extremities(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
-);
-
-CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
-CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_backward_extremities(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
-);
-
-CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
-CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_edges(
-    event_id TEXT NOT NULL,
-    prev_event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    is_state INTEGER NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state)
-);
-
-CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
-CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
-
-
-CREATE TABLE IF NOT EXISTS room_depth(
-    room_id TEXT NOT NULL,
-    min_depth INTEGER NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (room_id)
-);
-
-CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
-
-
-create TABLE IF NOT EXISTS event_destinations(
-    event_id TEXT NOT NULL,
-    destination TEXT NOT NULL,
-    delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered
-    CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE
-);
-
-CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
-
-
-CREATE TABLE IF NOT EXISTS state_forward_extremities(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
-);
-
-CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
-    room_id, type, state_key
-);
-CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
-
-
-CREATE TABLE IF NOT EXISTS event_auth(
-    event_id TEXT NOT NULL,
-    auth_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id)
-);
-
-CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
-CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
\ No newline at end of file
diff --git a/synapse/storage/schema/current/11/event_signatures.sql b/synapse/storage/schema/current/11/event_signatures.sql
deleted file mode 100644
index c28c39c48a..0000000000
--- a/synapse/storage/schema/current/11/event_signatures.sql
+++ /dev/null
@@ -1,65 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS event_content_hashes (
-    event_id TEXT,
-    algorithm TEXT,
-    hash BLOB,
-    CONSTRAINT uniqueness UNIQUE (event_id, algorithm)
-);
-
-CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(
-    event_id
-);
-
-
-CREATE TABLE IF NOT EXISTS event_reference_hashes (
-    event_id TEXT,
-    algorithm TEXT,
-    hash BLOB,
-    CONSTRAINT uniqueness UNIQUE (event_id, algorithm)
-);
-
-CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes (
-    event_id
-);
-
-
-CREATE TABLE IF NOT EXISTS event_signatures (
-    event_id TEXT,
-    signature_name TEXT,
-    key_id TEXT,
-    signature BLOB,
-    CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id)
-);
-
-CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures (
-    event_id
-);
-
-
-CREATE TABLE IF NOT EXISTS event_edge_hashes(
-    event_id TEXT,
-    prev_event_id TEXT,
-    algorithm TEXT,
-    hash BLOB,
-    CONSTRAINT uniqueness UNIQUE (
-        event_id, prev_event_id, algorithm
-    )
-);
-
-CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(
-    event_id
-);
diff --git a/synapse/storage/schema/current/11/im.sql b/synapse/storage/schema/current/11/im.sql
deleted file mode 100644
index dd00c1cd2f..0000000000
--- a/synapse/storage/schema/current/11/im.sql
+++ /dev/null
@@ -1,125 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS events(
-    stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
-    topological_ordering INTEGER NOT NULL,
-    event_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    content TEXT NOT NULL,
-    unrecognized_keys TEXT,
-    processed BOOL NOT NULL,
-    outlier BOOL NOT NULL,
-    depth INTEGER DEFAULT 0 NOT NULL,
-    CONSTRAINT ev_uniq UNIQUE (event_id)
-);
-
-CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id);
-CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
-CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
-CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
-
-
-CREATE TABLE IF NOT EXISTS event_json(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    internal_metadata NOT NULL,
-    json BLOB NOT NULL,
-    CONSTRAINT ev_j_uniq UNIQUE (event_id)
-);
-
-CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id);
-CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
-
-
-CREATE TABLE IF NOT EXISTS state_events(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    prev_state TEXT
-);
-
-CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id);
-CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
-CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
-CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
-
-
-CREATE TABLE IF NOT EXISTS current_state_events(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE
-);
-
-CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id);
-CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
-CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
-CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
-
-CREATE TABLE IF NOT EXISTS room_memberships(
-    event_id TEXT NOT NULL,
-    user_id TEXT NOT NULL,
-    sender TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    membership TEXT NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id);
-CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
-CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
-
-CREATE TABLE IF NOT EXISTS feedback(
-    event_id TEXT NOT NULL,
-    feedback_type TEXT,
-    target_event_id TEXT,
-    sender TEXT,
-    room_id TEXT
-);
-
-CREATE TABLE IF NOT EXISTS topics(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    topic TEXT NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id);
-CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
-
-CREATE TABLE IF NOT EXISTS room_names(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    name TEXT NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id);
-CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
-
-CREATE TABLE IF NOT EXISTS rooms(
-    room_id TEXT PRIMARY KEY NOT NULL,
-    is_public INTEGER,
-    creator TEXT
-);
-
-CREATE TABLE IF NOT EXISTS room_hosts(
-    room_id TEXT NOT NULL,
-    host TEXT NOT NULL,
-    CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE
-);
-
-CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/current/11/keys.sql b/synapse/storage/schema/current/11/keys.sql
deleted file mode 100644
index a9e0a4fe0d..0000000000
--- a/synapse/storage/schema/current/11/keys.sql
+++ /dev/null
@@ -1,31 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS server_tls_certificates(
-  server_name TEXT, -- Server name.
-  fingerprint TEXT, -- Certificate fingerprint.
-  from_server TEXT, -- Which key server the certificate was fetched from.
-  ts_added_ms INTEGER, -- When the certifcate was added.
-  tls_certificate BLOB, -- DER encoded x509 certificate.
-  CONSTRAINT uniqueness UNIQUE (server_name, fingerprint)
-);
-
-CREATE TABLE IF NOT EXISTS server_signature_keys(
-  server_name TEXT, -- Server name.
-  key_id TEXT, -- Key version.
-  from_server TEXT, -- Which key server the key was fetched form.
-  ts_added_ms INTEGER, -- When the key was added.
-  verify_key BLOB, -- NACL verification key.
-  CONSTRAINT uniqueness UNIQUE (server_name, key_id)
-);
diff --git a/synapse/storage/schema/current/11/media_repository.sql b/synapse/storage/schema/current/11/media_repository.sql
deleted file mode 100644
index afdf48cbfb..0000000000
--- a/synapse/storage/schema/current/11/media_repository.sql
+++ /dev/null
@@ -1,68 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS local_media_repository (
-    media_id TEXT, -- The id used to refer to the media.
-    media_type TEXT, -- The MIME-type of the media.
-    media_length INTEGER, -- Length of the media in bytes.
-    created_ts INTEGER, -- When the content was uploaded in ms.
-    upload_name TEXT, -- The name the media was uploaded with.
-    user_id TEXT, -- The user who uploaded the file.
-    CONSTRAINT uniqueness UNIQUE (media_id)
-);
-
-CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
-    media_id TEXT, -- The id used to refer to the media.
-    thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
-    thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
-    thumbnail_method TEXT, -- The method used to make the thumbnail.
-    thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    CONSTRAINT uniqueness UNIQUE (
-        media_id, thumbnail_width, thumbnail_height, thumbnail_type
-    )
-);
-
-CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
-    ON local_media_repository_thumbnails (media_id);
-
-CREATE TABLE IF NOT EXISTS remote_media_cache (
-    media_origin TEXT, -- The remote HS the media came from.
-    media_id TEXT, -- The id used to refer to the media on that server.
-    media_type TEXT, -- The MIME-type of the media.
-    created_ts INTEGER, -- When the content was uploaded in ms.
-    upload_name TEXT, -- The name the media was uploaded with.
-    media_length INTEGER, -- Length of the media in bytes.
-    filesystem_id TEXT, -- The name used to store the media on disk.
-    CONSTRAINT uniqueness UNIQUE (media_origin, media_id)
-);
-
-CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
-    media_origin TEXT, -- The remote HS the media came from.
-    media_id TEXT, -- The id used to refer to the media.
-    thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
-    thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_method TEXT, -- The method used to make the thumbnail
-    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
-    thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    filesystem_id TEXT, -- The name used to store the media on disk.
-    CONSTRAINT uniqueness UNIQUE (
-        media_origin, media_id, thumbnail_width, thumbnail_height,
-        thumbnail_type, thumbnail_type
-    )
-);
-
-CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
-    ON local_media_repository_thumbnails (media_id);
diff --git a/synapse/storage/schema/current/11/presence.sql b/synapse/storage/schema/current/11/presence.sql
deleted file mode 100644
index f9f8db9697..0000000000
--- a/synapse/storage/schema/current/11/presence.sql
+++ /dev/null
@@ -1,38 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS presence(
-  user_id INTEGER NOT NULL,
-  state INTEGER,
-  status_msg TEXT,
-  mtime INTEGER, -- miliseconds since last state change
-  FOREIGN KEY(user_id) REFERENCES users(id)
-);
-
--- For each of /my/ users which possibly-remote users are allowed to see their
--- presence state
-CREATE TABLE IF NOT EXISTS presence_allow_inbound(
-  observed_user_id INTEGER NOT NULL,
-  observer_user_id TEXT, -- a UserID,
-  FOREIGN KEY(observed_user_id) REFERENCES users(id)
-);
-
--- For each of /my/ users (watcher), which possibly-remote users are they
--- watching?
-CREATE TABLE IF NOT EXISTS presence_list(
-  user_id INTEGER NOT NULL,
-  observed_user_id TEXT, -- a UserID,
-  accepted BOOLEAN,
-  FOREIGN KEY(user_id) REFERENCES users(id)
-);
diff --git a/synapse/storage/schema/current/11/profiles.sql b/synapse/storage/schema/current/11/profiles.sql
deleted file mode 100644
index f06a528b4d..0000000000
--- a/synapse/storage/schema/current/11/profiles.sql
+++ /dev/null
@@ -1,20 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS profiles(
-    user_id INTEGER NOT NULL,
-    displayname TEXT,
-    avatar_url TEXT,
-    FOREIGN KEY(user_id) REFERENCES users(id)
-);
diff --git a/synapse/storage/schema/current/11/redactions.sql b/synapse/storage/schema/current/11/redactions.sql
deleted file mode 100644
index 5011d95db8..0000000000
--- a/synapse/storage/schema/current/11/redactions.sql
+++ /dev/null
@@ -1,22 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS redactions (
-    event_id TEXT NOT NULL,
-    redacts TEXT NOT NULL,
-    CONSTRAINT ev_uniq UNIQUE (event_id)
-);
-
-CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
-CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/current/11/room_aliases.sql b/synapse/storage/schema/current/11/room_aliases.sql
deleted file mode 100644
index 0d2df01603..0000000000
--- a/synapse/storage/schema/current/11/room_aliases.sql
+++ /dev/null
@@ -1,27 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS room_aliases(
-    room_alias TEXT NOT NULL,
-    room_id TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS room_alias_servers(
-    room_alias TEXT NOT NULL,
-    server TEXT NOT NULL
-);
-
-
-
diff --git a/synapse/storage/schema/current/11/state.sql b/synapse/storage/schema/current/11/state.sql
deleted file mode 100644
index 1fe8f1e430..0000000000
--- a/synapse/storage/schema/current/11/state.sql
+++ /dev/null
@@ -1,47 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-CREATE TABLE IF NOT EXISTS state_groups(
-    id INTEGER PRIMARY KEY,
-    room_id TEXT NOT NULL,
-    event_id TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS state_groups_state(
-    state_group INTEGER NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    event_id TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS event_to_state_groups(
-    event_id TEXT NOT NULL,
-    state_group INTEGER NOT NULL,
-    CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id)
-);
-
-CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
-
-CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(
-    state_group
-);
-CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(
-    room_id, type, state_key
-);
-
-CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(
-    event_id
-);
\ No newline at end of file
diff --git a/synapse/storage/schema/current/11/transactions.sql b/synapse/storage/schema/current/11/transactions.sql
deleted file mode 100644
index 2d30f99b06..0000000000
--- a/synapse/storage/schema/current/11/transactions.sql
+++ /dev/null
@@ -1,68 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--- Stores what transaction ids we have received and what our response was
-CREATE TABLE IF NOT EXISTS received_transactions(
-    transaction_id TEXT, 
-    origin TEXT, 
-    ts INTEGER,
-    response_code INTEGER,
-    response_json TEXT,
-    has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
-    CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE
-);
-
-CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin);
-CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
-
-
--- Stores what transactions we've sent, what their response was (if we got one) and whether we have
--- since referenced the transaction in another outgoing transaction
-CREATE TABLE IF NOT EXISTS sent_transactions(
-    id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
-    transaction_id TEXT,
-    destination TEXT,
-    response_code INTEGER DEFAULT 0,
-    response_json TEXT,
-    ts INTEGER
-);
-
-CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(
-    destination
-);
-CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
--- So that we can do an efficient look up of all transactions that have yet to be successfully
--- sent.
-CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code);
-
-
--- For sent transactions only.
-CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
-    transaction_id INTEGER,
-    destination TEXT,
-    pdu_id TEXT,
-    pdu_origin TEXT
-);
-
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination);
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination);
-
--- To track destination health
-CREATE TABLE IF NOT EXISTS destinations(
-    destination TEXT PRIMARY KEY,
-    retry_last_ts INTEGER,
-    retry_interval INTEGER
-);
diff --git a/synapse/storage/schema/current/11/users.sql b/synapse/storage/schema/current/11/users.sql
deleted file mode 100644
index 08ccfdac0a..0000000000
--- a/synapse/storage/schema/current/11/users.sql
+++ /dev/null
@@ -1,45 +0,0 @@
-/* Copyright 2014, 2015 OpenMarket Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-CREATE TABLE IF NOT EXISTS users(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    name TEXT,
-    password_hash TEXT,
-    creation_ts INTEGER,
-    admin BOOL DEFAULT 0 NOT NULL,
-    UNIQUE(name) ON CONFLICT ROLLBACK
-);
-
-CREATE TABLE IF NOT EXISTS access_tokens(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    user_id INTEGER NOT NULL,
-    device_id TEXT,
-    token TEXT NOT NULL,
-    last_used INTEGER,
-    FOREIGN KEY(user_id) REFERENCES users(id),
-    UNIQUE(token) ON CONFLICT ROLLBACK
-);
-
-CREATE TABLE IF NOT EXISTS user_ips (
-    user TEXT NOT NULL,
-    access_token TEXT NOT NULL,
-    device_id TEXT,
-    ip TEXT NOT NULL,
-    user_agent TEXT NOT NULL,
-    last_seen INTEGER NOT NULL,
-    CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE
-);
-
-CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
-

From 8c8354e85a702ef36de15db50d294e1a724ced7d Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 13:34:38 +0000
Subject: [PATCH 06/14] Actually add full_schemas dir

---
 .../schema/full_schemas/11/event_edges.sql    |  89 +++++++++++++
 .../full_schemas/11/event_signatures.sql      |  65 +++++++++
 synapse/storage/schema/full_schemas/11/im.sql | 125 ++++++++++++++++++
 .../storage/schema/full_schemas/11/keys.sql   |  31 +++++
 .../full_schemas/11/media_repository.sql      |  68 ++++++++++
 .../schema/full_schemas/11/presence.sql       |  38 ++++++
 .../schema/full_schemas/11/profiles.sql       |  20 +++
 .../schema/full_schemas/11/redactions.sql     |  22 +++
 .../schema/full_schemas/11/room_aliases.sql   |  27 ++++
 .../storage/schema/full_schemas/11/state.sql  |  47 +++++++
 .../schema/full_schemas/11/transactions.sql   |  68 ++++++++++
 .../storage/schema/full_schemas/11/users.sql  |  45 +++++++
 12 files changed, 645 insertions(+)
 create mode 100644 synapse/storage/schema/full_schemas/11/event_edges.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/event_signatures.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/im.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/keys.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/media_repository.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/presence.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/profiles.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/redactions.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/room_aliases.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/state.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/transactions.sql
 create mode 100644 synapse/storage/schema/full_schemas/11/users.sql

diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
new file mode 100644
index 0000000000..1e766d6db2
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -0,0 +1,89 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS event_forward_extremities(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
+CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_backward_extremities(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
+CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_edges(
+    event_id TEXT NOT NULL,
+    prev_event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    is_state INTEGER NOT NULL,
+    CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state)
+);
+
+CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
+CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
+
+
+CREATE TABLE IF NOT EXISTS room_depth(
+    room_id TEXT NOT NULL,
+    min_depth INTEGER NOT NULL,
+    CONSTRAINT uniqueness UNIQUE (room_id)
+);
+
+CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
+
+
+create TABLE IF NOT EXISTS event_destinations(
+    event_id TEXT NOT NULL,
+    destination TEXT NOT NULL,
+    delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered
+    CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
+
+
+CREATE TABLE IF NOT EXISTS state_forward_extremities(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
+    room_id, type, state_key
+);
+CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_auth(
+    event_id TEXT NOT NULL,
+    auth_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id)
+);
+
+CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
+CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
\ No newline at end of file
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
new file mode 100644
index 0000000000..c28c39c48a
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -0,0 +1,65 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS event_content_hashes (
+    event_id TEXT,
+    algorithm TEXT,
+    hash BLOB,
+    CONSTRAINT uniqueness UNIQUE (event_id, algorithm)
+);
+
+CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(
+    event_id
+);
+
+
+CREATE TABLE IF NOT EXISTS event_reference_hashes (
+    event_id TEXT,
+    algorithm TEXT,
+    hash BLOB,
+    CONSTRAINT uniqueness UNIQUE (event_id, algorithm)
+);
+
+CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes (
+    event_id
+);
+
+
+CREATE TABLE IF NOT EXISTS event_signatures (
+    event_id TEXT,
+    signature_name TEXT,
+    key_id TEXT,
+    signature BLOB,
+    CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id)
+);
+
+CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures (
+    event_id
+);
+
+
+CREATE TABLE IF NOT EXISTS event_edge_hashes(
+    event_id TEXT,
+    prev_event_id TEXT,
+    algorithm TEXT,
+    hash BLOB,
+    CONSTRAINT uniqueness UNIQUE (
+        event_id, prev_event_id, algorithm
+    )
+);
+
+CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(
+    event_id
+);
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
new file mode 100644
index 0000000000..dd00c1cd2f
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -0,0 +1,125 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS events(
+    stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
+    topological_ordering INTEGER NOT NULL,
+    event_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    content TEXT NOT NULL,
+    unrecognized_keys TEXT,
+    processed BOOL NOT NULL,
+    outlier BOOL NOT NULL,
+    depth INTEGER DEFAULT 0 NOT NULL,
+    CONSTRAINT ev_uniq UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id);
+CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
+CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
+CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
+
+
+CREATE TABLE IF NOT EXISTS event_json(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    internal_metadata NOT NULL,
+    json BLOB NOT NULL,
+    CONSTRAINT ev_j_uniq UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id);
+CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
+
+
+CREATE TABLE IF NOT EXISTS state_events(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    prev_state TEXT
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id);
+CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
+CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
+CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
+
+
+CREATE TABLE IF NOT EXISTS current_state_events(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id);
+CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
+CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
+CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
+
+CREATE TABLE IF NOT EXISTS room_memberships(
+    event_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    sender TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    membership TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id);
+CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
+CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
+
+CREATE TABLE IF NOT EXISTS feedback(
+    event_id TEXT NOT NULL,
+    feedback_type TEXT,
+    target_event_id TEXT,
+    sender TEXT,
+    room_id TEXT
+);
+
+CREATE TABLE IF NOT EXISTS topics(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    topic TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id);
+CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
+
+CREATE TABLE IF NOT EXISTS room_names(
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    name TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id);
+CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
+
+CREATE TABLE IF NOT EXISTS rooms(
+    room_id TEXT PRIMARY KEY NOT NULL,
+    is_public INTEGER,
+    creator TEXT
+);
+
+CREATE TABLE IF NOT EXISTS room_hosts(
+    room_id TEXT NOT NULL,
+    host TEXT NOT NULL,
+    CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE
+);
+
+CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
new file mode 100644
index 0000000000..a9e0a4fe0d
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -0,0 +1,31 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS server_tls_certificates(
+  server_name TEXT, -- Server name.
+  fingerprint TEXT, -- Certificate fingerprint.
+  from_server TEXT, -- Which key server the certificate was fetched from.
+  ts_added_ms INTEGER, -- When the certifcate was added.
+  tls_certificate BLOB, -- DER encoded x509 certificate.
+  CONSTRAINT uniqueness UNIQUE (server_name, fingerprint)
+);
+
+CREATE TABLE IF NOT EXISTS server_signature_keys(
+  server_name TEXT, -- Server name.
+  key_id TEXT, -- Key version.
+  from_server TEXT, -- Which key server the key was fetched form.
+  ts_added_ms INTEGER, -- When the key was added.
+  verify_key BLOB, -- NACL verification key.
+  CONSTRAINT uniqueness UNIQUE (server_name, key_id)
+);
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
new file mode 100644
index 0000000000..afdf48cbfb
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -0,0 +1,68 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS local_media_repository (
+    media_id TEXT, -- The id used to refer to the media.
+    media_type TEXT, -- The MIME-type of the media.
+    media_length INTEGER, -- Length of the media in bytes.
+    created_ts INTEGER, -- When the content was uploaded in ms.
+    upload_name TEXT, -- The name the media was uploaded with.
+    user_id TEXT, -- The user who uploaded the file.
+    CONSTRAINT uniqueness UNIQUE (media_id)
+);
+
+CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
+    media_id TEXT, -- The id used to refer to the media.
+    thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
+    thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
+    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
+    thumbnail_method TEXT, -- The method used to make the thumbnail.
+    thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
+    CONSTRAINT uniqueness UNIQUE (
+        media_id, thumbnail_width, thumbnail_height, thumbnail_type
+    )
+);
+
+CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
+    ON local_media_repository_thumbnails (media_id);
+
+CREATE TABLE IF NOT EXISTS remote_media_cache (
+    media_origin TEXT, -- The remote HS the media came from.
+    media_id TEXT, -- The id used to refer to the media on that server.
+    media_type TEXT, -- The MIME-type of the media.
+    created_ts INTEGER, -- When the content was uploaded in ms.
+    upload_name TEXT, -- The name the media was uploaded with.
+    media_length INTEGER, -- Length of the media in bytes.
+    filesystem_id TEXT, -- The name used to store the media on disk.
+    CONSTRAINT uniqueness UNIQUE (media_origin, media_id)
+);
+
+CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
+    media_origin TEXT, -- The remote HS the media came from.
+    media_id TEXT, -- The id used to refer to the media.
+    thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
+    thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
+    thumbnail_method TEXT, -- The method used to make the thumbnail
+    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
+    thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
+    filesystem_id TEXT, -- The name used to store the media on disk.
+    CONSTRAINT uniqueness UNIQUE (
+        media_origin, media_id, thumbnail_width, thumbnail_height,
+        thumbnail_type, thumbnail_type
+    )
+);
+
+CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
+    ON local_media_repository_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
new file mode 100644
index 0000000000..f9f8db9697
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -0,0 +1,38 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS presence(
+  user_id INTEGER NOT NULL,
+  state INTEGER,
+  status_msg TEXT,
+  mtime INTEGER, -- miliseconds since last state change
+  FOREIGN KEY(user_id) REFERENCES users(id)
+);
+
+-- For each of /my/ users which possibly-remote users are allowed to see their
+-- presence state
+CREATE TABLE IF NOT EXISTS presence_allow_inbound(
+  observed_user_id INTEGER NOT NULL,
+  observer_user_id TEXT, -- a UserID,
+  FOREIGN KEY(observed_user_id) REFERENCES users(id)
+);
+
+-- For each of /my/ users (watcher), which possibly-remote users are they
+-- watching?
+CREATE TABLE IF NOT EXISTS presence_list(
+  user_id INTEGER NOT NULL,
+  observed_user_id TEXT, -- a UserID,
+  accepted BOOLEAN,
+  FOREIGN KEY(user_id) REFERENCES users(id)
+);
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
new file mode 100644
index 0000000000..f06a528b4d
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -0,0 +1,20 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS profiles(
+    user_id INTEGER NOT NULL,
+    displayname TEXT,
+    avatar_url TEXT,
+    FOREIGN KEY(user_id) REFERENCES users(id)
+);
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
new file mode 100644
index 0000000000..5011d95db8
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -0,0 +1,22 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS redactions (
+    event_id TEXT NOT NULL,
+    redacts TEXT NOT NULL,
+    CONSTRAINT ev_uniq UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
+CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
new file mode 100644
index 0000000000..0d2df01603
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -0,0 +1,27 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS room_aliases(
+    room_alias TEXT NOT NULL,
+    room_id TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS room_alias_servers(
+    room_alias TEXT NOT NULL,
+    server TEXT NOT NULL
+);
+
+
+
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
new file mode 100644
index 0000000000..1fe8f1e430
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -0,0 +1,47 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS state_groups(
+    id INTEGER PRIMARY KEY,
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS state_groups_state(
+    state_group INTEGER NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    event_id TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS event_to_state_groups(
+    event_id TEXT NOT NULL,
+    state_group INTEGER NOT NULL,
+    CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
+
+CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(
+    state_group
+);
+CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(
+    room_id, type, state_key
+);
+
+CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(
+    event_id
+);
\ No newline at end of file
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
new file mode 100644
index 0000000000..2d30f99b06
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -0,0 +1,68 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-- Stores what transaction ids we have received and what our response was
+CREATE TABLE IF NOT EXISTS received_transactions(
+    transaction_id TEXT, 
+    origin TEXT, 
+    ts INTEGER,
+    response_code INTEGER,
+    response_json TEXT,
+    has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
+    CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin);
+CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
+
+
+-- Stores what transactions we've sent, what their response was (if we got one) and whether we have
+-- since referenced the transaction in another outgoing transaction
+CREATE TABLE IF NOT EXISTS sent_transactions(
+    id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
+    transaction_id TEXT,
+    destination TEXT,
+    response_code INTEGER DEFAULT 0,
+    response_json TEXT,
+    ts INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
+CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(
+    destination
+);
+CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
+-- So that we can do an efficient look up of all transactions that have yet to be successfully
+-- sent.
+CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code);
+
+
+-- For sent transactions only.
+CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
+    transaction_id INTEGER,
+    destination TEXT,
+    pdu_id TEXT,
+    pdu_origin TEXT
+);
+
+CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination);
+CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
+CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination);
+
+-- To track destination health
+CREATE TABLE IF NOT EXISTS destinations(
+    destination TEXT PRIMARY KEY,
+    retry_last_ts INTEGER,
+    retry_interval INTEGER
+);
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
new file mode 100644
index 0000000000..08ccfdac0a
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -0,0 +1,45 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS users(
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT,
+    password_hash TEXT,
+    creation_ts INTEGER,
+    admin BOOL DEFAULT 0 NOT NULL,
+    UNIQUE(name) ON CONFLICT ROLLBACK
+);
+
+CREATE TABLE IF NOT EXISTS access_tokens(
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    device_id TEXT,
+    token TEXT NOT NULL,
+    last_used INTEGER,
+    FOREIGN KEY(user_id) REFERENCES users(id),
+    UNIQUE(token) ON CONFLICT ROLLBACK
+);
+
+CREATE TABLE IF NOT EXISTS user_ips (
+    user TEXT NOT NULL,
+    access_token TEXT NOT NULL,
+    device_id TEXT,
+    ip TEXT NOT NULL,
+    user_agent TEXT NOT NULL,
+    last_seen INTEGER NOT NULL,
+    CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+

From 640e53935dc2585c7a6365980cd288e5ed176b38 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 13:43:17 +0000
Subject: [PATCH 07/14] Use context manager with db conn to correctly commit
 and rollback

---
 synapse/storage/__init__.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index a22a0e2f39..d637e8c4f7 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -55,7 +55,7 @@ import re
 logger = logging.getLogger(__name__)
 
 
-# Remember to update this number every time an change is made to database
+# Remember to update this number every time a change is made to database
 # schema files, so the users will be informed on server restarts.
 SCHEMA_VERSION = 14
 
@@ -583,19 +583,19 @@ def prepare_database(db_conn):
     """Prepares a database for usage. Will either create all necessary tables
     or upgrade from an older schema version.
     """
-    cur = db_conn.cursor()
-    version_info = _get_or_create_schema_state(cur)
+    with db_conn:
+        cur = db_conn.cursor()
+        version_info = _get_or_create_schema_state(cur)
 
-    if version_info:
-        user_version, delta_files, upgraded = version_info
-        _upgrade_existing_database(cur, user_version, delta_files, upgraded)
-    else:
-        _setup_new_database(cur)
+        if version_info:
+            user_version, delta_files, upgraded = version_info
+            _upgrade_existing_database(cur, user_version, delta_files, upgraded)
+        else:
+            _setup_new_database(cur)
 
-    cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,))
-    db_conn.commit()
+        cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,))
 
-    cur.close()
+        cur.close()
 
 
 def _setup_new_database(cur):

From b4c38738f438ee2b035a92d6e26b0f078956575f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 13:43:35 +0000
Subject: [PATCH 08/14] Change to use logger in db upgrade script

---
 synapse/storage/schema/delta/14/upgrade_appservice_db.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
index 55e43c41ab..847b1c5b89 100644
--- a/synapse/storage/schema/delta/14/upgrade_appservice_db.py
+++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py
@@ -1,11 +1,14 @@
 import json
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 def run_upgrade(cur):
     cur.execute("SELECT id, regex FROM application_services_regex")
     for row in cur.fetchall():
         try:
-            print "checking %s..." % row[0]
+            logger.debug("Checking %s..." % row[0])
             json.loads(row[1])
         except ValueError:
             # row isn't in json, make it so.

From 5eefd1f618a19cc27cfdaf915dc3abd3720cd0e7 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 13:52:18 +0000
Subject: [PATCH 09/14] Add unique constraint on schema_version.lock schema.
 Use conflict clause in sql.

---
 synapse/storage/__init__.py               | 6 +++---
 synapse/storage/schema/schema_version.sql | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index d637e8c4f7..c09228c37f 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -645,7 +645,7 @@ def _setup_new_database(cur):
     cur.executescript(sql_script)
 
     cur.execute(
-        "INSERT INTO schema_version (version, upgraded)"
+        "INSERT OR REPLACE INTO schema_version (version, upgraded)"
         " VALUES (?,?)",
         (max_current_ver, False)
     )
@@ -748,7 +748,7 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
             )
 
             cur.execute(
-                "INSERT INTO schema_version (version, upgraded)"
+                "INSERT OR REPLACE INTO schema_version (version, upgraded)"
                 " VALUES (?,?)",
                 (v, True)
             )
@@ -803,7 +803,7 @@ def prepare_sqlite3_database(db_conn):
 
             if row and row[0]:
                 db_conn.execute(
-                    "INSERT INTO schema_version (version, upgraded)"
+                    "INSERT OR REPLACE INTO schema_version (version, upgraded)"
                     " VALUES (?,?)",
                     (row[0], False)
                 )
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql
index 83a8c7b7ce..20b1481ba5 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/schema_version.sql
@@ -17,7 +17,8 @@ CREATE TABLE IF NOT EXISTS schema_version(
     Lock char(1) NOT NULL DEFAULT 'X',  -- Makes sure this table only has one row.
     version INTEGER NOT NULL,
     upgraded BOOL NOT NULL,  -- Whether we reached this version from an upgrade or an initial schema.
-    CONSTRAINT schema_version_lock CHECK (Lock='X') ON CONFLICT REPLACE
+    CONSTRAINT schema_version_lock_x CHECK (Lock='X')
+    CONSTRAINT schema_version_lock_uniq UNIQUE (Lock)
 );
 
 CREATE TABLE IF NOT EXISTS schema_deltas(

From 16dd87d848f686835cbff82333e00c6411b1436b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 14:03:21 +0000
Subject: [PATCH 10/14] Don't assume db conn is a Context Manager.

Twisted adbapi wrapped connections aren't context managers.
---
 synapse/storage/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index c09228c37f..87189e54b8 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -583,7 +583,7 @@ def prepare_database(db_conn):
     """Prepares a database for usage. Will either create all necessary tables
     or upgrade from an older schema version.
     """
-    with db_conn:
+    try:
         cur = db_conn.cursor()
         version_info = _get_or_create_schema_state(cur)
 
@@ -596,6 +596,10 @@ def prepare_database(db_conn):
         cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,))
 
         cur.close()
+        db_conn.commit()
+    except:
+        db_conn.rollback()
+        raise
 
 
 def _setup_new_database(cur):

From f701197227998e9fe270034be3c053cfa1f12ccf Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 14:20:14 +0000
Subject: [PATCH 11/14] Add example directory structures in doc

---
 synapse/storage/__init__.py | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 87189e54b8..8b2f8f8963 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -608,10 +608,27 @@ def _setup_new_database(cur):
 
     The "full_schemas" directory has subdirectories named after versions. This
     function searches for the highest version less than or equal to
-    `SCHEMA_VERSION` and excutes all .sql files in that directory.
+    `SCHEMA_VERSION` and executes all .sql files in that directory.
 
     The function will then apply all deltas for all versions after the base
     version.
+
+    Example directory structure:
+
+        schema/
+            delta/
+                ...
+            full_schemas/
+                3/
+                    test.sql
+                    ...
+                11/
+                    foo.sql
+                    bar.sql
+                ...
+
+    In the example foo.sql and bar.sql would be run, and then any delta files
+    for versions strictly greater than 11.
     """
     current_dir = os.path.join(dir_path, "schema", "full_schemas")
     directory_entries = os.listdir(current_dir)
@@ -675,6 +692,24 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
 
     This is a no-op of current_version == SCHEMA_VERSION.
 
+    Example directory structure:
+
+        schema/
+            delta/
+                11/
+                    foo.sql
+                    ...
+                12/
+                    foo.sql
+                    bar.py
+                ...
+            full_schemas/
+                ...
+
+    In the example, if current_version is 11, then foo.sql will be run if and
+    only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in
+    some arbitrary order.
+
     Args:
         cur (Cursor)
         current_version (int): The current version of the schema

From 5681264faa1457ac67a66ad2474ef644f0efeb55 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 14:21:53 +0000
Subject: [PATCH 12/14] s/%r/%s/

---
 synapse/storage/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 8b2f8f8963..98b877bd08 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -659,7 +659,7 @@ def _setup_new_database(cur):
     sql_script = "BEGIN TRANSACTION;\n"
     for filename in fnmatch.filter(directory_entries, "*.sql"):
         sql_loc = os.path.join(sql_dir, filename)
-        logger.debug("Applying schema %r", sql_loc)
+        logger.debug("Applying schema %s", sql_loc)
         sql_script += read_schema(sql_loc)
         sql_script += "\n"
     sql_script += "COMMIT TRANSACTION;"

From 17d319a20df6e92ddbb8ca5b9e08615d2975b466 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 15:06:22 +0000
Subject: [PATCH 13/14] s/schema_deltas/applied_schema_deltas/

---
 synapse/storage/__init__.py               | 16 +++++++++-------
 synapse/storage/schema/schema_version.sql |  4 ++--
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 98b877bd08..e2d5e5a41d 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -674,12 +674,13 @@ def _setup_new_database(cur):
     _upgrade_existing_database(
         cur,
         current_version=max_current_ver,
-        delta_files=[],
+        applied_delta_files=[],
         upgraded=False
     )
 
 
-def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
+def _upgrade_existing_database(cur, current_version, applied_delta_files,
+                               upgraded):
     """Upgrades an existing database.
 
     Delta files can either be SQL stored in *.sql files, or python modules
@@ -712,8 +713,9 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
 
     Args:
         cur (Cursor)
-        current_version (int): The current version of the schema
-        delta_files (list): A list of deltas that have already been applied
+        current_version (int): The current version of the schema.
+        applied_delta_files (list): A list of deltas that have already been
+            applied.
         upgraded (bool): Whether the current version was generated by having
             applied deltas or from full schema file. If `True` the function
             will never apply delta files for the given `current_version`, since
@@ -746,7 +748,7 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
         directory_entries.sort()
         for file_name in directory_entries:
             relative_path = os.path.join(str(v), file_name)
-            if relative_path in delta_files:
+            if relative_path in applied_delta_files:
                 continue
 
             absolute_path = os.path.join(
@@ -781,7 +783,7 @@ def _upgrade_existing_database(cur, current_version, delta_files, upgraded):
 
             # Mark as done.
             cur.execute(
-                "INSERT INTO schema_deltas (version, file)"
+                "INSERT INTO applied_schema_deltas (version, file)"
                 " VALUES (?,?)",
                 (v, relative_path)
             )
@@ -807,7 +809,7 @@ def _get_or_create_schema_state(txn):
 
     if current_version:
         txn.execute(
-            "SELECT file FROM schema_deltas WHERE version >= ?",
+            "SELECT file FROM applied_schema_deltas WHERE version >= ?",
             (current_version,)
         )
         return current_version, txn.fetchall(), upgraded
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql
index 20b1481ba5..0431e2d051 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/schema_version.sql
@@ -21,10 +21,10 @@ CREATE TABLE IF NOT EXISTS schema_version(
     CONSTRAINT schema_version_lock_uniq UNIQUE (Lock)
 );
 
-CREATE TABLE IF NOT EXISTS schema_deltas(
+CREATE TABLE IF NOT EXISTS applied_schema_deltas(
     version INTEGER NOT NULL,
     file TEXT NOT NULL,
     CONSTRAINT schema_deltas_ver_file UNIQUE (version, file) ON CONFLICT IGNORE
 );
 
-CREATE INDEX IF NOT EXISTS schema_deltas_ver ON schema_deltas(version);
+CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version);

From d56c01fff46d9adbccb8149d61f318f11040775e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 4 Mar 2015 15:10:05 +0000
Subject: [PATCH 14/14] Note that we don't specify execution order

---
 synapse/storage/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index e2d5e5a41d..a3ff995695 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -691,6 +691,10 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files,
     even if there has been no version bump. This is useful for development
     where orthogonal schema changes may happen on separate branches.
 
+    Different delta files for the same version *must* be orthogonal and give
+    the same result when applied in any order. No guarantees are made on the
+    order of execution of these scripts.
+
     This is a no-op of current_version == SCHEMA_VERSION.
 
     Example directory structure: