Add sqlite schema

synapse/storage/schema/delta/24/fts.py

@@ -15,7 +15,9 @@
 import logging
 
 from synapse.storage import get_statements
-from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines import PostgresEngine, Sqlite3Engine
+
+import ujson
 
 logger = logging.getLogger(__name__)
 
@@ -46,13 +48,70 @@ INSERT INTO event_search SELECT
 
 CREATE INDEX event_search_fts_idx ON event_search USING gin(vector);
 CREATE INDEX event_search_ev_idx ON event_search(event_id);
+CREATE INDEX event_search_ev_ridx ON event_search(room_id);
 """
 
 
+SQLITE_TABLE = (
+    "CREATE VIRTUAL TABLE event_search USING fts3 ( event_id, room_id, key, value)"
+)
+SQLITE_INDEX = "CREATE INDEX event_search_ev_idx ON event_search(event_id)"
+
+
 def run_upgrade(cur, database_engine, *args, **kwargs):
-    if not isinstance(database_engine, PostgresEngine):
-        # We only support FTS for postgres currently.
+    if isinstance(database_engine, PostgresEngine):
+        for statement in get_statements(POSTGRES_SQL.splitlines()):
+            cur.execute(statement)
         return
 
-    for statement in get_statements(POSTGRES_SQL.splitlines()):
-        cur.execute(statement)
+    if isinstance(database_engine, Sqlite3Engine):
+        cur.execute(SQLITE_TABLE)
+
+        rowid = -1
+        while True:
+            cur.execute(
+                "SELECT rowid, json FROM event_json"
+                " WHERE rowid > ?"
+                " ORDER BY rowid ASC LIMIT 100",
+                (rowid,)
+            )
+
+            res = cur.fetchall()
+
+            if not res:
+                break
+
+            events = [
+                ujson.loads(js)
+                for _, js in res
+            ]
+
+            rowid = max(rid for rid, _ in res)
+
+            rows = []
+            for ev in events:
+                if ev["type"] == "m.room.message":
+                    rows.append((
+                        ev["event_id"], ev["room_id"], "content.body",
+                        ev["content"]["body"]
+                    ))
+                if ev["type"] == "m.room.name":
+                    rows.append((
+                        ev["event_id"], ev["room_id"], "content.name",
+                        ev["content"]["name"]
+                    ))
+                if ev["type"] == "m.room.topic":
+                    rows.append((
+                        ev["event_id"], ev["room_id"], "content.topic",
+                        ev["content"]["topic"]
+                    ))
+
+            if rows:
+                logger.info(rows)
+                cur.executemany(
+                    "INSERT INTO event_search (event_id, room_id, key, value)"
+                    " VALUES (?,?,?,?)",
+                    rows
+                )
+
+        # cur.execute(SQLITE_INDEX)