Document which versions of Synapse have compatible schema versions. (#16661)

pull/16703/head
Patrick Cloke 2023-11-28 11:01:24 -05:00 committed by GitHub
parent b0ed14d815
commit 77882b6a7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 216 additions and 1 deletions

View File

@ -6,6 +6,7 @@ on:
- docs/** - docs/**
- book.toml - book.toml
- .github/workflows/docs-pr.yaml - .github/workflows/docs-pr.yaml
- scripts-dev/schema_versions.py
jobs: jobs:
pages: pages:
@ -13,12 +14,22 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with:
# Fetch all history so that the schema_versions script works.
fetch-depth: 0
- name: Setup mdbook - name: Setup mdbook
uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
with: with:
mdbook-version: '0.4.17' mdbook-version: '0.4.17'
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'"
- name: Build the documentation - name: Build the documentation
# mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
# However, we're using docs/README.md for other purposes and need to pick a new page # However, we're using docs/README.md for other purposes and need to pick a new page

View File

@ -51,12 +51,22 @@ jobs:
- pre - pre
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with:
# Fetch all history so that the schema_versions script works.
fetch-depth: 0
- name: Setup mdbook - name: Setup mdbook
uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0 uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
with: with:
mdbook-version: '0.4.17' mdbook-version: '0.4.17'
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'"
- name: Build the documentation - name: Build the documentation
# mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md. # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
# However, we're using docs/README.md for other purposes and need to pick a new page # However, we're using docs/README.md for other purposes and need to pick a new page

View File

@ -37,3 +37,6 @@ additional-css = [
] ]
additional-js = ["docs/website_files/table-of-contents.js"] additional-js = ["docs/website_files/table-of-contents.js"]
theme = "docs/website_files/theme" theme = "docs/website_files/theme"
[preprocessor.schema_versions]
command = "./scripts-dev/schema_versions.py"

1
changelog.d/16661.doc Normal file
View File

@ -0,0 +1 @@
Add schema rollback information to documentation.

View File

@ -88,6 +88,15 @@ process, for example:
dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
``` ```
Generally Synapse database schemas are compatible across multiple versions, once
a version of Synapse is deployed you may not be able to rollback automatically.
The following table gives the version ranges and the earliest version they can
be rolled back to. E.g. Synapse versions v1.58.0 through v1.61.1 can be rolled
back safely to v1.57.0, but starting with v1.62.0 it is only safe to rollback to
v1.61.0.
<!-- REPLACE_WITH_SCHEMA_VERSIONS -->
# Upgrading to v1.93.0 # Upgrading to v1.93.0
## Minimum supported Rust version ## Minimum supported Rust version

181
scripts-dev/schema_versions.py Executable file
View File

@ -0,0 +1,181 @@
#!/usr/bin/env python
# Copyright 2023 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A script to calculate which versions of Synapse have backwards-compatible
database schemas. It creates a Markdown table of Synapse versions and the earliest
compatible version.
It is compatible with the mdbook protocol for preprocessors (see
https://rust-lang.github.io/mdBook/for_developers/preprocessors.html#implementing-a-preprocessor-with-a-different-language):
Exit 0 to denote support for all renderers:
./scripts-dev/schema_versions.py supports <mdbook renderer>
Parse a JSON list from stdin and add the table to the proper documetnation page:
./scripts-dev/schema_versions.py
Additionally, the script supports dumping the table to stdout for debugging:
./scripts-dev/schema_versions.py dump
"""
import io
import json
import sys
from collections import defaultdict
from typing import Any, Dict, Iterator, Optional, Tuple
import git
from packaging import version
# The schema version has moved around over the years.
SCHEMA_VERSION_FILES = (
"synapse/storage/schema/__init__.py",
"synapse/storage/prepare_database.py",
"synapse/storage/__init__.py",
"synapse/app/homeserver.py",
)
# Skip versions of Synapse < v1.0, they're old and essentially not
# compatible with today's federation.
OLDEST_SHOWN_VERSION = version.parse("v1.0")
def get_schema_versions(tag: git.Tag) -> Tuple[Optional[int], Optional[int]]:
"""Get the schema and schema compat versions for a tag."""
schema_version = None
schema_compat_version = None
for file in SCHEMA_VERSION_FILES:
try:
schema_file = tag.commit.tree / file
except KeyError:
continue
# We (usually) can't execute the code since it might have unknown imports.
if file != "synapse/storage/schema/__init__.py":
with io.BytesIO(schema_file.data_stream.read()) as f:
for line in f.readlines():
if line.startswith(b"SCHEMA_VERSION"):
schema_version = int(line.split()[2])
# Bail early.
break
else:
# SCHEMA_COMPAT_VERSION is sometimes across multiple lines, the easist
# thing to do is exec the code. Luckily it has only ever existed in
# a file which imports nothing else from Synapse.
locals: Dict[str, Any] = {}
exec(schema_file.data_stream.read().decode("utf-8"), {}, locals)
schema_version = locals["SCHEMA_VERSION"]
schema_compat_version = locals.get("SCHEMA_COMPAT_VERSION")
return schema_version, schema_compat_version
def get_tags(repo: git.Repo) -> Iterator[git.Tag]:
"""Return an iterator of tags sorted by version."""
tags = []
for tag in repo.tags:
# All "real" Synapse tags are of the form vX.Y.Z.
if not tag.name.startswith("v"):
continue
# There's a weird tag from the initial react UI.
if tag.name == "v0.1":
continue
try:
tag_version = version.parse(tag.name)
except version.InvalidVersion:
# Skip invalid versions.
continue
# Skip pre- and post-release versions.
if tag_version.is_prerelease or tag_version.is_postrelease or tag_version.local:
continue
# Skip old versions.
if tag_version < OLDEST_SHOWN_VERSION:
continue
tags.append((tag_version, tag))
# Sort based on the version number (not lexically).
return (tag for _, tag in sorted(tags, key=lambda t: t[0]))
def calculate_version_chart() -> str:
repo = git.Repo(path=".")
# Map of schema version -> Synapse versions which are at that schema version.
schema_versions = defaultdict(list)
# Map of schema version -> Synapse versions which are compatible with that
# schema version.
schema_compat_versions = defaultdict(list)
# Find ranges of versions which are compatible with a schema version.
#
# There are two modes of operation:
#
# 1. Pre-schema_compat_version (i.e. schema_compat_version of None), then
# Synapse is compatible up/downgrading to a version with
# schema_version >= its current version.
#
# 2. Post-schema_compat_version (i.e. schema_compat_version is *not* None),
# then Synapse is compatible up/downgrading to a version with
# schema version >= schema_compat_version.
#
# This is more generous and avoids versions that cannot be rolled back.
#
# See https://github.com/matrix-org/synapse/pull/9933 which was included in v1.37.0.
for tag in get_tags(repo):
schema_version, schema_compat_version = get_schema_versions(tag)
# If a schema compat version is given, prefer that over the schema version.
schema_versions[schema_version].append(tag.name)
schema_compat_versions[schema_compat_version or schema_version].append(tag.name)
# Generate a table which maps the latest Synapse version compatible with each
# schema version.
result = f"| {'Versions': ^19} | Compatible version |\n"
result += f"|{'-' * (19 + 2)}|{'-' * (18 + 2)}|\n"
for schema_version, synapse_versions in schema_compat_versions.items():
result += f"| {synapse_versions[0] + ' ' + synapse_versions[-1]: ^19} | {schema_versions[schema_version][0]: ^18} |\n"
return result
if __name__ == "__main__":
if len(sys.argv) == 3 and sys.argv[1] == "supports":
# We don't care about the renderer which is being used, which is the second argument.
sys.exit(0)
elif len(sys.argv) == 2 and sys.argv[1] == "dump":
print(calculate_version_chart())
else:
# Expect JSON data on stdin.
context, book = json.load(sys.stdin)
for section in book["sections"]:
if "Chapter" in section and section["Chapter"]["path"] == "upgrade.md":
section["Chapter"]["content"] = section["Chapter"]["content"].replace(
"<!-- REPLACE_WITH_SCHEMA_VERSIONS -->", calculate_version_chart()
)
# Print the result back out to stdout.
print(json.dumps(book))