OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity
MatrixSynapse/scripts-dev/release.py

593 lines
19 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2020 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""An interactive script for doing a release. See `cli()` below.
"""
import glob
import os
import re
import subprocess
import sys
import urllib.request
from os import path
from tempfile import TemporaryDirectory
from typing import List, Optional, Tuple
import attr
import click
import commonmark
import git
import redbaron
from click.exceptions import ClickException
from github import Github
from packaging import version
def run_until_successful(command, *args, **kwargs):
while True:
completed_process = subprocess.run(command, *args, **kwargs)
exit_code = completed_process.returncode
if exit_code == 0:
# successful, so nothing more to do here.
return completed_process
print(f"The command {command!r} failed with exit code {exit_code}.")
print("Please try to correct the failure and then re-run.")
click.confirm("Try again?", abort=True)
@click.group()
def cli():
"""An interactive script to walk through the parts of creating a release.
Requires the dev dependencies be installed, which can be done via:
pip install -e .[dev]
Then to use:
./scripts-dev/release.py prepare
# ... ask others to look at the changelog ...
./scripts-dev/release.py tag
# ... wait for assets to build ...
./scripts-dev/release.py publish
./scripts-dev/release.py upload
# Optional: generate some nice links for the announcement
./scripts-dev/release.py upload
If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the
`tag`/`publish` command, then a new draft release will be created/published.
"""
@cli.command()
def prepare():
"""Do the initial stages of creating a release, including creating release
branch, updating changelog and pushing to GitHub.
"""
# Make sure we're in a git repo.
try:
repo = git.Repo()
except git.InvalidGitRepositoryError:
raise click.ClickException("Not in Synapse repo.")
if repo.is_dirty():
raise click.ClickException("Uncommitted changes exist.")
click.secho("Updating git repo...")
repo.remote().fetch()
# Get the current version and AST from root Synapse module.
current_version, parsed_synapse_ast, version_node = parse_version_from_module()
# Figure out what sort of release we're doing and calcuate the new version.
rc = click.confirm("RC", default=True)
if current_version.pre:
# If the current version is an RC we don't need to bump any of the
# version numbers (other than the RC number).
if rc:
new_version = "{}.{}.{}rc{}".format(
current_version.major,
current_version.minor,
current_version.micro,
current_version.pre[1] + 1,
)
else:
new_version = "{}.{}.{}".format(
current_version.major,
current_version.minor,
current_version.micro,
)
else:
# If this is a new release cycle then we need to know if it's a minor
# or a patch version bump.
release_type = click.prompt(
"Release type",
type=click.Choice(("minor", "patch")),
show_choices=True,
default="minor",
)
if release_type == "minor":
if rc:
new_version = "{}.{}.{}rc1".format(
current_version.major,
current_version.minor + 1,
0,
)
else:
new_version = "{}.{}.{}".format(
current_version.major,
current_version.minor + 1,
0,
)
else:
if rc:
new_version = "{}.{}.{}rc1".format(
current_version.major,
current_version.minor,
current_version.micro + 1,
)
else:
new_version = "{}.{}.{}".format(
current_version.major,
current_version.minor,
current_version.micro + 1,
)
# Confirm the calculated version is OK.
if not click.confirm(f"Create new version: {new_version}?", default=True):
click.get_current_context().abort()
# Switch to the release branch.
parsed_new_version = version.parse(new_version)
# We assume for debian changelogs that we only do RCs or full releases.
assert not parsed_new_version.is_devrelease
assert not parsed_new_version.is_postrelease
release_branch_name = (
f"release-v{parsed_new_version.major}.{parsed_new_version.minor}"
)
release_branch = find_ref(repo, release_branch_name)
if release_branch:
if release_branch.is_remote():
# If the release branch only exists on the remote we check it out
# locally.
repo.git.checkout(release_branch_name)
release_branch = repo.active_branch
else:
# If a branch doesn't exist we create one. We ask which one branch it
# should be based off, defaulting to sensible values depending on the
# release type.
if current_version.is_prerelease:
default = release_branch_name
elif release_type == "minor":
default = "develop"
else:
default = "master"
branch_name = click.prompt(
"Which branch should the release be based on?", default=default
)
base_branch = find_ref(repo, branch_name)
if not base_branch:
print(f"Could not find base branch {branch_name}!")
click.get_current_context().abort()
# Check out the base branch and ensure it's up to date
repo.head.reference = base_branch
repo.head.reset(index=True, working_tree=True)
if not base_branch.is_remote():
update_branch(repo)
# Create the new release branch
release_branch = repo.create_head(release_branch_name, commit=base_branch)
# Switch to the release branch and ensure its up to date.
repo.git.checkout(release_branch_name)
update_branch(repo)
# Update the `__version__` variable and write it back to the file.
version_node.value = '"' + new_version + '"'
with open("synapse/__init__.py", "w") as f:
f.write(parsed_synapse_ast.dumps())
# Generate changelogs.
generate_and_write_changelog(current_version)
# Generate debian changelogs
if parsed_new_version.pre is not None:
# If this is an RC then we need to coerce the version string to match
# Debian norms, e.g. 1.39.0rc2 gets converted to 1.39.0~rc2.
base_ver = parsed_new_version.base_version
pre_type, pre_num = parsed_new_version.pre
debian_version = f"{base_ver}~{pre_type}{pre_num}"
else:
debian_version = new_version
run_until_successful(
f'dch -M -v {debian_version} "New synapse release {debian_version}."',
shell=True,
)
run_until_successful('dch -M -r -D stable ""', shell=True)
# Show the user the changes and ask if they want to edit the change log.
repo.git.add("-u")
subprocess.run("git diff --cached", shell=True)
if click.confirm("Edit changelog?", default=False):
click.edit(filename="CHANGES.md")
# Commit the changes.
repo.git.add("-u")
repo.git.commit("-m", new_version)
# We give the option to bail here in case the user wants to make sure things
# are OK before pushing.
if not click.confirm("Push branch to github?", default=True):
print("")
print("Run when ready to push:")
print("")
print(f"\tgit push -u {repo.remote().name} {repo.active_branch.name}")
print("")
sys.exit(0)
# Otherwise, push and open the changelog in the browser.
repo.git.push("-u", repo.remote().name, repo.active_branch.name)
print("Opening the changelog in your browser...")
print("Please ask others to give it a check.")
click.launch(
f"https://github.com/matrix-org/synapse/blob/{repo.active_branch.name}/CHANGES.md"
)
@cli.command()
@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"])
def tag(gh_token: Optional[str]):
"""Tags the release and generates a draft GitHub release"""
# Make sure we're in a git repo.
try:
repo = git.Repo()
except git.InvalidGitRepositoryError:
raise click.ClickException("Not in Synapse repo.")
if repo.is_dirty():
raise click.ClickException("Uncommitted changes exist.")
click.secho("Updating git repo...")
repo.remote().fetch()
# Find out the version and tag name.
current_version, _, _ = parse_version_from_module()
tag_name = f"v{current_version}"
# Check we haven't released this version.
if tag_name in repo.tags:
raise click.ClickException(f"Tag {tag_name} already exists!\n")
# Get the appropriate changelogs and tag.
changes = get_changes_for_version(current_version)
click.echo_via_pager(changes)
if click.confirm("Edit text?", default=False):
changes = click.edit(changes, require_save=False)
repo.create_tag(tag_name, message=changes, sign=True)
if not click.confirm("Push tag to GitHub?", default=True):
print("")
print("Run when ready to push:")
print("")
print(f"\tgit push {repo.remote().name} tag {current_version}")
print("")
return
repo.git.push(repo.remote().name, "tag", tag_name)
# If no token was given, we bail here
if not gh_token:
print("Launching the GitHub release page in your browser.")
print("Please correct the title and create a draft.")
if current_version.is_prerelease:
print("As this is an RC, remember to mark it as a pre-release!")
print("(by the way, this step can be automated by passing --gh-token,")
print("or one of the GH_TOKEN or GITHUB_TOKEN env vars.)")
click.launch(f"https://github.com/matrix-org/synapse/releases/edit/{tag_name}")
print("Once done, you need to wait for the release assets to build.")
if click.confirm("Launch the release assets actions page?", default=True):
click.launch(
f"https://github.com/matrix-org/synapse/actions?query=branch%3A{tag_name}"
)
return
# Create a new draft release
gh = Github(gh_token)
gh_repo = gh.get_repo("matrix-org/synapse")
release = gh_repo.create_git_release(
tag=tag_name,
name=tag_name,
message=changes,
draft=True,
prerelease=current_version.is_prerelease,
)
# Open the release and the actions where we are building the assets.
print("Launching the release page and the actions page.")
click.launch(release.html_url)
click.launch(
f"https://github.com/matrix-org/synapse/actions?query=branch%3A{tag_name}"
)
click.echo("Wait for release assets to be built")
@cli.command()
@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True)
def publish(gh_token: str):
"""Publish release."""
# Make sure we're in a git repo.
try:
repo = git.Repo()
except git.InvalidGitRepositoryError:
raise click.ClickException("Not in Synapse repo.")
if repo.is_dirty():
raise click.ClickException("Uncommitted changes exist.")
current_version, _, _ = parse_version_from_module()
tag_name = f"v{current_version}"
if not click.confirm(f"Publish {tag_name}?", default=True):
return
# Publish the draft release
gh = Github(gh_token)
gh_repo = gh.get_repo("matrix-org/synapse")
for release in gh_repo.get_releases():
if release.title == tag_name:
break
else:
raise ClickException(f"Failed to find GitHub release for {tag_name}")
assert release.title == tag_name
if not release.draft:
click.echo("Release already published.")
return
release = release.update_release(
name=release.title,
message=release.body,
tag_name=release.tag_name,
prerelease=release.prerelease,
draft=False,
)
@cli.command()
def upload():
"""Upload release to pypi."""
current_version, _, _ = parse_version_from_module()
tag_name = f"v{current_version}"
pypi_asset_names = [
f"matrix_synapse-{current_version}-py3-none-any.whl",
f"matrix-synapse-{current_version}.tar.gz",
]
with TemporaryDirectory(prefix=f"synapse_upload_{tag_name}_") as tmpdir:
for name in pypi_asset_names:
filename = path.join(tmpdir, name)
url = f"https://github.com/matrix-org/synapse/releases/download/{tag_name}/{name}"
click.echo(f"Downloading {name} into {filename}")
urllib.request.urlretrieve(url, filename=filename)
if click.confirm("Upload to PyPI?", default=True):
subprocess.run("twine upload *", shell=True, cwd=tmpdir)
click.echo(
f"Done! Remember to merge the tag {tag_name} into the appropriate branches"
)
@cli.command()
def announce():
"""Generate markdown to announce the release."""
current_version, _, _ = parse_version_from_module()
tag_name = f"v{current_version}"
click.echo(
f"""
Hi everyone. Synapse {current_version} has just been released.
[notes](https://github.com/matrix-org/synapse/releases/tag/{tag_name}) |\
[docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \
[debs](https://packages.matrix.org/debian/) | \
[pypi](https://pypi.org/project/matrix-synapse/{current_version}/)"""
)
if "rc" in tag_name:
click.echo(
"""
Announce the RC in
- #homeowners:matrix.org (Synapse Announcements)
- #synapse-dev:matrix.org"""
)
else:
click.echo(
"""
Announce the release in
- #homeowners:matrix.org (Synapse Announcements), bumping the version in the topic
- #synapse:matrix.org (Synapse Admins), bumping the version in the topic
- #synapse-dev:matrix.org
- #synapse-package-maintainers:matrix.org"""
)
def parse_version_from_module() -> Tuple[
version.Version, redbaron.RedBaron, redbaron.Node
]:
# Parse the AST and load the `__version__` node so that we can edit it
# later.
with open("synapse/__init__.py") as f:
red = redbaron.RedBaron(f.read())
version_node = None
for node in red:
if node.type != "assignment":
continue
if node.target.type != "name":
continue
if node.target.value != "__version__":
continue
version_node = node
break
if not version_node:
print("Failed to find '__version__' definition in synapse/__init__.py")
sys.exit(1)
# Parse the current version.
current_version = version.parse(version_node.value.value.strip('"'))
assert isinstance(current_version, version.Version)
return current_version, red, version_node
def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
"""Find the branch/ref, looking first locally then in the remote."""
if ref_name in repo.refs:
return repo.refs[ref_name]
elif ref_name in repo.remote().refs:
return repo.remote().refs[ref_name]
else:
return None
def update_branch(repo: git.Repo):
"""Ensure branch is up to date if it has a remote"""
if repo.active_branch.tracking_branch():
repo.git.merge(repo.active_branch.tracking_branch().name)
def get_changes_for_version(wanted_version: version.Version) -> str:
"""Get the changelogs for the given version.
If an RC then will only get the changelog for that RC version, otherwise if
its a full release will get the changelog for the release and all its RCs.
"""
with open("CHANGES.md") as f:
changes = f.read()
# First we parse the changelog so that we can split it into sections based
# on the release headings.
ast = commonmark.Parser().parse(changes)
@attr.s(auto_attribs=True)
class VersionSection:
title: str
# These are 0-based.
start_line: int
end_line: Optional[int] = None # Is none if its the last entry
headings: List[VersionSection] = []
for node, _ in ast.walker():
# We look for all text nodes that are in a level 1 heading.
if node.t != "text":
continue
if node.parent.t != "heading" or node.parent.level != 1:
continue
# If we have a previous heading then we update its `end_line`.
if headings:
headings[-1].end_line = node.parent.sourcepos[0][0] - 1
headings.append(VersionSection(node.literal, node.parent.sourcepos[0][0] - 1))
changes_by_line = changes.split("\n")
version_changelog = [] # The lines we want to include in the changelog
# Go through each section and find any that match the requested version.
regex = re.compile(r"^Synapse v?(\S+)")
for section in headings:
groups = regex.match(section.title)
if not groups:
continue
heading_version = version.parse(groups.group(1))
heading_base_version = version.parse(heading_version.base_version)
# Check if heading version matches the requested version, or if its an
# RC of the requested version.
if wanted_version not in (heading_version, heading_base_version):
continue
version_changelog.extend(changes_by_line[section.start_line : section.end_line])
return "\n".join(version_changelog)
def generate_and_write_changelog(current_version: version.Version):
# We do this by getting a draft so that we can edit it before writing to the
# changelog.
result = run_until_successful(
"python3 -m towncrier --draft", shell=True, capture_output=True
)
new_changes = result.stdout.decode("utf-8")
new_changes = new_changes.replace(
"No significant changes.", f"No significant changes since {current_version}."
)
# Prepend changes to changelog
with open("CHANGES.md", "r+") as f:
existing_content = f.read()
f.seek(0, 0)
f.write(new_changes)
f.write("\n")
f.write(existing_content)
# Remove all the news fragments
for f in glob.iglob("changelog.d/*.*"):
os.remove(f)
if __name__ == "__main__":
cli()
Reference in New Issue View Git Blame Copy Permalink