riot-web/scripts/deploy.py

#!/usr/bin/env python3
#
# download and unpack a element-web tarball.
#
# Allows `bundles` to be extracted to a common directory, and a link to
# config.json to be added.

import argparse
import errno
import os
import os.path
import subprocess
import sys
import tarfile
import shutil
import glob
from urllib.request import urlretrieve


class DeployException(Exception):
    pass


def create_relative_symlink(linkname, target):
    relpath = os.path.relpath(target, os.path.dirname(linkname))
    print("Symlink %s -> %s" % (linkname, relpath))

    try:
        os.symlink(relpath, linkname)
    except OSError as e:
        if e.errno == errno.EEXIST:
            # atomic modification
            os.symlink(relpath, linkname + ".tmp")
            os.rename(linkname + ".tmp", linkname)
        else:
            raise e


def move_bundles(source, dest):
    """Move the contents of the 'bundles' directory to a common dir

    We check that we will not be overwriting anything before we proceed.

    Args:
        source (str): path to 'bundles' within the extracted tarball
        dest (str): target common directory
    """

    if not os.path.isdir(dest):
        os.mkdir(dest)

    # build a map from source to destination, checking for non-existence as we go.
    renames = {}
    for f in os.listdir(source):
        dst = os.path.join(dest, f)
        if os.path.exists(dst):
            print(
                "Skipping bundle. The bundle includes '%s' which we have previously deployed."
                % f
            )
        else:
            renames[os.path.join(source, f)] = dst

    for (src, dst) in renames.items():
        print("Move %s -> %s" % (src, dst))
        os.rename(src, dst)


class Deployer:
    def __init__(self):
        self.packages_path = "."
        self.bundles_path = None
        self.should_clean = False
        self.symlink_latest = None
        # filename -> symlink path e.g 'config.localhost.json' => '../localhost/config.json'
        self.symlink_paths = {}
        self.verify_signature = True

    def fetch(self, tarball, extract_path):
        """Download a tarball, verifies it if needed, and unpacks it

        Returns:
            (str) the path to the unpacked directory
        """
        print("Fetching %s to %s" % (tarball, extract_path))

        name_str = os.path.basename(tarball).replace(".tar.gz", "")
        extracted_dir = os.path.join(extract_path, name_str)
        if os.path.exists(extracted_dir):
            raise DeployException('Cannot unpack %s: %s already exists' % (
                tarball, extracted_dir))

        downloaded = False
        if tarball.startswith("http://") or tarball.startswith("https://"):
            tarball = self.download_and_verify(tarball)
            print("Downloaded file: %s" % tarball)
            downloaded = True

        try:
            with tarfile.open(tarball) as tar:
                tar.extractall(extract_path)
        finally:
            if self.should_clean and downloaded:
                os.remove(tarball)

        print("Extracted into: %s" % extracted_dir)
        return extracted_dir

    def deploy(self, extracted_dir):
        """Applies symlinks and handles the bundles directory on an extracted tarball"""
        print("Deploying %s" % extracted_dir)

        if self.symlink_paths:
            for link_path, file_path in self.symlink_paths.items():
                create_relative_symlink(
                    target=file_path,
                    linkname=os.path.join(extracted_dir, link_path)
                )

        if self.bundles_path:
            extracted_bundles = os.path.join(extracted_dir, 'bundles')
            move_bundles(source=extracted_bundles, dest=self.bundles_path)

            # replace the extracted_bundles dir (which may not be empty if some
            # bundles were skipped) with a symlink to the common dir.
            shutil.rmtree(extracted_bundles)
            create_relative_symlink(
                target=self.bundles_path,
                linkname=extracted_bundles,
            )

        if self.symlink_latest:
            create_relative_symlink(
                target=extracted_dir,
                linkname=self.symlink_latest,
            )

    def download_and_verify(self, url):
        tarball = self.download_file(url)

        if self.verify_signature:
            sigfile = self.download_file(url + ".asc")
            subprocess.check_call(["gpg", "--verify", sigfile, tarball])

        return tarball

    def download_file(self, url):
        if not os.path.isdir(self.packages_path):
            os.mkdir(self.packages_path)
        local_filename = os.path.join(self.packages_path,
                                      url.split('/')[-1])
        sys.stdout.write("Downloading %s -> %s..." % (url, local_filename))
        sys.stdout.flush()
        urlretrieve(url, local_filename)
        print ("Done")
        return local_filename


if __name__ == "__main__":
    parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
    parser.add_argument(
        "-p", "--packages-dir", default="./packages", help=(
            "The directory to download the tarball into. (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-e", "--extract-path", default="./deploys", help=(
            "The location to extract .tar.gz files to. (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-b", "--bundles-dir", nargs='?', default="./bundles", help=(
            "A directory to move the contents of the 'bundles' directory to. A \
            symlink to the bundles directory will also be written inside the \
            extracted tarball. Example: './bundles'. \
            (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-c", "--clean", action="store_true", default=False, help=(
            "Remove .tar.gz files after they have been downloaded and extracted. \
            (Default: %(default)s)"
        )
    )
    parser.add_argument(
        "--include", nargs='*', default=['./config*.json'], help=(
            "Symlink these files into the root of the deployed tarball. \
             Useful for config files and home pages. Supports glob syntax. \
             (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-s", "--symlink", dest="symlink", default="./latest", help=(
            "Write a symlink to this location pointing to the extracted tarball. \
            New builds will keep overwriting this symlink. The symlink will point \
            to the webapp directory INSIDE the tarball."
        )
    )
    parser.add_argument(
        "target", help=(
            "filename of extracted directory, tarball, or URL to download."
        ),
    )

    args = parser.parse_args()

    deployer = Deployer()
    deployer.packages_path = args.packages_dir
    deployer.bundles_path = args.bundles_dir
    deployer.should_clean = args.clean
    deployer.symlink_latest = args.symlink

    for include in args.include:
        deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })

    if os.path.isdir(args.target):
        # If the given directory contains a single directory then use that instead, the ci package wraps in an extra dir
        files = os.listdir(args.target)
        if len(files) == 1 and os.path.isdir(os.path.join(args.target, files[0])):
            extracted_dir = os.path.join(args.target, files[0])
        else:
            extracted_dir = args.target
    else:
        extracted_dir = deployer.fetch(args.target, args.extract_path)
    deployer.deploy(extracted_dir)