OpenCloud
/
element-web
mirror of https://github.com/vector-im/element-web
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #2974 from vector-im/rav/deploy_script 

Deploy script
pull/2960/merge
David Baker 2017-01-19 11:32:46 +00:00 committed by GitHub
parent 971314d75a d1fbbf90c0
commit 6c4bf58ba5
2 changed files with 213 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
scripts/deploy.py Executable file
View File

@ -0,0 +1,183 @@
#!/usr/bin/env python
#
# download and unpack a riot-web tarball.
#
# Allows `bundles` to be extracted to a common directory, and a link to
# config.json to be added.
from __future__ import print_function
import argparse
import os
import os.path
import subprocess
import sys
import tarfile
try:
# python3
from urllib.request import urlretrieve
except ImportError:
# python2
from urllib import urlretrieve
class DeployException(Exception):
pass
def create_relative_symlink(linkname, target):
relpath = os.path.relpath(target, os.path.dirname(linkname))
print ("Symlink %s -> %s" % (linkname, relpath))
os.symlink(relpath, linkname)
def move_bundles(source, dest):
"""Move the contents of the 'bundles' directory to a common dir
We check that we will not be overwriting anything before we proceed.
Args:
source (str): path to 'bundles' within the extracted tarball
dest (str): target common directory
"""
if not os.path.isdir(dest):
os.mkdir(dest)
# build a map from source to destination, checking for non-existence as we go.
renames = {}
for f in os.listdir(source):
dst = os.path.join(dest, f)
if os.path.exists(dst):
raise DeployException(
"Not deploying. The bundle includes '%s' which we have previously deployed."
% f
)
renames[os.path.join(source, f)] = dst
for (src, dst) in renames.iteritems():
print ("Move %s -> %s" % (src, dst))
os.rename(src, dst)
class Deployer:
def __init__(self):
self.packages_path = "."
self.bundles_path = None
self.should_clean = False
self.config_location = None
self.verify_signature = True
def deploy(self, tarball, extract_path):
"""Download a tarball if necessary, and unpack it
Returns:
(str) the path to the unpacked deployment
"""
print("Deploying %s to %s" % (tarball, extract_path))
name_str = os.path.basename(tarball).replace(".tar.gz", "")
extracted_dir = os.path.join(extract_path, name_str)
if os.path.exists(extracted_dir):
raise DeployException('Cannot unpack %s: %s already exists' % (
tarball, extracted_dir))
downloaded = False
if tarball.startswith("http://") or tarball.startswith("https://"):
tarball = self.download_and_verify(tarball)
print("Downloaded file: %s" % tarball)
downloaded = True
try:
with tarfile.open(tarball) as tar:
tar.extractall(extract_path)
finally:
if self.should_clean and downloaded:
os.remove(tarball)
print ("Extracted into: %s" % extracted_dir)
if self.config_location:
create_relative_symlink(
target=self.config_location,
linkname=os.path.join(extracted_dir, 'config.json')
)
if self.bundles_path:
extracted_bundles = os.path.join(extracted_dir, 'bundles')
move_bundles(source=extracted_bundles, dest=self.bundles_path)
# replace the (hopefully now empty) extracted_bundles dir with a
# symlink to the common dir.
os.rmdir(extracted_bundles)
create_relative_symlink(
target=self.bundles_path,
linkname=extracted_bundles,
)
return extracted_dir
def download_and_verify(self, url):
tarball = self.download_file(url)
if self.verify_signature:
sigfile = self.download_file(url + ".asc")
subprocess.check_call(["gpg", "--verify", sigfile, tarball])
return tarball
def download_file(self, url):
if not os.path.isdir(self.packages_path):
os.mkdir(self.packages_path)
local_filename = os.path.join(self.packages_path,
url.split('/')[-1])
sys.stdout.write("Downloading %s -> %s..." % (url, local_filename))
sys.stdout.flush()
urlretrieve(url, local_filename)
print ("Done")
return local_filename
if __name__ == "__main__":
parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
parser.add_argument(
"-p", "--packages-dir", default="./packages", help=(
"The directory to download the tarball into. (Default: '%(default)s')"
)
)
parser.add_argument(
"-e", "--extract-path", default="./deploys", help=(
"The location to extract .tar.gz files to. (Default: '%(default)s')"
)
)
parser.add_argument(
"-b", "--bundles-dir", nargs='?', default="./bundles", help=(
"A directory to move the contents of the 'bundles' directory to. A \
symlink to the bundles directory will also be written inside the \
extracted tarball. Example: './bundles'. \
(Default: '%(default)s')"
)
)
parser.add_argument(
"-c", "--clean", action="store_true", default=False, help=(
"Remove .tar.gz files after they have been downloaded and extracted. \
(Default: %(default)s)"
)
)
parser.add_argument(
"--config", nargs='?', default='./config.json', help=(
"Write a symlink at config.json in the extracted tarball to this \
location. (Default: '%(default)s')"
)
)
parser.add_argument(
"tarball", help=(
"filename of tarball, or URL to download."
),
)
args = parser.parse_args()
deployer = Deployer()
deployer.packages_path = args.packages_dir
deployer.bundles_path = args.bundles_dir
deployer.should_clean = args.clean
deployer.config_location = args.config
deployer.deploy(args.tarball, args.extract_path)

113
scripts/redeploy.py
View File

@ -14,32 +14,17 @@ from __future__ import print_function
import json, requests, tarfile, argparse, os, errno
import time
from urlparse import urljoin
from flask import Flask, jsonify, request, abort
from deploy import Deployer, DeployException
app = Flask(__name__)
arg_jenkins_url = None
deployer = None
arg_extract_path = None
arg_bundles_path = None
arg_should_clean = None
arg_symlink = None
arg_config_location = None
class DeployException(Exception):
pass
def download_file(url):
local_filename = url.split('/')[-1]
r = requests.get(url, stream=True)
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
return local_filename
def untar_to(tarball, dest):
with tarfile.open(tarball) as tar:
tar.extractall(dest)
def create_symlink(source, linkname):
try:
@ -137,17 +122,20 @@ def fetch_jenkins_build(job_name, build_num):
# see half-written files.
build_dir = os.path.join(arg_extract_path, "%s-#%s" % (job_name, build_num))
try:
deploy_tarball(tar_gz_url, build_dir)
extracted_dir = deploy_tarball(tar_gz_url, build_dir)
except DeployException as e:
abort(400, e.message)
create_symlink(source=extracted_dir, linkname=arg_symlink)
return jsonify({})
def deploy_tarball(tar_gz_url, build_dir):
"""Download a tarball from jenkins and deploy it as the new version
"""
print("Deploying %s to %s" % (tar_gz_url, build_dir))
"""Download a tarball from jenkins and unpack it
Returns:
(str) the path to the unpacked deployment
"""
if os.path.exists(build_dir):
raise DeployException(
"Not deploying. We have previously deployed this build."
@ -156,62 +144,8 @@ def deploy_tarball(tar_gz_url, build_dir):
# we rely on the fact that flask only serves one request at a time to
# ensure that we do not overwrite a tarball from a concurrent request.
filename = download_file(tar_gz_url)
print("Downloaded file: %s" % filename)
try:
untar_to(filename, build_dir)
print("Extracted to: %s" % build_dir)
finally:
if arg_should_clean:
os.remove(filename)
name_str = filename.replace(".tar.gz", "")
extracted_dir = os.path.join(build_dir, name_str)
if arg_config_location:
create_symlink(source=arg_config_location, linkname=os.path.join(extracted_dir, 'config.json'))
if arg_bundles_path:
extracted_bundles = os.path.join(extracted_dir, 'bundles')
move_bundles(source=extracted_bundles, dest=arg_bundles_path)
# replace the (hopefully now empty) extracted_bundles dir with a
# symlink to the common dir.
relpath = os.path.relpath(arg_bundles_path, extracted_dir)
os.rmdir(extracted_bundles)
print ("Symlink %s -> %s" % (extracted_bundles, relpath))
os.symlink(relpath, extracted_bundles)
create_symlink(source=extracted_dir, linkname=arg_symlink)
def move_bundles(source, dest):
"""Move the contents of the 'bundles' directory to a common dir
We check that we will not be overwriting anything before we proceed.
Args:
source (str): path to 'bundles' within the extracted tarball
dest (str): target common directory
"""
if not os.path.isdir(dest):
os.mkdir(dest)
# build a map from source to destination, checking for non-existence as we go.
renames = {}
for f in os.listdir(source):
dst = os.path.join(dest, f)
if os.path.exists(dst):
raise DeployException(
"Not deploying. The bundle includes '%s' which we have previously deployed."
% f
)
renames[os.path.join(source, f)] = dst
for (src, dst) in renames.iteritems():
print ("Move %s -> %s" % (src, dst))
os.rename(src, dst)
return deployer.deploy(tar_gz_url, build_dir)
if __name__ == "__main__":
@ -270,21 +204,34 @@ if __name__ == "__main__":
else:
arg_jenkins_url = args.jenkins + "/"
arg_extract_path = args.extract
arg_bundles_path = args.bundles_dir
arg_should_clean = args.clean
arg_symlink = args.symlink
arg_config_location = args.config
if not os.path.isdir(arg_extract_path):
os.mkdir(arg_extract_path)
deployer = Deployer()
deployer.bundles_path = args.bundles_dir
deployer.should_clean = args.clean
deployer.config_location = args.config
# we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to
# the jenkins server (and the jenkins server not being compromised and/or
# github not serving it compromised source). If that's not good enough for
# you, don't use riot.im/develop.
deployer.verify_signature = False
if args.tarball_uri is not None:
build_dir = os.path.join(arg_extract_path, "test-%i" % (time.time()))
deploy_tarball(args.tarball_uri, build_dir)
else:
print(
"Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config location: %s" %
(args.port, arg_extract_path,
" (clean after)" if arg_should_clean else "", arg_symlink, arg_jenkins_url, arg_config_location)
(args.port,
arg_extract_path,
" (clean after)" if deployer.should_clean else "",
arg_symlink,
arg_jenkins_url,
deployer.config_location,
)
)
app.run(host="0.0.0.0", port=args.port, debug=True)