From e4082d4f45fb212d5fa04279b69731f607e288f8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Sep 2017 14:25:33 +0100 Subject: [PATCH 1/7] Support multiple configs on the command line --- scripts/deploy.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index e7ad3f78a8..65c3bac6e9 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -165,7 +165,7 @@ if __name__ == "__main__": ) ) parser.add_argument( - "--config", nargs='?', default='./config.json', help=( + "--config", nargs='*', default='./config.json', help=( "Write a symlink at config.json in the extracted tarball to this \ location. (Default: '%(default)s')" ) @@ -182,8 +182,7 @@ if __name__ == "__main__": deployer.packages_path = args.packages_dir deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean - deployer.config_locations = { - "config.json": args.config, - } + + deployer.config_locations = { os.path.basename(c): c for c in args.config } deployer.deploy(args.tarball, args.extract_path) From be938ac0f4cbc701a101df5cb7e2b2b4e9ece96e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Sep 2017 14:38:41 +0100 Subject: [PATCH 2/7] Accept globs for the config location and make the default config*.sjon so we'll automatically symlink any config files in the current dir. --- scripts/deploy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 65c3bac6e9..e710e2c38d 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -14,6 +14,7 @@ import subprocess import sys import tarfile import shutil +import glob try: # python3 @@ -165,7 +166,7 @@ if __name__ == "__main__": ) ) parser.add_argument( - "--config", nargs='*', default='./config.json', help=( + "--config", nargs='*', default='./config*.json', help=( "Write a symlink at config.json in the extracted tarball to this \ location. (Default: '%(default)s')" ) @@ -183,6 +184,9 @@ if __name__ == "__main__": deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean - deployer.config_locations = { os.path.basename(c): c for c in args.config } + deployer.config_locations = {} + + for c in args.config: + deployer.config_locations.update({ os.path.basename(c): pth for pth in glob.iglob(c) }) deployer.deploy(args.tarball, args.extract_path) From 6ff91789b2a93c1bd5b33c3fded78f24327dce1b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Sep 2017 17:04:31 +0100 Subject: [PATCH 3/7] Rename 'config' option to 'symlink' because that's really all it's doing and we can use it for other things --- scripts/deploy.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index e710e2c38d..d80f5329af 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -67,7 +67,7 @@ class Deployer: self.bundles_path = None self.should_clean = False # filename -> symlink path e.g 'config.localhost.json' => '../localhost/config.json' - self.config_locations = {} + self.symlink_paths = {} self.verify_signature = True def deploy(self, tarball, extract_path): @@ -99,11 +99,11 @@ class Deployer: print ("Extracted into: %s" % extracted_dir) - if self.config_locations: - for config_filename, config_loc in self.config_locations.iteritems(): + if self.symlink_paths: + for link_path, file_path in self.symlink_paths.iteritems(): create_relative_symlink( - target=config_loc, - linkname=os.path.join(extracted_dir, config_filename) + target=file_path, + linkname=os.path.join(extracted_dir, link_path) ) if self.bundles_path: @@ -166,9 +166,10 @@ if __name__ == "__main__": ) ) parser.add_argument( - "--config", nargs='*', default='./config*.json', help=( - "Write a symlink at config.json in the extracted tarball to this \ - location. (Default: '%(default)s')" + "--symlink", nargs='*', default='./config*.json', help=( + "Symlink these files into the root of the deployed tarball. \ + Useful for config files and home pages. Supports glob syntax. \ + (Default: '%(default)s')" ) ) parser.add_argument( @@ -184,9 +185,11 @@ if __name__ == "__main__": deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean - deployer.config_locations = {} + deployer.symlink_paths = {} - for c in args.config: - deployer.config_locations.update({ os.path.basename(c): pth for pth in glob.iglob(c) }) + for symlink in args.symlink: + deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(symlink) }) + + print("%r", (deployer.symlink_paths,)) deployer.deploy(args.tarball, args.extract_path) From b9b4ac3166d20038e2723366e6e54658f254ba22 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Sep 2017 17:22:47 +0100 Subject: [PATCH 4/7] Rename symlink to 'include' & add to redeploy Because redeploy.py already has a 'symlink' option that does something else. --- scripts/deploy.py | 6 +++--- scripts/redeploy.py | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index d80f5329af..0f877ab085 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -166,7 +166,7 @@ if __name__ == "__main__": ) ) parser.add_argument( - "--symlink", nargs='*', default='./config*.json', help=( + "--include", nargs='*', default='./config*.json', help=( "Symlink these files into the root of the deployed tarball. \ Useful for config files and home pages. Supports glob syntax. \ (Default: '%(default)s')" @@ -187,8 +187,8 @@ if __name__ == "__main__": deployer.symlink_paths = {} - for symlink in args.symlink: - deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(symlink) }) + for include in args.include: + deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) }) print("%r", (deployer.symlink_paths,)) diff --git a/scripts/redeploy.py b/scripts/redeploy.py index e10a48c008..a43fb8a117 100755 --- a/scripts/redeploy.py +++ b/scripts/redeploy.py @@ -15,6 +15,7 @@ import json, requests, tarfile, argparse, os, errno import time import traceback from urlparse import urljoin +import glob from flask import Flask, jsonify, request, abort @@ -188,15 +189,12 @@ if __name__ == "__main__": ) ) - def _raise(ex): - raise ex - - # --config config.json=../../config.json --config config.localhost.json=./localhost.json + # --include ../../config.json ./localhost.json homepages/* parser.add_argument( - "--config", action="append", dest="configs", - type=lambda kv: kv.split("=", 1) if "=" in kv else _raise(Exception("Missing =")), help=( - "A list of configs to symlink into the extracted tarball. \ - For example, --config config.json=../config.json config2.json=../test/config.json" + "--include", nargs='*', default='./config*.json', help=( + "Symlink these files into the root of the deployed tarball. \ + Useful for config files and home pages. Supports glob syntax. \ + (Default: '%(default)s')" ) ) parser.add_argument( @@ -220,7 +218,11 @@ if __name__ == "__main__": deployer = Deployer() deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean - deployer.config_locations = dict(args.configs) if args.configs else {} + + deployer.symlink_paths = {} + + for include in args.include: + deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) }) # we don't pgp-sign jenkins artifacts; instead we rely on HTTPS access to @@ -234,13 +236,13 @@ if __name__ == "__main__": deploy_tarball(args.tarball_uri, build_dir) else: print( - "Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Config locations: %s" % + "Listening on port %s. Extracting to %s%s. Symlinking to %s. Jenkins URL: %s. Include patterns: %s" % (args.port, arg_extract_path, " (clean after)" if deployer.should_clean else "", arg_symlink, arg_jenkins_url, - deployer.config_locations, + deployer.symlink_paths, ) ) app.run(host="0.0.0.0", port=args.port, debug=True) From 21d233b5af3efc8af7ac550f760c160d76e2b42e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Sep 2017 17:48:47 +0100 Subject: [PATCH 5/7] oops, leftover debug --- scripts/deploy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 0f877ab085..3501c250a7 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -190,6 +190,4 @@ if __name__ == "__main__": for include in args.include: deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) }) - print("%r", (deployer.symlink_paths,)) - deployer.deploy(args.tarball, args.extract_path) From 6ec26932afcbe3618bec6d532e91f7e765536bee Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Sep 2017 17:54:52 +0100 Subject: [PATCH 6/7] This is the default anyway --- scripts/redeploy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/redeploy.py b/scripts/redeploy.py index a43fb8a117..fb550dcf8e 100755 --- a/scripts/redeploy.py +++ b/scripts/redeploy.py @@ -219,8 +219,6 @@ if __name__ == "__main__": deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean - deployer.symlink_paths = {} - for include in args.include: deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) }) From b4ea938be3feefdd79a75be6ebb5f91431897095 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 20 Sep 2017 18:17:06 +0100 Subject: [PATCH 7/7] This is the default --- scripts/deploy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 3501c250a7..e8fd9aa455 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -185,8 +185,6 @@ if __name__ == "__main__": deployer.bundles_path = args.bundles_dir deployer.should_clean = args.clean - deployer.symlink_paths = {} - for include in args.include: deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })