backports: add kernel uploader support
v4 cover backports: add kernel uploader support
This is a simple respin of some old patches I had for adding kernel uploading support into our scripts. I had agreed to split this up but in the end just never got to it. Might as well merge it and evolve it.
Luis R. Rodriguez (8):
lib/bpgit.py: enable extra arguments on git describe lib/bpgit.py: add git status support lib/bpgit.py: add git clean support lib/bpgit.py: add git tree verification support lib/bpgit.py: add support for git paranoia lib/bpgpg.py: add simple gpg helper for gpg signing lib/bpkup.py: add or own kernel uploader helper lib gentree.py: add kernel upload support
gentree.py | 237 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- lib/bpgit.py | 93 ++++++++++++++++++++++- lib/bpgpg.py | 22 ++++++ lib/bpkup.py | 44 +++++++++++ 4 files changed, 388 insertions(+), 8 deletions(-) create mode 100644 lib/bpgpg.py create mode 100644 lib/bpkup.py
Signature · 1 line
1.8.4.rc3
PATCH v4 1/8 lib/bpgit.py: enable extra arguments on git describe
This lets us throw at it whatever extra stuff we want.
--- gentree.py | 5 +++-- lib/bpgit.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-)
gentree.py +3 -2
@@ -468,9 +468,10 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, backports_version = "(see git)" kernel_version = "(see git)" else: - backports_version = git.describe(tree=source_dir) + backports_version = git.describe(tree=source_dir, extra_args=['--long']) kernel_version = git.describe(rev=args.git_revision or 'HEAD', - tree=args.kerneldir) + tree=args.kerneldir, + extra_args=['--long']) f = open(os.path.join(args.outdir, 'versions'), 'w') f.write('BACKPORTS_VERSION="%s"\n' % backports_version) f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
lib/bpgit.py +8 -2
@@ -27,8 +27,14 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha -def describe(rev='HEAD', tree=None): - process = subprocess.Popen(['git', 'describe', '--always', '--long', rev], +def describe(rev='HEAD', tree=None, extra_args=[]): + cmd = ['git', 'describe', '--always'] + + cmd.extend(extra_args) + if rev is not None: + cmd.append(rev) + + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, universal_newlines=True, cwd=tree) stdout = process.communicate()[0]
Signature · 1 line
1.8.4.rc3
PATCH v4 2/8 lib/bpgit.py: add git status support
In reply to Luis R. RodriguezWe use --porcelain given that this spits out the results in an easy-to-parse format for scripts and will remain stable across git versions and regardless of user configuration.
We will use this later.
--- lib/bpgit.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+)
lib/bpgit.py +54 -0
@@ -27,6 +27,60 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha +def status(tree=None): + ''' + For interpretation of the porcelain output refer to + the git status(1) man page. In summary the first column is the + index state, the second represents the working tree state and + the third column are files in cases of renames. This gives back + None in case no changes are present, otherwise it returns a list + of dict entries with key values: index, work_tree, and files. The + files are a list of all files found on that entry, in case of a rename + this would consist of a list of 2 files. + + As an example if you had this on your git status: + + R udev/foo.sh -> poo/foo.sh + D scripts/bar.sh + ?? .poo.py.swp + + This would be transposed into the following dict: + + results = status(tree=your_git_tree_path) + if not results: + return 0 + for entry in results: + print entry + + {'files': [' udev/foo.sh', 'poo/foo.sh'], 'index': 'R', 'work_tree': ' '} + {'files': [' scripts/bar.sh'], 'index': 'D', 'work_tree': ' '} + {'files': [' .poo.py.swp'], 'index': '?', 'work_tree': '?'} + ''' + def split_status(entry): + if len(entry) == 0: + return None + if len(entry) == 1: + return dict(index=entry[0], work_tree=None, files=None) + if len(entry) == 2: + return dict(index=entry[0], work_tree=entry[1], files=None) + else: + return dict(index=entry[0], work_tree=entry[1], + files=entry[2:].split(' -> ')) + + cmd = ['git', 'status', '--porcelain'] + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, + universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + _check(process) + + list_status = stdout.split('\n') + if (len(list_status) == 1 and list_status[0] == ''): + return None + return [split_status(entry) for entry in list_status] + def describe(rev='HEAD', tree=None, extra_args=[]): cmd = ['git', 'describe', '--always']
Signature · 1 line
1.8.4.rc3
PATCH v4 3/8 lib/bpgit.py: add git clean support
In reply to Luis R. RodriguezWe'll use this later.
--- lib/bpgit.py | 10 ++++++++++ 1 file changed, 10 insertions(+)
lib/bpgit.py +10 -0
@@ -27,6 +27,16 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha +def clean(tree=None): + cmd = ['git', 'clean', '-f', '-x', '-d', '-q'] + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + _check(process) + def status(tree=None): ''' For interpretation of the porcelain output refer to
Signature · 1 line
1.8.4.rc3
PATCH v4 4/8 lib/bpgit.py: add git tree verification support
In reply to Luis R. RodriguezThis verifies that the tree provided has its last commit both tagged and also digitally signed. In short it does:
git tag -v $(git describe --dirty)
This will be used later.
--- lib/bpgit.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)
lib/bpgit.py +12 -0
@@ -107,6 +107,18 @@ def describe(rev='HEAD', tree=None, extra_args=[]): return stdout.strip() +def verify(git_tree): + tag = describe(rev=None, tree=git_tree, extra_args=['--dirty']) + cmd = ['git', 'tag', '-v', tag] + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=git_tree) + stdout = process.communicate()[0] + process.wait() + + return dict(r=process.returncode, output=stdout) + def init(tree=None): process = subprocess.Popen(['git', 'init'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
Signature · 1 line
1.8.4.rc3
PATCH v4 5/8 lib/bpgit.py: add support for git paranoia
In reply to Luis R. RodriguezThis cleans sanitizes a source tree and ensures
no content is present from what was intended.
--- lib/bpgit.py | 7 +++++++ 1 file changed, 7 insertions(+)
lib/bpgit.py +7 -0
@@ -119,6 +119,13 @@ def verify(git_tree): return dict(r=process.returncode, output=stdout) +def paranoia(tree): + clean(tree) + poo = status(tree) + if (poo): + return dict(r=-1, output=poo) + return verify(tree) + def init(tree=None): process = subprocess.Popen(['git', 'init'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
Signature · 1 line
1.8.4.rc3
PATCH v4 6/8 lib/bpgpg.py: add simple gpg helper for gpg signing
In reply to Luis R. RodriguezNote that kup expects --armor --detach-sign, we'll pass that as part of our extra_args. The python gpg lib doesn't provide support for random arguments as this one, so just add our own little helper and carry it over.
--- lib/bpgpg.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 lib/bpgpg.py
lib/bpgpg.py new file +22 -0
@@ -0,0 +1,22 @@ +import subprocess, os + +class GpgError(Exception): + pass +class ExecutionError(GpgError): + def __init__(self, errcode): + self.error_code = errcode + +def sign(input_file, extra_args=[]): + cmd = ['gpg', '--sign'] + + cmd.extend(extra_args) + cmd.append(input_file) + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True) + stdout = process.communicate()[0] + process.wait() + if process.returncode != 0: + raise ExecutionError(process.returncode) + return stdout
Signature · 1 line
1.8.4.rc3
PATCH v4 7/8 lib/bpkup.py: add or own kernel uploader helper lib
In reply to Luis R. Rodriguezkup is written in perl, add some basic python interfaces for it to allow us to upload to kernel.org backport releases.
--- lib/bpkup.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 lib/bpkup.py
lib/bpkup.py new file +44 -0
@@ -0,0 +1,44 @@ +import subprocess, os + +class KupError(Exception): + pass +class ExecutionError(KupError): + def __init__(self, errcode): + self.error_code = errcode + +def _check(process): + if process.returncode != 0: + raise ExecutionError(process.returncode) + +def mkdir(path): + cmd = ['kup', 'mkdir', path] + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout + +def ls(path=None): + cmd = ['kup', 'ls', path] + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout + +def put(tar_bz2, signed_tar, path): + cmd = ['kup', 'put', tar_bz2, signed_tar, path] + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout
Signature · 1 line
1.8.4.rc3
PATCH v4 8/8 gentree.py: add kernel upload support
In reply to Luis R. RodriguezThis streamlines our release process making it easier for transitioning releases onto other backport developers.
--- gentree.py | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 228 insertions(+), 4 deletions(-)
gentree.py +228 -4
@@ -4,6 +4,7 @@ # import argparse, sys, os, errno, shutil, re, subprocess +import tarfile, bz2 # find self source_dir = os.path.abspath(os.path.dirname(__file__))
@@ -11,6 +12,8 @@ sys.path.append(source_dir) # and import libraries we have from lib import kconfig, patch, make from lib import bpgit as git +from lib import bpgpg as gpg +from lib import bpkup as kup def read_copy_list(copyfile): """
@@ -268,6 +271,169 @@ def git_debug_snapshot(args, name): return git.commit_all(name, tree=args.outdir) +def get_rel_spec_stable(rel): + """ + Returns release specs for a linux-stable backports based release. + """ + if ("rc" in rel): + m = re.match(r"(?P<VERSION>\d+)\.+" \ + "(?P<PATCHLEVEL>\d+)[.]*" \ + "(?P<SUBLEVEL>\d*)" \ + "[-rc]+(?P<RC_VERSION>\d+)\-+" \ + "(?P<RELMOD_UPDATE>\d+)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + else: + m = re.match(r"(?P<VERSION>\d+)\.+" \ + "(?P<PATCHLEVEL>\d+)[.]*" \ + "(?P<SUBLEVEL>\d*)\-+" \ + "(?P<RELMOD_UPDATE>\d+)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + if (not m): + return m + return m.groupdict() + +def get_rel_spec_next(rel): + """ + Returns release specs for a linux-next backports based release. + """ + m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \ + "(?P<RELMOD_UPDATE>\d*)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + if (not m): + return m + return m.groupdict() + +def get_rel_prep(rel): + """ + Returns a dict with prep work details we need prior to + uploading a backports release to kernel.org + """ + rel_specs = get_rel_spec_stable(rel) + is_stable = True + rel_tag = "" + paths = list() + if (not rel_specs): + rel_specs = get_rel_spec_next(rel) + if (not rel_specs): + sys.stdout.write("rel: %s\n" % rel) + return None + if (rel_specs['RELMOD_UPDATE'] == '0' or + rel_specs['RELMOD_UPDATE'] == '1'): + return None + is_stable = False + date = rel_specs['DATE_VERSION'] + year = date[0:4] + if (len(year) != 4): + return None + month = date[4:6] + if (len(month) != 2): + return None + day = date[6:8] + if (len(day) != 2): + return None + paths.append(year) + paths.append(month) + paths.append(day) + rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "") + else: + ignore = "-" + if (not rel_specs['RELMOD_UPDATE']): + return None + if (rel_specs['RELMOD_UPDATE'] == '0'): + return None + ignore += rel_specs['RELMOD_UPDATE'] + if (rel_specs['RELMOD_TYPE'] != ''): + ignore += rel_specs['RELMOD_TYPE'] + base_rel = rel.replace(ignore, "") + paths.append(base_rel) + rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "") + + rel_prep = dict(stable = is_stable, + expected_tag = rel_tag, + paths_to_create = paths) + return rel_prep + +def create_tar_and_bz2(tar_name, dir_to_tar): + """ + We need both a tar file and bzip2 for kernel.org, the tar file + gets signed, then we upload the compressed version, kup-server + in the backend decompresses and verifies the tarball against + our signature. + """ + parent = os.path.dirname(tar_name) + tar = tarfile.open(tar_name, "w") + tar.add(dir_to_tar) + tar.close() + + tar_file = open(tar_name, "r") + + bz2_file = bz2.BZ2File(tar_name + ".bz2", 'wb', compresslevel=9) + bz2_file.write(tar_file.read()) + bz2_file.close() + + tar.close() + +def upload_release(args, rel_prep, logwrite=lambda x:None): + """ + Given a path of a relase make tarball out of it, PGP sign it, and + then upload it to kernel.org using kup. + + The linux-next based release do not require a RELMOD_UPDATE + given that typically only one release is made per day. Using + RELMOD_UPDATE for these releases is allowed though and if + present it must be > 1. + + The linux-stable based releases require a RELMOD_UPDATE. + + RELMOD_UPDATE must be numeric and > 0 just as the RC releases + of the Linux kernel. + + The tree must also be tagged with the respective release, without + the RELMOD_TYPE. For linux-next based releases this consists of + backports- followed by DATE_VERSION and if RELMOD_TYPE is present. + For linux-stable releases this consists of v followed by the + full release version except the RELMOD_TYPE. + + Uploads will not be allowed if these rules are not followed. + """ + korg_path = "/pub/linux/kernel/projects/backports" + + if (rel_prep['stable']): + korg_path += "stable" + + parent = os.path.dirname(args.outdir) + release = os.path.basename(args.outdir) + tar_name = parent + '/' + release + ".tar" + bzip2_name = tar_name + ".bz2" + + create_tar_and_bz2(tar_name, args.outdir) + + logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign'])) + + logwrite("------------------------------------------------------") + + if (not args.kup_test): + logwrite("About to upload, current target path contents:") + else: + logwrite("kup-test: current target path contents:") + + logwrite(kup.ls(path=korg_path)) + + for path in rel_prep['paths_to_create']: + korg_path += '/' + path + if (not args.kup_test): + logwrite(kup.mkdir(path)) + if (not args.kup_test): + logwrite(kup.put(bzip2_name, tar_name + '.asc', korg_path)) + logwrite("\nFinished upload!\n") + logwrite("Target path contents:") + logwrite(kup.ls(path=korg_path)) + else: + kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (bzip2_name, tar_name + '.asc', korg_path) + logwrite("kup-test: skipping cmd: %s" % kup_cmd) def _main(): # set up and parse arguments
@@ -296,6 +462,15 @@ def _main(): help='Print more verbose information') parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str, action='append', default=[], help='Extra driver directory/copy-list.') + parser.add_argument('--kup', const=True, default=False, action="store_const", + help='For maintainers: upload a release to kernel.org') + parser.add_argument('--kup-test', const=True, default=False, action="store_const", + help='For maintainers: do all the work as if you were about to ' + + 'upload to kernel.org but do not do the final `kup put` ' + + 'and also do not run any `kup mkdir` commands. This will ' + + 'however run `kup ls` on the target paths so ' + + 'at the very least we test your kup configuration. ' + + 'If this is your first time uploading use this first!') args = parser.parse_args() def logwrite(msg):
@@ -307,16 +482,22 @@ def _main(): git_revision=args.git_revision, clean=args.clean, refresh=args.refresh, base_name=args.base_name, gitdebug=args.gitdebug, verbose=args.verbose, - extra_driver=args.extra_driver, logwrite=logwrite) + extra_driver=args.extra_driver, + kup=args.kup, + kup_test=args.kup_test, + logwrite=logwrite) def process(kerneldir, outdir, copy_list_file, git_revision=None, clean=False, refresh=False, base_name="Linux", gitdebug=False, - verbose=False, extra_driver=[], logwrite=lambda x:None, + verbose=False, extra_driver=[], kup=False, + kup_test=False, + logwrite=lambda x:None, git_tracked_version=False): class Args(object): def __init__(self, kerneldir, outdir, copy_list_file, git_revision, clean, refresh, base_name, - gitdebug, verbose, extra_driver): + gitdebug, verbose, extra_driver, kup, + kup_test): self.kerneldir = kerneldir self.outdir = outdir self.copy_list = copy_list_file
@@ -327,10 +508,48 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, self.gitdebug = gitdebug self.verbose = verbose self.extra_driver = extra_driver + self.kup = kup + self.kup_test = kup_test + def git_paranoia(tree=None, logwrite=lambda x:None): + data = git.paranoia(tree) + if (data['r'] != 0): + logwrite('Cannot use %s' % tree) + logwrite('%s' % data['output']) + sys.exit(data['r']) + else: + logwrite('Validated tree: %s' % tree) + args = Args(kerneldir, outdir, copy_list_file, git_revision, clean, refresh, base_name, - gitdebug, verbose, extra_driver) + gitdebug, verbose, extra_driver, kup, kup_test) + rel_prep = None + # start processing ... + if (args.kup or args.kup_test): + git_paranoia(source_dir, logwrite) + git_paranoia(kerneldir, logwrite) + + rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty']) + release = os.path.basename(args.outdir) + version = release.replace("backports-", "") + + rel_prep = get_rel_prep(version) + if (not rel_prep): + logwrite('Invalid backports release name: %s' % release) + logwrite('For rules on the release name see upload_release()') + sys.exit(1) + rel_type = "linux-stable" + if (not rel_prep['stable']): + rel_type = "linux-next" + if (rel_prep['expected_tag'] != rel_describe): + logwrite('Unexpected %s based backports release tag on' % rel_type) + logwrite('the backports tree tree: %s\n' % rel_describe) + logwrite('You asked to make a release with this ') + logwrite('directory name: %s' % release) + logwrite('The actual expected tag we should find on') + logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag']) + logwrite('For rules on the release name see upload_release()') + sys.exit(1) copy_list = read_copy_list(args.copy_list) deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
@@ -360,6 +579,8 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, # FIXME: should we add a git version of this (e.g. --git-extra-driver)? for src, copy_list in args.extra_driver: + if (args.kup or args.kup_test): + git_paranoia(src) copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir) git_debug_snapshot(args, 'Add driver sources')
@@ -561,6 +782,9 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, fo.close() git_debug_snapshot(args, "disable unsatisfied Makefile parts") + if (args.kup or args.kup_test): + upload_release(args, rel_prep, logwrite=logwrite) + logwrite('Done!') return 0
Signature · 1 line
1.8.4.rc3
Re: [PATCH v4 8/8] gentree.py: add kernel upload support
Quoted 118 lines · Luis R. Rodriguez
This streamlines our release process making it easier for transitioning releases onto other backport developers.
Signed-off-by Luis R. Rodriguez <[email protected]>--- gentree.py | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 228 insertions(+), 4 deletions(-)gentree.py +88
-0@@ -4,6 +4,7 @@ # import argparse, sys, os, errno, shutil, re, subprocess +import tarfile, bz2 # find self source_dir = os.path.abspath(os.path.dirname(__file__))@@ -11,6 +12,8 @@ sys.path.append(source_dir) # and import libraries we have from lib import kconfig, patch, make from lib import bpgit as git +from lib import bpgpg as gpg +from lib import bpkup as kup def read_copy_list(copyfile): """@@ -268,6 +271,169 @@ def git_debug_snapshot(args, name): return git.commit_all(name, tree=args.outdir) +def get_rel_spec_stable(rel): + """ + Returns release specs for a linux-stable backports based release. + """ + if ("rc" in rel): + m = re.match(r"(?P<VERSION>\d+)\.+" \ + "(?P<PATCHLEVEL>\d+)[.]*" \ + "(?P<SUBLEVEL>\d*)" \ + "[-rc]+(?P<RC_VERSION>\d+)\-+" \ + "(?P<RELMOD_UPDATE>\d+)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + else: + m = re.match(r"(?P<VERSION>\d+)\.+" \ + "(?P<PATCHLEVEL>\d+)[.]*" \ + "(?P<SUBLEVEL>\d*)\-+" \ + "(?P<RELMOD_UPDATE>\d+)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + if (not m): + return m + return m.groupdict() + +def get_rel_spec_next(rel): + """ + Returns release specs for a linux-next backports based release. + """ + m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \ + "(?P<RELMOD_UPDATE>\d*)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + if (not m): + return m + return m.groupdict() + +def get_rel_prep(rel): + """ + Returns a dict with prep work details we need prior to + uploading a backports release to kernel.org + """ + rel_specs = get_rel_spec_stable(rel) + is_stable = True + rel_tag = "" + paths = list() + if (not rel_specs): + rel_specs = get_rel_spec_next(rel) + if (not rel_specs): + sys.stdout.write("rel: %s\n" % rel) + return None + if (rel_specs['RELMOD_UPDATE'] == '0' or + rel_specs['RELMOD_UPDATE'] == '1'): + return None + is_stable = False + date = rel_specs['DATE_VERSION'] + year = date[0:4] + if (len(year) != 4): + return None + month = date[4:6] + if (len(month) != 2): + return None + day = date[6:8] + if (len(day) != 2): + return None + paths.append(year) + paths.append(month) + paths.append(day) + rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "") + else: + ignore = "-" + if (not rel_specs['RELMOD_UPDATE']): + return None + if (rel_specs['RELMOD_UPDATE'] == '0'): + return None + ignore += rel_specs['RELMOD_UPDATE'] + if (rel_specs['RELMOD_TYPE'] != ''): + ignore += rel_specs['RELMOD_TYPE'] + base_rel = rel.replace(ignore, "") + paths.append(base_rel) + rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "") + + rel_prep = dict(stable = is_stable, + expected_tag = rel_tag, + paths_to_create = paths) + return rel_prep +
gentree.py def git_debug_snapshot(args, name): · line 368
Quoted 10 lines
gentree.py +10
-0+def create_tar_and_bz2(tar_name, dir_to_tar): + """ + We need both a tar file and bzip2 for kernel.org, the tar file + gets signed, then we upload the compressed version, kup-server + in the backend decompresses and verifies the tarball against + our signature. + """ + parent = os.path.dirname(tar_name) + tar = tarfile.open(tar_name, "w") + tar.add(dir_to_tar)
This adds the file under its full path, this results in a tar with /home/hauke/... and so on in it.
Use this to only get the basename in the tar:tar.add(dir_to_tar, os.path.basename(dir_to_tar))
gentree.py def git_debug_snapshot(args, name): · line 377
Quoted 9 lines
gentree.py +9
-0+ tar.close() + + tar_file = open(tar_name, "r") + + bz2_file = bz2.BZ2File(tar_name + ".bz2", 'wb', compresslevel=9) + bz2_file.write(tar_file.read()) + bz2_file.close() + + tar.close()
I think this additional close is not necessary.
Quoted 174 lines
gentree.py +121
-4+ +def upload_release(args, rel_prep, logwrite=lambda x:None): + """ + Given a path of a relase make tarball out of it, PGP sign it, and + then upload it to kernel.org using kup. + + The linux-next based release do not require a RELMOD_UPDATE + given that typically only one release is made per day. Using + RELMOD_UPDATE for these releases is allowed though and if + present it must be > 1. + + The linux-stable based releases require a RELMOD_UPDATE. + + RELMOD_UPDATE must be numeric and > 0 just as the RC releases + of the Linux kernel. + + The tree must also be tagged with the respective release, without + the RELMOD_TYPE. For linux-next based releases this consists of + backports- followed by DATE_VERSION and if RELMOD_TYPE is present. + For linux-stable releases this consists of v followed by the + full release version except the RELMOD_TYPE. + + Uploads will not be allowed if these rules are not followed. + """ + korg_path = "/pub/linux/kernel/projects/backports" + + if (rel_prep['stable']): + korg_path += "stable" + + parent = os.path.dirname(args.outdir) + release = os.path.basename(args.outdir) + tar_name = parent + '/' + release + ".tar" + bzip2_name = tar_name + ".bz2" + + create_tar_and_bz2(tar_name, args.outdir) + + logwrite(gpg.sign(tar_name, extra_args=['--armor', '--detach-sign'])) + + logwrite("------------------------------------------------------") + + if (not args.kup_test): + logwrite("About to upload, current target path contents:") + else: + logwrite("kup-test: current target path contents:") + + logwrite(kup.ls(path=korg_path)) + + for path in rel_prep['paths_to_create']: + korg_path += '/' + path + if (not args.kup_test): + logwrite(kup.mkdir(path)) + if (not args.kup_test): + logwrite(kup.put(bzip2_name, tar_name + '.asc', korg_path)) + logwrite("\nFinished upload!\n") + logwrite("Target path contents:") + logwrite(kup.ls(path=korg_path)) + else: + kup_cmd = "kup put /\n\t\t%s /\n\t\t%s /\n\t\t%s" % (bzip2_name, tar_name + '.asc', korg_path) + logwrite("kup-test: skipping cmd: %s" % kup_cmd) def _main(): # set up and parse arguments@@ -296,6 +462,15 @@ def _main(): help='Print more verbose information') parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str, action='append', default=[], help='Extra driver directory/copy-list.') + parser.add_argument('--kup', const=True, default=False, action="store_const", + help='For maintainers: upload a release to kernel.org') + parser.add_argument('--kup-test', const=True, default=False, action="store_const", + help='For maintainers: do all the work as if you were about to ' + + 'upload to kernel.org but do not do the final `kup put` ' + + 'and also do not run any `kup mkdir` commands. This will ' + + 'however run `kup ls` on the target paths so ' + + 'at the very least we test your kup configuration. ' + + 'If this is your first time uploading use this first!') args = parser.parse_args() def logwrite(msg):@@ -307,16 +482,22 @@ def _main(): git_revision=args.git_revision, clean=args.clean, refresh=args.refresh, base_name=args.base_name, gitdebug=args.gitdebug, verbose=args.verbose, - extra_driver=args.extra_driver, logwrite=logwrite) + extra_driver=args.extra_driver, + kup=args.kup, + kup_test=args.kup_test, + logwrite=logwrite) def process(kerneldir, outdir, copy_list_file, git_revision=None, clean=False, refresh=False, base_name="Linux", gitdebug=False, - verbose=False, extra_driver=[], logwrite=lambda x:None, + verbose=False, extra_driver=[], kup=False, + kup_test=False, + logwrite=lambda x:None, git_tracked_version=False): class Args(object): def __init__(self, kerneldir, outdir, copy_list_file, git_revision, clean, refresh, base_name, - gitdebug, verbose, extra_driver): + gitdebug, verbose, extra_driver, kup, + kup_test): self.kerneldir = kerneldir self.outdir = outdir self.copy_list = copy_list_file@@ -327,10 +508,48 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, self.gitdebug = gitdebug self.verbose = verbose self.extra_driver = extra_driver + self.kup = kup + self.kup_test = kup_test + def git_paranoia(tree=None, logwrite=lambda x:None): + data = git.paranoia(tree) + if (data['r'] != 0): + logwrite('Cannot use %s' % tree) + logwrite('%s' % data['output']) + sys.exit(data['r']) + else: + logwrite('Validated tree: %s' % tree) + args = Args(kerneldir, outdir, copy_list_file, git_revision, clean, refresh, base_name, - gitdebug, verbose, extra_driver) + gitdebug, verbose, extra_driver, kup, kup_test) + rel_prep = None + # start processing ... + if (args.kup or args.kup_test): + git_paranoia(source_dir, logwrite) + git_paranoia(kerneldir, logwrite) + + rel_describe = git.describe(rev=None, tree=source_dir, extra_args=['--dirty']) + release = os.path.basename(args.outdir) + version = release.replace("backports-", "") + + rel_prep = get_rel_prep(version) + if (not rel_prep): + logwrite('Invalid backports release name: %s' % release) + logwrite('For rules on the release name see upload_release()') + sys.exit(1) + rel_type = "linux-stable" + if (not rel_prep['stable']): + rel_type = "linux-next" + if (rel_prep['expected_tag'] != rel_describe): + logwrite('Unexpected %s based backports release tag on' % rel_type) + logwrite('the backports tree tree: %s\n' % rel_describe) + logwrite('You asked to make a release with this ') + logwrite('directory name: %s' % release) + logwrite('The actual expected tag we should find on') + logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag']) + logwrite('For rules on the release name see upload_release()') + sys.exit(1) copy_list = read_copy_list(args.copy_list) deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))@@ -360,6 +579,8 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, # FIXME: should we add a git version of this (e.g. --git-extra-driver)? for src, copy_list in args.extra_driver: + if (args.kup or args.kup_test): + git_paranoia(src) copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir) git_debug_snapshot(args, 'Add driver sources')@@ -561,6 +782,9 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, fo.close() git_debug_snapshot(args, "disable unsatisfied Makefile parts") + if (args.kup or args.kup_test): + upload_release(args, rel_prep, logwrite=logwrite) + logwrite('Done!') return 0