backports: add kernel uploading support
cover backports: add kernel uploading support
From: "Luis R. Rodriguez" <[email protected]>
There were a few scripts left to convert to Python. One was git-paranoia and another was gen-release.sh. This converts both into the new framework under Python. The new option is --kup, this would only be used by whoever is maintaining releases for backports.
The Python gpg library didn't have support for "--armor --detach-sign" so for now we just add our own. For kup we add our own small wrapper.
A few thing we could do to help new maintainers or later enhancements but we can do that later:
* Add option to import PGP keys from the trees we manage or are reading data from.
* Add option to build tarball for non kup usage
We should just go hack up git paranoia upstream on git itself as well.
Luis R. Rodriguez (9):
lib/bpgit.py: make git describe --long optional lib/bpgit.py: add git describe --dirty support lib/bpgit.py: add git status support
lib/bpgit.py: add git clean support
lib/bpgit.py: add git tree verification support lib/bpgit.py: add support for git paranoia lib/bpgpg.py: add simple gpg helpers for --armor --detach-sign lib/bpkup.py: add or own kernel uploader helper lib gentree.py: add kernel upload support
gentree.py | 211 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- lib/bpgit.py | 56 +++++++++++++++- lib/bpgpg.py | 12 ++++ lib/bpkup.py | 38 +++++++++++ 4 files changed, 309 insertions(+), 8 deletions(-) create mode 100644 lib/bpgpg.py create mode 100644 lib/bpkup.py
Signature · 1 line
1.7.10.4
PATCH 1/9 lib/bpgit.py: make git describe --long optional
From: "Luis R. Rodriguez" <[email protected]>
When calling git describe through our Python library the long option is always used. Make this optional but keep the existing behaviour.
This will be used later without the long option and using an argument saves us from having to parse it. We need to use get_long over long given that long is a Python type.
--- gentree.py | 5 +++-- lib/bpgit.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-)
gentree.py +3 -2
@@ -447,9 +447,10 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, backports_version = "(see git)" kernel_version = "(see git)" else: - backports_version = git.describe(tree=source_dir) + backports_version = git.describe(tree=source_dir, get_long=True) kernel_version = git.describe(rev=args.git_revision or 'HEAD', - tree=args.kerneldir) + tree=args.kerneldir, + get_long=True) f = open(os.path.join(args.outdir, 'versions'), 'w') f.write('BACKPORTS_VERSION="%s"\n' % backports_version) f.write('BACKPORTED_KERNEL_VERSION="%s"\n' % kernel_version)
lib/bpgit.py +9 -2
@@ -27,8 +27,15 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha -def describe(rev='HEAD', tree=None): - process = subprocess.Popen(['git', 'describe', '--always', '--long', rev], +def describe(rev='HEAD', tree=None, get_long=False): + cmd = ['git', 'describe', '--always'] + + if (get_long): + cmd.append('--long') + + cmd.append(rev) + + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True, universal_newlines=True, cwd=tree) stdout = process.communicate()[0]
Signature · 1 line
1.7.10.4
Re: [PATCH 1/9] lib/bpgit.py: make git describe --long optional
Why not do:
lib/bpgit.py def rev_parse(rev='HEAD', tree=None): · line 36
Quoted 9 lines
lib/bpgit.py +7
-2-def describe(rev='HEAD', tree=None): - process = subprocess.Popen(['git', 'describe', '--always', '--long', rev], +def describe(rev='HEAD', tree=None, get_long=False): + cmd = ['git', 'describe', '--always'] + + if (get_long): + cmd.append('--long') + + cmd.append(rev)
def describe(rev='HEAD', tree=None, extra_args=[]):
cmd = ['git', 'describe', '--always']
cmd.extend(extra_args)
if rev is not None:
cmd.append(rev)
process = ...that way you can do
bpgit.describe(rev=None, extra_args=['--dirty'])
or
bpgit.describe(extra_args=['--long'])
and don't need two different patches.
johannes
Re: [PATCH 1/9] lib/bpgit.py: make git describe --long optional
Quoted 21 lines · Johannes Berg
Why not do:
lib/bpgit.py def rev_parse(rev='HEAD', tree=None): · line 36
Quoted 9 lines
lib/bpgit.py +7
-2-def describe(rev='HEAD', tree=None): - process = subprocess.Popen(['git', 'describe', '--always', '--long', rev], +def describe(rev='HEAD', tree=None, get_long=False): + cmd = ['git', 'describe', '--always'] + + if (get_long): + cmd.append('--long') + + cmd.append(rev)def
describe(rev='HEAD', tree=None, extra_args=[]):cmd = ['git', 'describe', '--always'] cmd.extend(extra_args) if rev is not None: cmd.append(rev) process = ...that way you can do
Quoted 8 lines
bpgit.describe(rev=None, extra_args=['--dirty'])or
bpgit.describe(extra_args=['--long'])and don't need two different patches.
Although probably overkill for git describe given that I don't foresee us adding more options I can see this being good practice for anything else we'll add so sure, I'll make these changes.
Luis
PATCH 2/9 lib/bpgit.py: add git describe --dirty support
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
git describe --dirty pegs a "-dirty" at the end of your tag if you haven't properly commited things. When the the --dirty flag is used assume the current state of the tree.
We will use this later.
--- lib/bpgit.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
lib/bpgit.py +5 -3
@@ -27,13 +27,15 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha -def describe(rev='HEAD', tree=None, get_long=False): +def describe(rev='HEAD', tree=None, get_long=False, dirty=False): cmd = ['git', 'describe', '--always'] if (get_long): cmd.append('--long') - - cmd.append(rev) + if (dirty): + cmd.append('--dirty') + else: + cmd.append(rev) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
Signature · 1 line
1.7.10.4
PATCH 3/9 lib/bpgit.py: add git status support
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
We use --porcelain given that this spits out the results in an easy-to-parse format for scripts and will remain stable across git versions and regardless of user configuration.
We will use this later.
--- lib/bpgit.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)
lib/bpgit.py +12 -0
@@ -27,6 +27,18 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha +def status(tree=None): + cmd = ['git', 'status', '--porcelain'] + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout.strip() + def describe(rev='HEAD', tree=None, get_long=False, dirty=False): cmd = ['git', 'describe', '--always']
Signature · 1 line
1.7.10.4
Re: [PATCH 3/9] lib/bpgit.py: add git status support
Quoted 22 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
We use --porcelain given that this spits out the results in an easy-to-parse format for scripts and will remain stable across git versions and regardless of user configuration.
We will use this later.
Signed-off-by Luis R. Rodriguez <[email protected]>--- lib/bpgit.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)lib/bpgit.py +1
-0@@ -27,6 +27,18 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha +def status(tree=None):
lib/bpgit.py def rev_parse(rev='HEAD', tree=None): · line 40
Quoted 10 lines
lib/bpgit.py +10
-0+ cmd = ['git', 'status', '--porcelain'] + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout.strip()
I think it would make some sense to parse this here into something more "pythonic", like a list of tuples or so.
johannes
Re: [PATCH 3/9] lib/bpgit.py: add git status support
Quoted 22 lines · Johannes Berg
Quoted 21 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
We use --porcelain given that this spits out the results in an easy-to-parse format for scripts and will remain stable across git versions and regardless of user configuration.
We will use this later.
Signed-off-by Luis R. Rodriguez <[email protected]>--- lib/bpgit.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)lib/bpgit.py +0
-0@@ -27,6 +27,18 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha
Quoted 14 lines
lib/bpgit.py def rev_parse(rev='HEAD', tree=None): · line 40
Quoted 11 lines
lib/bpgit.py +11
-0+def status(tree=None): + cmd = ['git', 'status', '--porcelain'] + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout.strip()I think it would make some sense to parse this here into something more "pythonic", like a list of tuples or so.
Sure.
Luis
PATCH 4/9 lib/bpgit.py: add git clean support
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
We'll use this later.
--- lib/bpgit.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)
lib/bpgit.py +12 -0
@@ -27,6 +27,18 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha +def clean(tree=None): + cmd = ['git', 'clean', '-f', '-x', '-d'] + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout.strip() + def status(tree=None): cmd = ['git', 'status', '--porcelain']
Signature · 1 line
1.7.10.4
Re: [PATCH 4/9] lib/bpgit.py: add git clean support
Quoted 14 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
We'll use this later.
Signed-off-by Luis R. Rodriguez <[email protected]>--- lib/bpgit.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)lib/bpgit.py +0
-0
lib/bpgit.py def rev_parse(rev='HEAD', tree=None): · line 31
Quoted 5 lines
lib/bpgit.py +2
-0@@ -27,6 +27,18 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha +def clean(tree=None): + cmd = ['git', 'clean', '-f', '-x', '-d']
why not add -q
lib/bpgit.py def rev_parse(rev='HEAD', tree=None): · line 40
Quoted 8 lines
lib/bpgit.py +8
-0+ process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout.strip()
Do you really care about its output?
johannes
Re: [PATCH 4/9] lib/bpgit.py: add git clean support
Quoted 18 lines · Johannes Berg
Quoted 17 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
We'll use this later.
Signed-off-by Luis R. Rodriguez <[email protected]>--- lib/bpgit.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)lib/bpgit.py +0
-0@@ -27,6 +27,18 @@ def rev_parse(rev='HEAD', tree=None): raise SHAError() return sha
Quoted 4 lines
lib/bpgit.py def rev_parse(rev='HEAD', tree=None): · line 31
Quoted 2 lines
lib/bpgit.py +2
-0+def clean(tree=None): + cmd = ['git', 'clean', '-f', '-x', '-d']why not add -q
Sure.
Quoted 10 lines
lib/bpgit.py def rev_parse(rev='HEAD', tree=None): · line 40
Quoted 8 lines
lib/bpgit.py +8
-0+ process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout.strip()Do you really care about its output?
Nah, I'll trash it.
Luis
PATCH 5/9 lib/bpgit.py: add git tree verification support
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
This verifies that the tree provided has its last commit both tagged and also digitally signed. In short it does:
git tag -v $(git describe --dirty)
This will be used later.
--- lib/bpgit.py | 12 ++++++++++++ 1 file changed, 12 insertions(+)
lib/bpgit.py +12 -0
@@ -70,6 +70,18 @@ def describe(rev='HEAD', tree=None, get_long=False, dirty=False): return stdout.strip() +def verify(tree=None): + tag = describe(tree=tree, dirty=True) + cmd = ['git', 'tag', '-v', tag] + + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True, cwd=tree) + stdout = process.communicate()[0] + process.wait() + + return dict(r=process.returncode, output=stdout) + def init(tree=None): process = subprocess.Popen(['git', 'init'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
Signature · 1 line
1.7.10.4
PATCH 6/9 lib/bpgit.py: add support for git paranoia
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
This cleans sanitizes a source tree and ensures
no content is present from what was intended.
--- lib/bpgit.py | 7 +++++++ 1 file changed, 7 insertions(+)
lib/bpgit.py +7 -0
@@ -82,6 +82,13 @@ def verify(tree=None): return dict(r=process.returncode, output=stdout) +def paranoia(tree=None): + clean(tree) + extra = status(tree) + if (len(extra) != 0): + return dict(r=-1, output=extra) + return verify(tree) + def init(tree=None): process = subprocess.Popen(['git', 'init'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
Signature · 1 line
1.7.10.4
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
Quoted 4 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
This cleans sanitizes a source tree and ensures
no content is present from what was intended.
All of this (including the previous patches) makes me think "--git-revision HEAD" should just be the default instead of trying to muck with the working directory of the kernel tree too much?
johannes
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
Quoted 9 lines · Johannes Berg
Quoted 4 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
This cleans sanitizes a source tree and ensures
no content is present from what was intended.All of this (including the previous patches) makes me think "--git-revision HEAD" should just be the default instead of trying to muck with the working directory of the kernel tree too much?
Not sure I follow, but in any case, I just want to start being able to start making daily releases daily and fast, can we address this later?
Luis
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
Quoted 8 lines · Luis R. Rodriguez
Quoted 6 lines · Johannes Berg
Quoted 4 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
This cleans sanitizes a source tree and ensures
no content is present from what was intended.
Quoted 6 lines
Quoted 3 lines
All of this (including the previous patches) makes me think "--git-revision HEAD" should just be the default instead of trying to muck with the working directory of the kernel tree too much?
Not sure I follow, but in any case, I just want to start being able to start making daily releases daily and fast, can we address this later?
I'm suggesting that you use --git-revision to do so, then you only have to verify the tag you passed to --git-revision is signed and not have to muck around with the tree. Technically, all the stuff you'd been doing there is racy too, if you edit the checkout while the script is running, and I think it's a bit pointless. I doubt it's even faster than using --git-revision if it checks everything.
IOW, why go to all this complexity? I'd say all you need to do is require that not passing --git-revision when --kup is passed is an error. Then you can make --kup verify the tag passed to --git-revision by just calling "git tag -v ..." for the --git-revision argument. This will error out if it's not a tag.
Basically this means you only need patch 4, 6, 7 and a modified version of 8.
johannes
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
Quoted 23 lines · Johannes Berg
Quoted 8 lines · Luis R. Rodriguez
Quoted 6 lines · Johannes Berg
Quoted 4 lines · Luis R. Rodriguez
From: "Luis R. Rodriguez" <[email protected]>
This cleans sanitizes a source tree and ensures
no content is present from what was intended.Quoted 6 lines
Quoted 3 lines
All of this (including the previous patches) makes me think "--git-revision HEAD" should just be the default instead of trying to muck with the working directory of the kernel tree too much?
Not sure I follow, but in any case, I just want to start being able to start making daily releases daily and fast, can we address this later?
I'm suggesting that you use --git-revision to do so, then you only have to verify the tag you passed to --git-revision is signed and not have to muck around with the tree. Technically, all the stuff you'd been doing there is racy too, if you edit the checkout while the script is running, and I think it's a bit pointless. I doubt it's even faster than using --git-revision if it checks everything.
Quoted 8 lines
IOW, why go to all this complexity? I'd say all you need to do is require that not passing --git-revision when --kup is passed is an error. Then you can make --kup verify the tag passed to --git-revision by just calling "git tag -v ..." for the --git-revision argument. This will error out if it's not a tag.
Basically this means you only need patch 4, 6, 7 and a modified version of 8.
I see what you're saying now and I actually did consider it, the reason I didn't use --git-revision is speed. Git reset --hard tag, and then copying the code directly is a lot faster. The race issues you point out though are worth reconsidering this though.
Luis
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
Quoted 4 lines · Luis R. Rodriguez
I see what you're saying now and I actually did consider it, the reason I didn't use --git-revision is speed. Git reset --hard tag, and then copying the code directly is a lot faster. The race issues you point out though are worth reconsidering this though.
I've given this some thought and I find that the theoretical race conditions also applies even if you use --git-revision on the output path. A true paranoid release model would then have a target output directory only the parent process and children could modify / update but given it seems we have considerations for non ext[34] filesystems (recent kconfig patch) its unclear we could resolve this for the truly paranoid. The speed considerations I raised then (10 seconds as of next-20130523 with python-git installed) could be discarded in favor of consistency of just using --git-revision for a release model provided we also address the RELMOD_UPDATE and RELMOD_TYPE specification for both daily and stable releases, defined on the upload_release() documentation. Right now we infer RELMOD_UPDATE based on the output directory but inferring RELMOD_TYPE from the same target output directory would be sloppy IMO so tackling these two with --git-revision would need to be addressed.
Luis
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
Quoted 4 lines · Luis R. Rodriguez
Right now we infer
RELMOD_UPDATEbased
on the output directory but inferringRELMOD_TYPEfrom the same target output directory would be sloppy IMO so tackling these two with --git-revision would need to be addressed.
Addressing RELMOD_TYPE seems easy with --git-revision, add arguments additional arguments: -s -n -p -c -u as release mode specifications as arguments to be passed to gentree.py either as:
a) standalone arguments, or;
b) as a conglomeration of release modification types, ie something
like --rel-mod=snpcu
We just need a reasonable way to address RELMOD_UPDATE then. My current set of patches infer the backports tag that we'd use, if we rely on --git-revision we could just have an inferred map of what the respective backports tag should be, but that still does not address the RELMOD_UPDATE to use.
If the backports tag is inferred from the linux tree used we have a few options:
1) Infer the RELMOD_UPDATE to use based on the latest backports tag,
and verify it
2) Require the RELMOD_UPDATE to be specified it and then git verify it
If we go down this road I like the first option. This however would likely require a map algorithm to be provided based on the linux tree to be used. This is fine for linux-next and linux-stable (given I merge Linus' tree on it but not sure if other folks are doing this) but lets consider that some folks might be using backports to generate releases (in theory) based on other trees like wireless-testing, let alone vendor trees (which I am not privy to) and not sure if that is a requirement.
If the backports tag is *not* inferred from the linux tree to use we could ask --backports-revision I suppose, and from that infer the RELMOD_UPDATE. I'm inclined towards this option -- but then do we use git ls-tree and git rev-parse as you do with --git-revision or do we go with git reset and git paranoia?
Do we have any use case for git paranoia then? Do we discard it? I'm still considering implementing this upstream as I find it useful to clean up my trees.
Luis
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
Quoted 7 lines · Luis R. Rodriguez
Addressing
RELMOD_TYPEseems easy with --git-revision, add arguments additional arguments: -s -n -p -c -u as release mode specifications as arguments to be passed togentree.pyeither as:a) standalone arguments, or;
b) as a conglomeration of release modification types, ie something
like --rel-mod=snpcu
I think this would fit better with the model of a separate tool, that calls the gentree code to generate internally. Then you'd have maybe
./devel/gen-and-kup.py -s -n -p -c -u ...
or something? Overall, that seems cleaner than cluttering gentree.py with all these options, although gentree.py would probably (internally only?) have to learn about being able to take patches and the skeleton code from another directory to address the paranoia for the backport tree itself.
Quoted 1 line
We just need a reasonable way to address
RELMOD_UPDATEthen.
That's when you generate a new one for the same tag, due to updates, right?
I don't really know how to handle this -- we don't really have a canonical database of what we've generated before.
johannes
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
Quoted 13 lines · Johannes Berg
Quoted 7 lines · Luis R. Rodriguez
Addressing
RELMOD_TYPEseems easy with --git-revision, add arguments additional arguments: -s -n -p -c -u as release mode specifications as arguments to be passed togentree.pyeither as:a) standalone arguments, or;
b) as a conglomeration of release modification types, ie something
like --rel-mod=snpcuI think this would fit better with the model of a separate tool, that calls the gentree code to generate internally. Then you'd have maybe
Quoted 7 lines
./devel/gen-and-kup.py-s -n -p -c -u ...or something? Overall, that seems cleaner than cluttering
gentree.pywith all these options, althoughgentree.pywould probably (internally only?) have to learn about being able to take patches and the skeleton code from another directory to address the paranoia for the backport tree itself.
I no longer have a need for the above.
Quoted 4 lines
Quoted 1 line
We just need a reasonable way to address
RELMOD_UPDATEthen.That's when you generate a new one for the same tag, due to updates, right?
Yeap.
Quoted 2 lines
I don't really know how to handle this -- we don't really have a canonical database of what we've generated before.
The releases we made before had patterns but I've decided that the additional patches mechanism in place before, while useful, made release management a pain. A better alternative for now is to ignore such additional patch mess under the new tree then and consider a different architecture for these modifications and consider this later with better thought if required.
Luis
Re: [PATCH 6/9] lib/bpgit.py: add support for git paranoia
In reply to Luis R. RodriguezQuoted 10 lines · Luis R. Rodriguez
Quoted 4 lines · Luis R. Rodriguez
I see what you're saying now and I actually did consider it, the reason I didn't use --git-revision is speed. Git reset --hard tag, and then copying the code directly is a lot faster. The race issues you point out though are worth reconsidering this though.
I've given this some thought and I find that the theoretical race conditions also applies even if you use --git-revision on the output path.
Well, to be honest, you have to trust the system anyway, otherwise it can mess up completely. So I guess the race is pretty much unimportant. It was just an additional thing to consider, more in case you start this as a long-running process and then (accidentally) work with the linux-next tree in another way.
However I'd argue that since the git object database would typically be corrupted by changes impacting --git-revision usage, it's still a lot safer than using checkouts.
Quoted 5 lines
A true paranoid release model would then have a target output directory only the parent process and children could modify / update but given it seems we have considerations for non ext[34] filesystems (recent kconfig patch) its unclear we could resolve this for the truly paranoid. The speed considerations I raised then (10 seconds as of
Quoted 8 lines
next-20130523 with python-git installed) could be discarded in favor of consistency of just using --git-revision for a release model provided we also address the
RELMOD_UPDATEandRELMOD_TYPEspecification for both daily and stable releases, defined on theupload_release()documentation. Right now we inferRELMOD_UPDATEbased on the output directory but inferringRELMOD_TYPEfrom the same target output directory would be sloppy IMO so tackling these two with --git-revision would need to be addressed.
Well like I said before, I think you should probably just write a separate script (possibly as a devel/ script) that you'd do something like
./devel/gen-and-kup.py /path/to/linux-next/ next-20130528
and then it would internally create a temporary directory (with tempdir thing) and run something like
./gentree.py --git-revision next-20130528 /path/to/linux-next/ /tmp/tmpdir/<relname>/
(obviously not calling the script but rather the function like git-tracker does)
and then work in /tmp/tmpdir/ to create the tarball etc. That way, you don't have temporary stuff lying around after the generation, and get rid of many of the changes you're making here since you wouldn't need the paranoia stuff.
johannes
PATCH 7/9 lib/bpgpg.py: add simple gpg helpers for --armor --detach-sign
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
The python gpg lib doesn't provide this, so just add our own little helper.
--- lib/bpgpg.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 lib/bpgpg.py
lib/bpgpg.py new file +12 -0
@@ -0,0 +1,12 @@ +import subprocess, os + +def sign(tar=None): + cmd = ['gpg', '--armor', '--detach-sign', tar] + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True) + stdout = process.communicate()[0] + process.wait() + if process.returncode != 0: + raise ExecutionError(process.returncode) + return stdout
Signature · 1 line
1.7.10.4
PATCH 8/9 lib/bpkup.py: add or own kernel uploader helper lib
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
kup is written in perl, add some basicpython interfaces for it to upload to kernel.org.
--- lib/bpkup.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 lib/bpkup.py
lib/bpkup.py new file +38 -0
@@ -0,0 +1,38 @@ +import subprocess, os + +def _check(process): + if process.returncode != 0: + raise ExecutionError(process.returncode) + +def mkdir(path=None): + cmd = ['kup', 'mkdir', path] + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout + +def ls(path=None): + cmd = ['kup', 'ls', path] + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout + +def put(tar_bz2=None, signed_tar=None, path=None): + cmd = ['kup', 'put', tar_bz2, signed_tar, path] + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=True, universal_newlines=True) + stdout = process.communicate()[0] + process.wait() + _check(process) + + return stdout
Signature · 1 line
1.7.10.4
Re: [PATCH 8/9] lib/bpkup.py: add or own kernel uploader helper lib
lib/bpkup.py line 30
Quoted 2 lines · Luis R. Rodriguez
lib/bpkup.py new file +2
-0+def put(tar_bz2=None, signed_tar=None, path=None): + cmd = ['kup', 'put', tar_bz2, signed_tar, path]
Without even looking, the =None here can't be right since if you don't pass any of these args, it'll just get completely confused and probably crash trying to pass None as an argument? Seems you should just require all these args.
johannes
Re: [PATCH 8/9] lib/bpkup.py: add or own kernel uploader helper lib
Quoted 9 lines · Johannes Berg
lib/bpkup.py line 30
Quoted 2 lines · Luis R. Rodriguez
lib/bpkup.py new file +2
-0+def put(tar_bz2=None, signed_tar=None, path=None): + cmd = ['kup', 'put', tar_bz2, signed_tar, path]Without even looking, the =None here can't be right since if you don't pass any of these args, it'll just get completely confused and probably crash trying to pass None as an argument? Seems you should just require all these args.
Sure, will make these changes, thanks.
Luis
PATCH 9/9 gentree.py: add kernel upload support
In reply to Luis R. RodriguezFrom: "Luis R. Rodriguez" <[email protected]>
This streamlines our release process making it easier for transitioning releases onto other backport developers.
--- gentree.py | 206 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 202 insertions(+), 4 deletions(-)
gentree.py +202 -4
@@ -4,6 +4,7 @@ # import argparse, sys, os, errno, shutil, re, subprocess +import tarfile, bz2 # find self source_dir = os.path.abspath(os.path.dirname(__file__))
@@ -11,6 +12,8 @@ sys.path.append(source_dir) # and import libraries we have from lib import kconfig, patch, make from lib import bpgit as git +from lib import bpgpg as gpg +from lib import bpkup as kup def read_copy_list(copyfile): """
@@ -250,6 +253,154 @@ def git_debug_snapshot(args, name): return git.commit_all(name, tree=args.outdir) +def get_rel_spec_stable(rel): + """ + Returns release specs for a linux-stable backports based release. + """ + if ("rc" in rel): + m = re.match(r"(?P<VERSION>\d+)\.+" \ + "(?P<PATCHLEVEL>\d+)[.]*" \ + "(?P<SUBLEVEL>\d*)" \ + "[-rc]+(?P<RC_VERSION>\d+)\-+" \ + "(?P<RELMOD_UPDATE>\d+)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + else: + m = re.match(r"(?P<VERSION>\d+)\.+" \ + "(?P<PATCHLEVEL>\d+)[.]*" \ + "(?P<SUBLEVEL>\d*)\-+" \ + "(?P<RELMOD_UPDATE>\d+)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + if (not m): + return m + return m.groupdict() + +def get_rel_spec_next(rel): + """ + Returns release specs for a linux-next backports based release. + """ + m = re.match(r"(?P<DATE_VERSION>\d+)[-]*" \ + "(?P<RELMOD_UPDATE>\d*)[-]*" \ + "(?P<RELMOD_TYPE>[usnpc]*)", \ + rel) + if (not m): + return m + return m.groupdict() + +def get_rel_prep(rel): + """ + Returns a dict with prep work details we need prior to + uploading a backports release to kernel.org + """ + rel_specs = get_rel_spec_stable(rel) + is_stable = True + rel_tag = "" + paths = list() + if (not rel_specs): + rel_specs = get_rel_spec_next(rel) + if (not rel_specs): + sys.stdout.write("rel: %s\n" % rel) + return None + if (rel_specs['RELMOD_UPDATE'] == '0' or + rel_specs['RELMOD_UPDATE'] == '1'): + return None + is_stable = False + date = rel_specs['DATE_VERSION'] + year = date[0:4] + if (len(year) != 4): + return None + month = date[4:6] + if (len(month) != 2): + return None + day = date[6:8] + if (len(day) != 2): + return None + paths.append(year) + paths.append(month) + paths.append(day) + rel_tag = "backports-" + rel.replace(rel_specs['RELMOD_TYPE'], "") + else: + ignore = "-" + if (not rel_specs['RELMOD_UPDATE']): + return None + if (rel_specs['RELMOD_UPDATE'] == '0'): + return None + ignore += rel_specs['RELMOD_UPDATE'] + if (rel_specs['RELMOD_TYPE'] != ''): + ignore += rel_specs['RELMOD_TYPE'] + base_rel = rel.replace(ignore, "") + paths.append(base_rel) + rel_tag = "v" + rel.replace(rel_specs['RELMOD_TYPE'], "") + + rel_prep = dict(stable = is_stable, + expected_tag = rel_tag, + paths_to_create = paths) + return rel_prep + +def create_tar_and_bz2(tar_name, dir_to_tar): + """ + We need both a tar file and bzip2 for kernel.org, the tar file + gets signed, then we upload the compressed version, kup-server + in the backend decompresses and verifies the tarball against + our signature. + """ + parent = os.path.dirname(tar_name) + tar = tarfile.open(tar_name, "w") + tar.add(dir_to_tar) + tar.close() + + tar_file = open(tar_name, "r") + + bz2_file = bz2.BZ2File(tar_name + ".bz2", 'wb', compresslevel=9) + bz2_file.write(tar_file.read()) + bz2_file.close() + + tar.close() + +def upload_release(args, rel_prep, logwrite=lambda x:None): + """ + Given a path of a relase make tarball out of it, PGP sign it, and + then upload it to kernel.org using kup. + + The linux-next based release do not require a RELMOD_UPDATE + given that typically only one release is made per day. Using + RELMOD_UPDATE for these releases is allowed though and if + present it must be > 1. + + The linux-stable based releases require a RELMOD_UPDATE. + + RELMOD_UPDATE must be numeric and > 0 just as the RC releases + of the Linux kernel. + + The tree must also be tagged with the respective release, without + the RELMOD_TYPE. For linux-next based releases this consists of + backports- followed by DATE_VERSION and if RELMOD_TYPE is present. + For linux-stable releases this consists of v followed by the + full release version except the RELMOD_TYPE. + + Uploads will not be allowed if these rules are not followed. + """ + korg_path = "/pub/linux/kernel/projects/backports/" + + if (rel_prep['stable']): + korg_path += "stable/" + + parent = os.path.dirname(args.outdir) + release = os.path.basename(args.outdir) + tar_name = parent + '/' + release + ".tar" + bzip2_name = tar_name + ".bz2" + + create_tar_and_bz2(tar_name, args.outdir) + + logwrite(kup.ls(korg_path)) + logwrite(gpg.sign(tar_name)) + return + + for path in rel_prep['paths_to_create']: + korg_path += path + logwrite(kup.mkdir(path)) + logwrite(kup.put(bzip2_name, tar_name + '.asc', korg_path)) def _main(): # set up and parse arguments
@@ -278,6 +429,8 @@ def _main(): help='Print more verbose information') parser.add_argument('--extra-driver', nargs=2, metavar=('<source dir>', '<copy-list>'), type=str, action='append', default=[], help='Extra driver directory/copy-list.') + parser.add_argument('--kup', const=True, default=False, action="store_const", + help='For maintainers: upload a release to kernel.org') args = parser.parse_args() def logwrite(msg):
@@ -289,16 +442,19 @@ def _main(): git_revision=args.git_revision, clean=args.clean, refresh=args.refresh, base_name=args.base_name, gitdebug=args.gitdebug, verbose=args.verbose, - extra_driver=args.extra_driver, logwrite=logwrite) + extra_driver=args.extra_driver, + kup=args.kup, + logwrite=logwrite) def process(kerneldir, outdir, copy_list_file, git_revision=None, clean=False, refresh=False, base_name="Linux", gitdebug=False, - verbose=False, extra_driver=[], logwrite=lambda x:None, + verbose=False, extra_driver=[], kup=False, + logwrite=lambda x:None, git_tracked_version=False): class Args(object): def __init__(self, kerneldir, outdir, copy_list_file, git_revision, clean, refresh, base_name, - gitdebug, verbose, extra_driver): + gitdebug, verbose, extra_driver, kup): self.kerneldir = kerneldir self.outdir = outdir self.copy_list = copy_list_file
@@ -309,10 +465,47 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, self.gitdebug = gitdebug self.verbose = verbose self.extra_driver = extra_driver + self.kup = kup + def git_paranoia(tree=None, logwrite=lambda x:None): + data = git.paranoia(tree) + if (data['r'] != 0): + logwrite('Cannot use %s' % tree) + logwrite('%s' % data['output']) + sys.exit(data['r']) + else: + logwrite('Validated tree: %s' % tree) + args = Args(kerneldir, outdir, copy_list_file, git_revision, clean, refresh, base_name, - gitdebug, verbose, extra_driver) + gitdebug, verbose, extra_driver, kup) + rel_prep = None + # start processing ... + if (args.kup): + git_paranoia(source_dir, logwrite) + git_paranoia(kerneldir, logwrite) + + rel_describe = git.describe(tree=source_dir) + release = os.path.basename(args.outdir) + version = release.replace("backports-", "") + + rel_prep = get_rel_prep(version) + if (not rel_prep): + logwrite('Invalid backports release name: %s' % release) + logwrite('For rules on the release name see upload_release()') + sys.exit(1) + rel_type = "linux-stable" + if (not rel_prep['stable']): + rel_type = "linux-next" + if (rel_prep['expected_tag'] != rel_describe): + logwrite('Unexpected %s based backports release tag on' % rel_type) + logwrite('the backports tree tree: %s\n' % rel_describe) + logwrite('You asked to make a release with this ') + logwrite('directory name: %s' % release) + logwrite('The actual expected tag we should find on') + logwrite('the backports tree then is: %s\n' % rel_prep['expected_tag']) + logwrite('For rules on the release name see upload_release()') + sys.exit(1) copy_list = read_copy_list(args.copy_list) deplist = read_dependencies(os.path.join(source_dir, 'dependencies'))
@@ -342,6 +535,8 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, # FIXME: should we add a git version of this (e.g. --git-extra-driver)? for src, copy_list in args.extra_driver: + if (args.kup): + git_paranoia(src) copy_files(src, read_copy_list(open(copy_list, 'r')), args.outdir) git_debug_snapshot(args, 'Add driver sources')
@@ -537,6 +732,9 @@ def process(kerneldir, outdir, copy_list_file, git_revision=None, fo.close() git_debug_snapshot(args, "disable unsatisfied Makefile parts") + if (args.kup): + upload_release(args, rel_prep) + logwrite('Done!') return 0
Signature · 1 line
1.7.10.4