bin/release_scripts/relnotes.sh: Add script to generate release notes template.
PATCH 1/6 bin/release_scripts/relnotes.sh: Add script to generate release notes template.
This is a capture of the command-line ickiness that we have been using to generate the basic release notes draft. It's frightfully ugly, but gets the job done for now.
--- bin/release_scripts/relnotes.sh | 82 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100755 bin/release_scripts/relnotes.sh
bin/release_scripts/relnotes.sh new file +82 -0
@@ -0,0 +1,82 @@ +#!/bin/bash + +# NOTE! This script will blissfully clobber any pre-existing poky repos, +# previous release notes with same filename, etc. +# +# This is a quick and ugly script to generate the release notes draft. +# +# Usage: ./relnotes.sh <release_ID> <rc#> <branch> <poky-ver> +# i.e. ./relnotes.sh yocto-2.0.1 rc6 jethro 14.0.1 +# +# Currently the revision in the git log portion is hard coded. You +# will want to change accordingly! +# i.e. git log --pretty=format:"%s" yocto-2.0..HEAD |grep CVE >> $HOME/CVE; +# Change the yocto-2.0 bit to whatever it needs to be for the given release. + + + +clear +REL_ID=$1 +RC=$2 +BRANCH=$3 +POKY_VER=$4 +RELEASE=$REL_ID +VHOSTS="/srv/www/vhosts" +AB_BASE="$VHOSTS/autobuilder.yoctoproject.org/pub/releases" +DL_BASE="$VHOSTS/downloads.yoctoproject.org/releases" +RC_SOURCE="$AB_BASE/$RELEASE.$RC" +RELEASE_DIR="$AB_BASE/$RELEASE" +MACHINES="$RELEASE_DIR/machines" +HOME=~/work/release +echo "REL_ID: $REL_ID" +echo "RC: $RC" +echo "BRANCH: $BRANCH" +echo "POKY_VER: $POKY_VER" +echo "RELEASE: $RELEASE" +echo "AB_BASE: $AB_BASE" +echo "DL_BASE: $DL_BASE" +echo "RC_SOURCE: $RC_SOURCE" +echo "RELEASE_DIR: $RELEASE_DIR" +echo "MACHINES: $MACHINES" +echo + +if [[ -e $HOME/RELEASENOTES.$RELEASE ]]; then + rm -v $HOME/RELEASENOTES.$RELEASE +fi +if [[ -e $HOME/CVE ]]; then + rm -v $HOME/CVE +fi +if [[ -e $HOME/FIXES ]]; then + rm -v $HOME/FIXES +fi +if [[ -d $HOME/poky ]]; then + rm -rvf $HOME/poky +fi + +for x in `ls $RELEASE_DIR/*.bz2.md5sum | grep -v "\-fsl" | grep -v "\-ppc"`; do + layer=`basename $x | sed 's/\-'"$BRANCH"'\-'"$POKY_VER"'.tar.bz2.md5sum//g'`; + tarball=`basename $x |sed 's/\.md5sum//g'`; + ghash=`ls $RELEASE_DIR/$layer*.bz2 | grep -v $BRANCH | sed 's/'"$layer"'-//g' | sed 's/.tar.bz2//g'`; + ghash=`basename $ghash` + echo "Release Name: " $layer-$BRANCH-$POKY_VER; + echo "Branch: $BRANCH"; + echo "Tag: $BRANCH-$POKY_VER"; + echo "Hash: " $ghash; + echo "md5: `cat $RELEASE_DIR/$tarball.md5sum`"; + echo "Download Locations:"; + echo "http://downloads.yoctoproject.org/releases/yocto/$RELEASE/$tarball"; + echo "http://mirrors.kernel.org/yocto/yocto/$RELEASE/$tarball"; echo ""; +done >> $HOME/RELEASENOTES.$RELEASE + +git clone git://git.yoctoproject.org/poky; +cd poky; +git checkout $BRANCH; +echo -e "\n-------\nSecurity Fixes\n-------" >> $HOME/CVE; +git log --pretty=format:"%s" yocto-2.0..HEAD |grep CVE >> $HOME/CVE; +echo -e "\n-------\nFixes\n-------" >> $HOME/FIXES; +git log --pretty=format:"%s" yocto-2.0..HEAD |grep -v CVE >> $HOME/FIXES; +cat $HOME/CVE >> $HOME/RELEASENOTES.$RELEASE; cat $HOME/FIXES >> $HOME/RELEASENOTES.$RELEASE +cd $HOME + + +
Signature · 1 line
2.4.3
PATCH 2/6 bin/release_scripts/release.py: remove stuff only used for initial testing
This is part of the release automation work, related to Yocto bug #8942. This patch removes the print_vars function and some other lines that were only used for testing and are otherwise useless.
--- bin/release_scripts/release.py | 37 ------------------------------------- 1 file changed, 37 deletions(-)
bin/release_scripts/release.py +0 -37
@@ -20,33 +20,6 @@ import shutil from shutil import rmtree, copyfile from subprocess import call -def print_vars(): - print "RELEASE: %s" %RELEASE - print "REL_TYPE: %s" %REL_TYPE - print "RC_DIR: %s" %RC_DIR - print "REL_ID: %s" %REL_ID - print "RC: %s" %RC - if MILESTONE != "": - print "Milestone: %s" %MILESTONE - if POKY_VER != "": - print "POKY_VER: %s" %POKY_VER - else: - print "POKY_VER: undefined!" - if BRANCH: - print "BRANCH: %s" %BRANCH - else: - print "BRANCH: undefined!" - - print "DL_BASE: %s" %DL_BASE - if RC_SOURCE != "": - print "RC_SOURCE: %s" %RC_SOURCE - print "RELEASE_DIR: %s" %RELEASE_DIR - print "ECLIPSE_DIR: %s" %ECLIPSE_DIR - print "PLUGIN_DIR: %s" %PLUGIN_DIR - print "DL_DIR: %s" %DL_DIR - print - return - def sanity_check(source, target): if not os.path.exists(source): print
@@ -64,7 +37,6 @@ def sanity_check(source, target): sys.exit() return - def sync_it(source, target, exclude_list): print "Syncing %s to %s" %(source, target) sanity_check(source, target)
@@ -112,7 +84,6 @@ def fix_tarballs(): print "Repackaging poky and eclipse tarballs...." os.chdir(RELEASE_DIR) os.mkdir(TARBALL_DIR) - #os.mkdir("%s" %TARBALL_DIR) os.system("mv %s/*.tar.bz2 %s" %(RELEASE_DIR, TARBALL_DIR)) os.system("rm *.md5sum") os.chdir(TARBALL_DIR)
@@ -352,12 +323,6 @@ if __name__ == '__main__': os.system("clear") print - # This is for testing convenience - #HOME_BASE = "/home/tgraydon/work/release" - #AB_BASE = HOME_BASE - #DL_BASE = os.path.join(HOME_BASE, "downloads") - - # This is the legit set of vars used for production release VHOSTS = "/srv/www/vhosts" AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases") DL_BASE = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases")
@@ -439,8 +404,6 @@ if __name__ == '__main__': BUILD_APP_DIR = os.path.join(RELEASE_DIR, "build-appliance") REL_MD5_FILE = RELEASE + ".md5sum" - print_vars() - # For all releases: # 1) Rsync the rc candidate to a staging dir where all work happens sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED)
Signature · 1 line
2.4.3
PATCH 3/6 bin/release_scripts/release.py: add a check for valid rc candidate
In reply to Graydon, TracyThis is part of the release automation work, related to Yocto bug #8942. This patch adds a check for a valid release candidate ID passed in from command line and quits if it is bogus. Adds a few comments/output for the user for the various release steps to improve clarity.
--- bin/release_scripts/release.py | 53 +++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 14 deletions(-)
bin/release_scripts/release.py +39 -14
@@ -360,19 +360,23 @@ if __name__ == '__main__': else: BRANCH = "" - if options.build: # Figure out the release name, type of release, and generate some vars, do some basic validation - chunks = split_thing(options.build, ".") + options.build = options.build.lower() + RC = split_thing(options.build, ".")[-1] + chunks = split_thing(options.build, ".") # i.e. split yocto-2.1_m1.rc1 chunks.pop() - RELEASE = rejoin_thing(chunks, ".") - rel_thing = split_thing(options.build, "-") - RC = split_thing(options.build, ".")[-1].lower() - RC_DIR = RELEASE + "." + RC - REL_ID = split_thing(RELEASE, "-")[-1] - milestone = split_thing(REL_ID, "_") - if len(milestone) == 1: - thing = split_thing(milestone[0], ".") + chunks[1] = chunks[1].upper() + RELEASE = rejoin_thing(chunks, ".") # i.e. yocto-2.1_m1 + REL_ID = split_thing(RELEASE, "-")[-1].upper() + RC_DIR = rejoin_thing([RELEASE, RC], ".") + RC_SOURCE = os.path.join(AB_BASE, RC_DIR) + if not os.path.exists(RC_SOURCE): + print "%s does not appear to be a valid RC dir. Check your args." %RC_SOURCE + sys.exit() + relstring = split_thing(REL_ID, "_") + if len(relstring) == 1: + thing = split_thing(relstring[0], ".") if len(thing) == 3: REL_TYPE = "point" elif len(thing) == 2:
@@ -385,14 +389,27 @@ if __name__ == '__main__': print "Please use -h or --help for options." sys.exit() else: - MILESTONE = milestone.pop() + MILESTONE = relstring.pop() REL_TYPE = "milestone" else: print "Build ID is a required argument." print "Please use -h or --help for options." sys.exit() - - RC_SOURCE = os.path.join(AB_BASE, RC_DIR) + + if not (RELEASE and RC and REL_ID and REL_TYPE): + print "Can't determine the release type. Check your args." + print "You gave me: %s" %options.build + sys.exit() + + print "RC_DIR: %s" %RC_DIR + print "RELEASE: %s" %RELEASE + print "RC: %s" %RC + print "REL_ID: %s" %REL_ID + print "REL_TYPE: %s" %REL_TYPE + if MILESTONE: + print "MILESTONE: %s" %MILESTONE + print + PLUGIN_DIR = os.path.join(DL_BASE, "eclipse-plugin", REL_ID) RELEASE_DIR = os.path.join(AB_BASE, RELEASE) DL_DIR = os.path.join(DL_BASE, RELEASE)
@@ -403,36 +420,44 @@ if __name__ == '__main__': ECLIPSE_DIR = os.path.join(RELEASE_DIR, "eclipse-plugin") BUILD_APP_DIR = os.path.join(RELEASE_DIR, "build-appliance") REL_MD5_FILE = RELEASE + ".md5sum" - + # For all releases: # 1) Rsync the rc candidate to a staging dir where all work happens sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED) # 2) Convert the symlinks in build-appliance dir. + print "Converting the build-appliance symlink." convert_symlinks(BUILD_APP_DIR) # 3) In machines dir, convert the symlinks, delete the cruft + print "Cleaning up the machines dirs, converting symlinks." dirlist = get_list(MACHINES) for dirname in dirlist: dirname = os.path.join(MACHINES, dirname) convert_symlinks(dirname) nuke_cruft(dirname, CRUFT_LIST) + print "Generating fresh md5sums." gen_md5sum(MACHINES) # For major and point releases if REL_TYPE == "major" or REL_TYPE == "point": # 4) Fix up the eclipse and poky tarballs + print "Cleaning up the eclipse, poky and other tarballs." fix_tarballs() # 5) Publish the eclipse stuff + print "Publishing the eclipse plugins." pub_eclipse(ECLIPSE_DIR, PLUGIN_DIR) # 6) Make the bsps + print "Generating the BSP tarballs." make_bsps(BSP_LIST, BSP_DIR) # 7) Generate the master md5sum file for the release (for all releases) + print "Generating the master md5sum table." gen_rel_md5(RELEASE_DIR, REL_MD5_FILE) # 8) sync to downloads + print "Publishing release to downloads." sync_it(RELEASE_DIR, DL_DIR, "")
Signature · 1 line
2.4.3
PATCH 4/6 bin/release_scripts/release.py: add ADT publishing feature to release script
In reply to Graydon, TracyThis is part of the release automation work, related to Yocto bug #8942. This patch adds the ADT repo publishing functionality for backward compatability for point releases, etc. Not used going forward since we have moved to the eSDK instead, so this defaults to NOT publishing the ADT repo. A flag has been added to cue publishing, along with an option to provide a custom target direactory name for oddball cases where release numbering may not be linear. i.e There was no 1.9 release, so ADT repo for 2.0_M1 would have actually been 1.8+snapshot. This allows for those corner cases.
--- bin/release_scripts/release.py | 58 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 4 deletions(-)
bin/release_scripts/release.py +54 -4
@@ -317,6 +317,37 @@ def gen_rel_md5(dirname, md5_file): f.close() return +def publish_adt(rel_id, rel_type, opts): + if opts: + ADT_DIR = os.path.join(ADT_BASE, opts) + else: + if rel_type == "milestone": + chunks = split_thing(rel_id, "_") + id_thing = float(chunks[0]) + id_thing = id_thing - 0.1 + rel_id = str(id_thing) + "+" + "snapshot" + ADT_DIR = os.path.join(ADT_BASE, rel_id) + print "ADT_DIR: %s" %ADT_DIR + if os.path.exists(ADT_DIR): + print "ADT_DIR %s EXISTS! Refusing to clobber!" %ADT_DIR + sys.exit() + else: + ADT_ROOTFS = os.path.join(ADT_DIR, "rootfs") + ADT_IPK = os.path.join(ADT_DIR, "adt-ipk") + QEMU_DIR = os.path.join(MACHINES, "qemu") + IPK_DIR = os.path.join(RELEASE_DIR, "ipk") + os.mkdir(ADT_DIR) + os.mkdir(ADT_ROOTFS) + dirlist = get_list(QEMU_DIR) + + for dirname in dirlist: + QEMU_SRC = os.path.join(QEMU_DIR, dirname) + QEMU_TARGET = os.path.join(ADT_ROOTFS, dirname) + print "QEMU_SRC: %s" %QEMU_SRC + sync_it(QEMU_SRC, QEMU_TARGET, "") + + sync_it(IPK_DIR, ADT_IPK, "") + return if __name__ == '__main__':
@@ -326,6 +357,7 @@ if __name__ == '__main__': VHOSTS = "/srv/www/vhosts" AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases") DL_BASE = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") + ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") # List of the directories we delete from all releases UNLOVED = ['rpm', 'deb', 'ptest', 'adt-installer-QA']
@@ -346,7 +378,12 @@ if __name__ == '__main__': parser.add_option("-p", "--poky-ver", type="string", dest="poky", help="Required for Major and Point releases. i.e. 14.0.0") - + parser.add_option("-a", action="store_true", dest="pub_adt", + help="Publish an ADT repo for the release. Default is NOT to publish.") + parser.add_option("-d", "--adt-dir", + type="string", dest="adt_dir", + help="Use when you need to publish the ADT repo to a custom location. i.e. python adtcopy -b yocto-2.0_M1.rc1 -a 1.8+snaphot") + (options, args) = parser.parse_args() REL_TYPE = ""
@@ -395,12 +432,12 @@ if __name__ == '__main__': print "Build ID is a required argument." print "Please use -h or --help for options." sys.exit() - + if not (RELEASE and RC and REL_ID and REL_TYPE): print "Can't determine the release type. Check your args." print "You gave me: %s" %options.build sys.exit() - + print "RC_DIR: %s" %RC_DIR print "RELEASE: %s" %RELEASE print "RC: %s" %RC
@@ -420,7 +457,7 @@ if __name__ == '__main__': ECLIPSE_DIR = os.path.join(RELEASE_DIR, "eclipse-plugin") BUILD_APP_DIR = os.path.join(RELEASE_DIR, "build-appliance") REL_MD5_FILE = RELEASE + ".md5sum" - + # For all releases: # 1) Rsync the rc candidate to a staging dir where all work happens sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED)
@@ -461,3 +498,16 @@ if __name__ == '__main__': print "Publishing release to downloads." sync_it(RELEASE_DIR, DL_DIR, "") + # 9) Publish the ADT repo. The default is NOT to publish the ADT. The ADT + # is deprecated as of 2.1_M1. However, we need to retain backward + # compatability for point releases, etc. We do this step after all the other + # stuff because we want the symlinks to have been converted, extraneous + # files deleted, and md5sums generated. + # + if options.pub_adt: + if options.adt_dir: + print "Publishing the ADT repo using custom dir %s" %options.adt_dir + publish_adt(REL_ID, REL_TYPE, options.adt_dir) + else: + print "Publishing ADT repo." + publish_adt(REL_ID, REL_TYPE, "")
Signature · 1 line
2.4.3
PATCH 5/6 bin/release_scripts/release.py: fix the tarball naming
In reply to Graydon, TracyFixed tarball repackaging to add missing branch name to the new tarball name. i.e. <blob>-<hash>.tar.bz2 was being renamed to <blob>-<poky-ver>.14.0.1.tar.bz2, which is wrong. Should be renamed to <blob>-<branch>-<poky-ver>.tar.bz2.
--- bin/release_scripts/release.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
bin/release_scripts/release.py +2 -1
@@ -94,7 +94,7 @@ def fix_tarballs(): filename = chunks[0] basename = split_thing(filename, "-") index = len(basename)-1 - basename[index] = POKY_VER + basename[index] = "-".join([BRANCH, POKY_VER]) new_name = rejoin_thing(basename, "-") chunks[0] = new_name new_blob = rejoin_thing(chunks, ".")
@@ -511,3 +511,4 @@ if __name__ == '__main__': else: print "Publishing ADT repo." publish_adt(REL_ID, REL_TYPE, "") +
Signature · 1 line
2.4.3
PATCH 6/6 bin/release_scripts/release.py: fix the BSP tarball naming
In reply to Graydon, TracyThe BSP packaging was missing the branch name from the repackaged tarball. The md5sums were also not being generated for the BSP tarballs. This patch fixes both these issues.
--- bin/release_scripts/release.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-)
bin/release_scripts/release.py +13 -10
@@ -200,21 +200,23 @@ def make_bsps(bsp_list, bsp_dir): print "Creating %s bsp dir" %dirname os.mkdir(dirname) target = os.path.join(dirname, POKY_TARBALL) - copyfile(poky_blob, target) - os.chdir(dirname) - print "Unpacking poky tarball." - os.system("tar jxf %s" %POKY_TARBALL) oldblob = POKY_TARBALL chunks = split_thing(oldblob, "-") chunks[0] = dirname new_blob = rejoin_thing(chunks, "-") + print "BSP tarball: %s" %new_blob new_dir = split_thing(blob_dir, "-") new_dir[0] = dirname new_dir = rejoin_thing(new_dir, "-") bin_dir = os.path.join(new_dir, "binary") + copyfile(poky_blob, target) + os.chdir(dirname) + print "Unpacking poky tarball." + os.system("tar jxf %s" %POKY_TARBALL) shutil.move(blob_dir, new_dir) os.remove(POKY_TARBALL) - os.mkdir(bin_dir) + if not os.path.exists(bin_dir): + os.mkdir(bin_dir) print "Getting binary files" os.system("rsync -arl %s/%s/ %s" %(MACHINES, dirname, bin_dir)) bsp_bin = os.path.join(bsp_dir, dirname, bin_dir)
@@ -224,8 +226,10 @@ def make_bsps(bsp_list, bsp_dir): print "Creating BSP tarball" os.system("tar jcf %s %s" %(new_blob, new_dir)) rmtree(new_dir) - print "Copying %s BSP tarball to machines dir" %new_blob - shutil.copy(new_blob, platform_dir) + print "Generating the md5sum." + os.system("md5sum %s > %s.md5sum" %(new_blob, new_blob)) + print "Copying %s BSP to platform dir" %dirname + os.system("mv * %s" %platform_dir) os.chdir(bsp_dir) print os.chdir(RELEASE_DIR)
@@ -356,7 +360,7 @@ if __name__ == '__main__': VHOSTS = "/srv/www/vhosts" AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases") - DL_BASE = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases") + DL_BASE = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases/yocto") ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org") # List of the directories we delete from all releases
@@ -453,7 +457,7 @@ if __name__ == '__main__': MACHINES = os.path.join(RELEASE_DIR, "machines") BSP_DIR = os.path.join(RELEASE_DIR, 'bsptarballs') TARBALL_DIR = os.path.join(RELEASE_DIR, "tarballs") - POKY_TARBALL = "poky-" + POKY_VER + ".tar.bz2" + POKY_TARBALL = "poky-" + BRANCH + "-" + POKY_VER + ".tar.bz2" ECLIPSE_DIR = os.path.join(RELEASE_DIR, "eclipse-plugin") BUILD_APP_DIR = os.path.join(RELEASE_DIR, "build-appliance") REL_MD5_FILE = RELEASE + ".md5sum"
@@ -511,4 +515,3 @@ if __name__ == '__main__': else: print "Publishing ADT repo." publish_adt(REL_ID, REL_TYPE, "") -
Signature · 1 line
2.4.3