Skip to contentSkip to conversation

bin/release_scripts/release.py

PATCH New Graydon, Tracy 1 participant 3 messages Feb 18, 2016

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 bin/release_scripts/release.py

From: Tracy Graydon <[email protected]>

This is part of the release automation work, related to Yocto bug #8942. This patch removes the print_vars function and some other lines that were only used for testing and are otherwise useless.

---
 bin/release_scripts/release.py | 37 -------------------------------------
 1 file changed, 37 deletions(-)
bin/release_scripts/release.py +0 -37
index 27ee409..3dab369 100755
@@ -20,33 +20,6 @@ import shutil
 from shutil import rmtree, copyfile
 from subprocess import call
 
-def print_vars():
-    print "RELEASE: %s" %RELEASE
-    print "REL_TYPE: %s" %REL_TYPE
-    print "RC_DIR: %s" %RC_DIR
-    print "REL_ID: %s" %REL_ID
-    print "RC: %s" %RC
-    if MILESTONE != "":
-        print "Milestone: %s" %MILESTONE
-    if POKY_VER != "":
-        print "POKY_VER: %s" %POKY_VER
-    else:
-       print "POKY_VER: undefined!"
-    if BRANCH:
-        print "BRANCH: %s" %BRANCH
-    else:
-        print "BRANCH: undefined!"
-
-    print "DL_BASE: %s" %DL_BASE
-    if RC_SOURCE != "":
-        print "RC_SOURCE: %s" %RC_SOURCE
-    print "RELEASE_DIR: %s" %RELEASE_DIR
-    print "ECLIPSE_DIR: %s" %ECLIPSE_DIR
-    print "PLUGIN_DIR: %s" %PLUGIN_DIR
-    print "DL_DIR: %s" %DL_DIR
-    print
-    return
-
 def sanity_check(source, target):
     if not os.path.exists(source):
        print
@@ -64,7 +37,6 @@ def sanity_check(source, target):
        sys.exit()
     return
 
-
 def sync_it(source, target, exclude_list):
     print "Syncing %s to %s" %(source, target)
     sanity_check(source, target)
@@ -112,7 +84,6 @@ def fix_tarballs():
     print "Repackaging poky and eclipse tarballs...."
     os.chdir(RELEASE_DIR)
     os.mkdir(TARBALL_DIR)
-    #os.mkdir("%s" %TARBALL_DIR)
     os.system("mv %s/*.tar.bz2 %s" %(RELEASE_DIR, TARBALL_DIR))
     os.system("rm *.md5sum")
     os.chdir(TARBALL_DIR)
@@ -352,12 +323,6 @@ if __name__ == '__main__':
     os.system("clear")
     print
    
-    # This is for testing convenience
-    #HOME_BASE = "/home/tgraydon/work/release"
-    #AB_BASE = HOME_BASE
-    #DL_BASE = os.path.join(HOME_BASE, "downloads")
-
-    # This is the legit set of vars used for production release
     VHOSTS = "/srv/www/vhosts"
     AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases")
     DL_BASE = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases")
@@ -439,8 +404,6 @@ if __name__ == '__main__':
     BUILD_APP_DIR = os.path.join(RELEASE_DIR, "build-appliance")
     REL_MD5_FILE = RELEASE + ".md5sum"
  
-    print_vars()
-
     # For all releases:
     # 1) Rsync the rc candidate to a staging dir where all work happens
     sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED)
Signature · 1 line
2.4.3
#1 lore raw

PATCH 2/3 bin/release_scripts/release.py

From: Tracy Graydon <[email protected]>

This is part of the release automation work, related to Yocto bug #8942. This patch adds a check for a valid release candidate ID passed in from command line and quits if it is bogus. Adds a few comments/output for the user for the various release steps to improve clarity.

---
 bin/release_scripts/release.py | 53 +++++++++++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 14 deletions(-)
bin/release_scripts/release.py +39 -14
index 3dab369..c332718 100755
@@ -360,19 +360,23 @@ if __name__ == '__main__':
     else:
         BRANCH = ""
 
-
     if options.build:
         # Figure out the release name, type of release, and generate some vars, do some basic validation
-        chunks = split_thing(options.build, ".")
+        options.build = options.build.lower()
+        RC = split_thing(options.build, ".")[-1]
+        chunks = split_thing(options.build, ".") # i.e. split yocto-2.1_m1.rc1
         chunks.pop()
-        RELEASE = rejoin_thing(chunks, ".")
-        rel_thing = split_thing(options.build, "-")
-        RC = split_thing(options.build, ".")[-1].lower()
-        RC_DIR = RELEASE + "." + RC
-        REL_ID = split_thing(RELEASE, "-")[-1]
-        milestone = split_thing(REL_ID, "_")
-        if len(milestone) == 1:
-            thing = split_thing(milestone[0], ".")
+        chunks[1] = chunks[1].upper()
+        RELEASE = rejoin_thing(chunks, ".")  # i.e. yocto-2.1_m1
+        REL_ID = split_thing(RELEASE, "-")[-1].upper()
+        RC_DIR = rejoin_thing([RELEASE, RC], ".")
+        RC_SOURCE = os.path.join(AB_BASE, RC_DIR)
+        if not os.path.exists(RC_SOURCE):
+            print "%s does not appear to be a valid RC dir. Check your args." %RC_SOURCE
+            sys.exit()
+        relstring = split_thing(REL_ID, "_")
+        if len(relstring) == 1:
+            thing = split_thing(relstring[0], ".")
             if len(thing) == 3:
                 REL_TYPE = "point"
             elif len(thing) == 2:
@@ -385,14 +389,27 @@ if __name__ == '__main__':
                 print "Please use -h or --help for options."
                 sys.exit()
         else:
-            MILESTONE = milestone.pop()
+            MILESTONE = relstring.pop()
             REL_TYPE = "milestone"
     else:
         print "Build ID is a required argument."
         print "Please use -h or --help for options."
         sys.exit()
-    
-    RC_SOURCE = os.path.join(AB_BASE, RC_DIR)
+
+    if not (RELEASE and RC and REL_ID and REL_TYPE):
+        print "Can't determine the release type. Check your args."
+        print "You gave me: %s" %options.build
+        sys.exit()
+
+    print "RC_DIR: %s" %RC_DIR
+    print "RELEASE: %s" %RELEASE
+    print "RC: %s" %RC
+    print "REL_ID: %s" %REL_ID
+    print "REL_TYPE: %s" %REL_TYPE
+    if MILESTONE:
+        print "MILESTONE: %s" %MILESTONE
+    print
+
     PLUGIN_DIR = os.path.join(DL_BASE, "eclipse-plugin", REL_ID)
     RELEASE_DIR = os.path.join(AB_BASE, RELEASE)
     DL_DIR = os.path.join(DL_BASE, RELEASE)
@@ -403,36 +420,44 @@ if __name__ == '__main__':
     ECLIPSE_DIR = os.path.join(RELEASE_DIR, "eclipse-plugin")
     BUILD_APP_DIR = os.path.join(RELEASE_DIR, "build-appliance")
     REL_MD5_FILE = RELEASE + ".md5sum"
- 
+       
     # For all releases:
     # 1) Rsync the rc candidate to a staging dir where all work happens
     sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED)
     
     # 2) Convert the symlinks in build-appliance dir.
+    print "Converting the build-appliance symlink."
     convert_symlinks(BUILD_APP_DIR)
 
     # 3) In machines dir, convert the symlinks, delete the cruft
+    print "Cleaning up the machines dirs, converting symlinks."
     dirlist = get_list(MACHINES)
     for dirname in dirlist:
         dirname = os.path.join(MACHINES, dirname)
         convert_symlinks(dirname)
         nuke_cruft(dirname, CRUFT_LIST)
+    print "Generating fresh md5sums."
     gen_md5sum(MACHINES)
     
     # For major and point releases
     if REL_TYPE == "major" or REL_TYPE == "point":
         # 4) Fix up the eclipse and poky tarballs
+        print "Cleaning up the eclipse, poky and other tarballs."
         fix_tarballs()
 
         # 5) Publish the eclipse stuff
+        print "Publishing the eclipse plugins."
         pub_eclipse(ECLIPSE_DIR, PLUGIN_DIR)
 
         # 6) Make the bsps
+        print "Generating the BSP tarballs."
         make_bsps(BSP_LIST, BSP_DIR)
 
     # 7) Generate the master md5sum file for the release (for all releases)
+    print "Generating the master md5sum table."
     gen_rel_md5(RELEASE_DIR, REL_MD5_FILE)
     
     # 8) sync to downloads
+    print "Publishing release to downloads."
     sync_it(RELEASE_DIR, DL_DIR, "")
 
Signature · 1 line
2.4.3
#2 lore raw

PATCH 3/3 bin/release_scripts/release.py

In reply to Graydon, Tracy

From: Tracy Graydon <[email protected]>

This is part of the release automation work, related to Yocto bug #8942. This patch adds the ADT repo publishing functionality for backward compatability for point releases, etc. Not used going forward since we have moved to the eSDK instead, so this defaults to NOT publishing the ADT repo. A flag has been added to cue publishing, along with an option to provide a custom target direactory name for oddball cases where release numbering may not be linear. i.e There was no 1.9 release, so ADT repo for 2.0_M1 would have actually been 1.8+snapshot. This allows for those corner cases.

---
 bin/release_scripts/release.py | 58 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 54 insertions(+), 4 deletions(-)
bin/release_scripts/release.py +54 -4
index c332718..83c198e 100755
@@ -317,6 +317,37 @@ def gen_rel_md5(dirname, md5_file):
     f.close()
     return
 
+def publish_adt(rel_id, rel_type, opts):
+    if opts:
+        ADT_DIR = os.path.join(ADT_BASE, opts)
+    else:
+        if rel_type == "milestone":
+            chunks = split_thing(rel_id, "_")
+            id_thing = float(chunks[0])
+            id_thing = id_thing - 0.1
+            rel_id = str(id_thing) + "+" + "snapshot"
+        ADT_DIR = os.path.join(ADT_BASE, rel_id)
+    print "ADT_DIR: %s" %ADT_DIR
+    if os.path.exists(ADT_DIR):
+        print "ADT_DIR %s EXISTS! Refusing to clobber!" %ADT_DIR
+        sys.exit()
+    else:
+        ADT_ROOTFS = os.path.join(ADT_DIR, "rootfs")
+        ADT_IPK = os.path.join(ADT_DIR, "adt-ipk")
+        QEMU_DIR = os.path.join(MACHINES, "qemu")
+        IPK_DIR = os.path.join(RELEASE_DIR, "ipk")
+        os.mkdir(ADT_DIR)
+        os.mkdir(ADT_ROOTFS)
+        dirlist = get_list(QEMU_DIR)
+
+        for dirname in dirlist:
+            QEMU_SRC = os.path.join(QEMU_DIR, dirname)
+            QEMU_TARGET = os.path.join(ADT_ROOTFS, dirname)
+            print "QEMU_SRC: %s" %QEMU_SRC
+            sync_it(QEMU_SRC, QEMU_TARGET, "")
+        
+        sync_it(IPK_DIR, ADT_IPK, "")
+    return
 
 if __name__ == '__main__':
     
@@ -326,6 +357,7 @@ if __name__ == '__main__':
     VHOSTS = "/srv/www/vhosts"
     AB_BASE = os.path.join(VHOSTS, "autobuilder.yoctoproject.org/pub/releases")
     DL_BASE = os.path.join(VHOSTS, "downloads.yoctoproject.org/releases")
+    ADT_BASE = os.path.join(VHOSTS, "adtrepo.yoctoproject.org")
 
     # List of the directories we delete from all releases
     UNLOVED = ['rpm', 'deb', 'ptest', 'adt-installer-QA']
@@ -346,7 +378,12 @@ if __name__ == '__main__':
     parser.add_option("-p", "--poky-ver",
                       type="string", dest="poky",
                       help="Required for Major and Point releases. i.e. 14.0.0")
-                     
+    parser.add_option("-a", action="store_true", dest="pub_adt",
+                      help="Publish an ADT repo for the release. Default is NOT to publish.")
+    parser.add_option("-d", "--adt-dir",
+                      type="string", dest="adt_dir",
+                      help="Use when you need to publish the ADT repo to a custom location. i.e. python adtcopy -b yocto-2.0_M1.rc1 -a 1.8+snaphot")
+
     (options, args) = parser.parse_args()
  
     REL_TYPE = ""
@@ -395,12 +432,12 @@ if __name__ == '__main__':
         print "Build ID is a required argument."
         print "Please use -h or --help for options."
         sys.exit()
-
+   
     if not (RELEASE and RC and REL_ID and REL_TYPE):
         print "Can't determine the release type. Check your args."
         print "You gave me: %s" %options.build
         sys.exit()
-
+    
     print "RC_DIR: %s" %RC_DIR
     print "RELEASE: %s" %RELEASE
     print "RC: %s" %RC
@@ -420,7 +457,7 @@ if __name__ == '__main__':
     ECLIPSE_DIR = os.path.join(RELEASE_DIR, "eclipse-plugin")
     BUILD_APP_DIR = os.path.join(RELEASE_DIR, "build-appliance")
     REL_MD5_FILE = RELEASE + ".md5sum"
-       
+
     # For all releases:
     # 1) Rsync the rc candidate to a staging dir where all work happens
     sync_it(RC_SOURCE, RELEASE_DIR, UNLOVED)
@@ -461,3 +498,16 @@ if __name__ == '__main__':
     print "Publishing release to downloads."
     sync_it(RELEASE_DIR, DL_DIR, "")
 
+    # 9) Publish the ADT repo. The default is NOT to publish the ADT. The ADT 
+    # is deprecated as of 2.1_M1. However, we need to retain backward 
+    # compatability for point releases, etc. We do this step after all the other
+    #  stuff because we want the symlinks to have been converted, extraneous 
+    # files deleted, and md5sums generated.
+    #
+    if options.pub_adt:
+        if options.adt_dir:
+            print "Publishing the ADT repo using custom dir %s" %options.adt_dir
+            publish_adt(REL_ID, REL_TYPE, options.adt_dir)
+        else:
+            print "Publishing ADT repo."
+            publish_adt(REL_ID, REL_TYPE, "")
Signature · 1 line
2.4.3