ProvisionGoogleVM.py: Initial commit
PATCH 1/7 ProvisionGoogleVM.py: Initial commit
From: California Sullivan <[email protected]>
This build step provisions a Google VM using the gcloud command line tool.
--- .../autobuilder/buildsteps/ProvisionGoogleVM.py | 85 ++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py new file +85 -0
@@ -0,0 +1,85 @@ +''' +Created on Aug 13, 2014 + +__author__ = "California Sullivan" +__copyright__ = "Copyright 2014, Intel Corp." +__credits__ = ["California Sullivan"] +__license__ = "GPL" +__version__ = "2.0" +__maintainer__ = "Elizabeth Flanagan" +__email__ = "[email protected]" +''' + +from buildbot.steps.shell import ShellCommand + +class ProvisionGoogleVM(ShellCommand): + haltOnFailure = True + flunkOnFailure = True + name = "Provision Google VM" + def __init__(self, factory, argdict=None, **kwargs): + self.vmname=None + self.vmcount=1 + self.startupscript=None + self.metadata=None + self.zone="us-central1-a" + self.image=None + self.machine=None + self.network=None + self.disksize=None + self.factory = factory + self.description = "Provision Google VM" + for k, v in argdict.iteritems(): + if type(v) is bool: + setattr(self, k, str(v)) + else: + setattr(self, k, v) + + if self.vmname is None or self.vmname == "": + self.finished(FAILURE) + else: + self.command = "gcloud compute instances create" + self.vmnames = "" + for x in range(0, self.vmcount): + self.vmnames += " " + self.vmname + "-" + str(x) + self.command += self.vmnames + if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]: + self.command += " --zone " + self.zone + else: + self.command += " --zone us-central1-a" + if self.disksize is not None and self.disksize != "": + self.command += " --boot-disk-size " + self.disksize + else: + self.command += " --boot-disk-size 200GB" + if self.image is None: + self.command += " --image debian-7" + else: + self.command += " --image " + self.image + + if self.machine is not None and self.machine in ["g1-small", "f1-micro", "n1-standard-1", "n1-standard-2", "n1-standard-4", "n1-standard-8", "n1-standard-16", "n1-highmem-2", "n1-highmem-4"]: + self.command += " --machine-type " + self.machine + else: + self.command += " --machine-type n1-standard-1" + if self.network is not None and self.network != "": + self.command += " --network " + self.network + if self.startupscript is not None and self.startupscript != "": + self.command += " --metadata-from-file startup-script=" + self.startupscript + if self.metadata is not None and self.metadata != "": + self.command += " --metadata " + self.scriptargs + self.command += " 1> /dev/null"; + + # Timeout needs to be passed to LoggingBuildStep as a kwarg + self.timeout = 1000 + kwargs['timeout']=self.timeout + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + description = ShellCommand.describe(self,done) + return description + + def commandComplete(self, cmd): + if not cmd.didFail(): + self.setProperty("vmnames", self.vmnames) + self.setProperty("zone", self.zone) + self.setProperty("vmcount", self.vmcount) + +
Signature · 1 line
1.9.3
PATCH 2/7 DeleteGoogleVMs.py: Initial commit
From: California Sullivan <[email protected]>
This build step deletes the Google VMs created in the previous ProvisionGoogleVM build step using Google's gcloud command line tool.
--- .../autobuilder/buildsteps/DeleteGoogleVMs.py | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/DeleteGoogleVMs.py
lib/python2.7/site-packages/autobuilder/buildsteps/DeleteGoogleVMs.py new file +52 -0
@@ -0,0 +1,52 @@ +''' +Created on Aug 13, 2014 + +__author__ = "California Sullivan" +__copyright__ = "Copyright 2014, Intel Corp." +__credits__ = ["California Sullivan"] +__license__ = "GPL" +__version__ = "2.0" +__maintainer__ = "Elizabeth Flanagan" +__email__ = "[email protected]" +''' + +from buildbot.steps.shell import ShellCommand + +class DeleteGoogleVMs(ShellCommand): + haltOnFailure = True + flunkOnFailure = True + name = "Delete Google VMs" + def __init__(self, factory, argdict=None, **kwargs): + + self.factory = factory + self.description = "Deleting Google VMs" + for k, v in argdict.iteritems(): + if type(v) is bool: + setattr(self, k, str(v)) + else: + setattr(self, k, v) + # Timeout needs to be passed to LoggingBuildStep as a kwarg + self.timeout = 1000 + kwargs['timeout']=self.timeout + ShellCommand.__init__(self, **kwargs) + + + def start(self): + self.vmnames = self.getProperty("vmnames") + self.vmcount = self.getProperty("vmcount") + self.zone = self.getProperty("zone") + + if self.vmnames is not None and self.vmnames != "": + self.command = "gcloud compute instances delete" + self.command += " " + self.vmnames + + if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]: + self.command += " --zone " + self.zone + else: + self.command += " --zone us-central1-a" + self.command += " --quiet" + ShellCommand.start(self) + + def describe(self, done=False): + description = ShellCommand.describe(self,done) + return description
Signature · 1 line
1.9.3
PATCH 3/7 bin/worker-init: Initial commit
In reply to [email protected]From: California Sullivan <[email protected]>
This script takes a fresh debian VM, completely sets up software and disks, starts an autobuilder worker, and then connects it to the controller. It is to be used by the ProvisionGoogleVMs build step.
--- bin/worker-init | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 bin/worker-init
bin/worker-init new file +39 -0
@@ -0,0 +1,39 @@ +#!/bin/bash + +if [ ! -f /var/log/firstboot.log ] +then + START_SECTOR=$(fdisk -l | grep sda1 | sed 's/\*//' | awk '{ print $2 }') + echo -e "d\nn\n\n\n$START_SECTOR\n\nw\n" | fdisk /dev/sda + touch /var/log/firstboot.log + reboot +fi + +if [ ! -f /var/log/secondboot.log ] +then + resize2fs /dev/sda1 + sudo apt-get update + yes | sudo apt-get install make + yes | sudo apt-get install git + yes | sudo apt-get install gcc + yes | sudo apt-get install g++ + yes | sudo apt-get install diffstat + yes | sudo apt-get install texinfo + yes | sudo apt-get install bzip2 + yes | sudo apt-get install gawk + yes | sudo apt-get install chrpath + yes | sudo apt-get install libsdl1.2-dev + yes "dummyinfo" | adduser autobuilder + touch /var/log/secondboot.log +fi + +WORKER_NUM=$(uname -n | grep -o [0-9]) + +cd /home/autobuilder +su autobuilder -c "yes 'yes' | git clone git://git.yoctoproject.org/yocto-autobuilder" + +cd yocto-autobuilder +su autobuilder -c '. ./yocto-autobuilder-setup' +su autobuilder -c "sed -i 's/example-worker/worker'$WORKER_NUM'/' yocto-worker/buildbot.tac" +su autobuilder -c 'sed -i "s/passwd =.*/passwd = '"'"'<PASS>'"'"'/" yocto-worker/buildbot.tac' +su autobuilder -c 'sed -i "s/buildmaster_host =.*/buildmaster_host = '"'"'<HOST_ADDR>'"'"'/" yocto-worker/buildbot.tac' +su autobuilder -c '. ./yocto-autobuilder-setup; ./yocto-start-autobuilder worker &'
Signature · 1 line
1.9.3
PATCH 4/7 yocto-autobuilder-setup: Add worker init script setup
In reply to [email protected]From: California Sullivan <[email protected]>
The password and address fields in the worker init script need to be set up.
--- yocto-autobuilder-setup | 5 +++++ 1 file changed, 5 insertions(+)
yocto-autobuilder-setup +5 -0
@@ -60,6 +60,7 @@ export OPTIMIZED_GIT_CLONE=False export OGIT_TRASH_DIR=/tmp/yocto-autobuilder/git/trash export OGIT_TRASH_CRON_TIME="0 0 * * *" export OGIT_TRASH_NICE_LEVEL="19" +HOST_ADDR=$(hostname -I) ########################## #
@@ -153,6 +154,10 @@ if [ -n "${config_check:+x}" ]; then sed -i "s/<USER>/$username/g" ${PWD}/yocto-worker/buildbot.tac sed -i "s/<PASS>/$password/g" ${PWD}/yocto-worker/buildbot.tac echo "" + echo " Updating worker-init script used for google cloud building. " + sed -i "s/<HOST_ADDR>/$HOST_ADDR/" ${PWD}/bin/worker-init + sed -i "s/<PASS>/$password/" ${PWD}/bin/worker-init + echo "" echo " If you wish to use your own generated username and password please " echo " modify the above files as needed. Please see the README for more " echo " information. "
Signature · 1 line
1.9.3
PATCH 5/7 README-GOOGLE-CLOUD: initial commit
In reply to [email protected]From: California Sullivan <[email protected]>
Goes over some limitations and gives an example of how to use it.
--- README-GOOGLE-CLOUD | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 README-GOOGLE-CLOUD
README-GOOGLE-CLOUD new file +68 -0
@@ -0,0 +1,68 @@ +Limitations +------------ +The current implementation of the Google Cloud building has a couple of +limitations. + +First of all, Google's gcloud command line tool is required for +all functions (see https://developers.google.com/cloud/sdk/gcloud/). + +Next, the controller machine is required to be authenticated with Google's cloud +system (see https://developers.google.com/cloud/sdk/gcloud/reference/auth/). + +Third, the workers must be on a network which has the worker port +(port 9989 by default) open. This can be done by setting up a network and +using that network in the ProvisionGoogleVM builstep. Example commands to +set up a network: + +gcloud compute networks create autobuilder-network +gcloud compute firewall-rules autobuilder-allow --network autobuilder-network \ + --allow tcp:8010 tcp:9989 tcp:80 tcp:22 + +This sets up a network that is suitable for both controllers and workers, as +it allows all default autobuilder connections, as well as ssh and http. The +key:value pair 'network':'autobuilder-network' can then be added to your +ProvisionGoogleVMs buildstep, allowing provisioned machines to connect to +controllers. + +Fourth, additional workers must be added to yocto-controller/controller.cfg +with the format: + BuildWorker("worker<N>", "<WORKER-PASSWORD>", max_builds=3) +where N is 0...however many workers you want and WORKER-PASSWORD is the default +password. For example, if I wanted to add three workers for the cloud builder: + + c['workers'] = [BuildWorker("example-worker", "eeieHim8V", max_builds=3),] + +would be changed to this: + + c['workers'] = [BuildWorker("example-worker", "eeieHim8V", max_builds=3), + BuildWorker("worker0", "eeieHim8V", max_builds=3), + BuildWorker("worker1", "eeieHim8V", max_builds=3), + BuildWorker("worker2", "eeieHim8V", max_builds=3),] + +Finally, the worker-init script is only an example and has a couple +limitations itself. It has only been tested for debian VMs and will likely not +work on others. Also, the <HOST_ADDR> and <PASS> fields will need to be +manually edited in it if your autobuilder was created before the addition of +the GoogleVM buildsteps or if the controller machine is behind a firewall. + +HowTo +---------- + +Assuming that the limitations above have been dealt with, this will provision +one new VM and completely set it up, including connections + + {'ProvisionGoogleVMs':{ + 'vmname':'autobuilder-worker-startup-test', + 'vmcount':1, + 'zone':'us-central1-a', + 'machine':'n1-standard-4', + 'disksize':'230GB', + 'startupscript':'/path/to/yocto-autobuilder/bin/worker-init', + 'network':'autobuilder-network'}} + +And this will delete the entire VM when you are finished with it: + + {'DeleteGoogleVMs': {}} + +Note that the current implementation of DeleteGoogleVM requires that its in the +same buildset as CreateGoogleVM.
Signature · 1 line
1.9.3
PATCH 6/7 ProvisionGoogleVMs.py: startup script now looks in bin directory of autobuilder.
In reply to [email protected]From: California Sullivan <[email protected]>
--- .../autobuilder/buildsteps/ProvisionGoogleVM.py | 85 --------------------- .../autobuilder/buildsteps/ProvisionGoogleVMs.py | 86 ++++++++++++++++++++++ 2 files changed, 86 insertions(+), 85 deletions(-) delete mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py create mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVMs.py
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py deleted +0 -85
@@ -1,85 +0,0 @@ -''' -Created on Aug 13, 2014 - -__author__ = "California Sullivan" -__copyright__ = "Copyright 2014, Intel Corp." -__credits__ = ["California Sullivan"] -__license__ = "GPL" -__version__ = "2.0" -__maintainer__ = "Elizabeth Flanagan" -__email__ = "[email protected]" -''' - -from buildbot.steps.shell import ShellCommand - -class ProvisionGoogleVM(ShellCommand): - haltOnFailure = True - flunkOnFailure = True - name = "Provision Google VM" - def __init__(self, factory, argdict=None, **kwargs): - self.vmname=None - self.vmcount=1 - self.startupscript=None - self.metadata=None - self.zone="us-central1-a" - self.image=None - self.machine=None - self.network=None - self.disksize=None - self.factory = factory - self.description = "Provision Google VM" - for k, v in argdict.iteritems(): - if type(v) is bool: - setattr(self, k, str(v)) - else: - setattr(self, k, v) - - if self.vmname is None or self.vmname == "": - self.finished(FAILURE) - else: - self.command = "gcloud compute instances create" - self.vmnames = "" - for x in range(0, self.vmcount): - self.vmnames += " " + self.vmname + "-" + str(x) - self.command += self.vmnames - if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]: - self.command += " --zone " + self.zone - else: - self.command += " --zone us-central1-a" - if self.disksize is not None and self.disksize != "": - self.command += " --boot-disk-size " + self.disksize - else: - self.command += " --boot-disk-size 200GB" - if self.image is None: - self.command += " --image debian-7" - else: - self.command += " --image " + self.image - - if self.machine is not None and self.machine in ["g1-small", "f1-micro", "n1-standard-1", "n1-standard-2", "n1-standard-4", "n1-standard-8", "n1-standard-16", "n1-highmem-2", "n1-highmem-4"]: - self.command += " --machine-type " + self.machine - else: - self.command += " --machine-type n1-standard-1" - if self.network is not None and self.network != "": - self.command += " --network " + self.network - if self.startupscript is not None and self.startupscript != "": - self.command += " --metadata-from-file startup-script=" + self.startupscript - if self.metadata is not None and self.metadata != "": - self.command += " --metadata " + self.scriptargs - self.command += " 1> /dev/null"; - - # Timeout needs to be passed to LoggingBuildStep as a kwarg - self.timeout = 1000 - kwargs['timeout']=self.timeout - ShellCommand.__init__(self, **kwargs) - - def describe(self, done=False): - description = ShellCommand.describe(self,done) - return description - - def commandComplete(self, cmd): - if not cmd.didFail(): - self.setProperty("vmnames", self.vmnames) - self.setProperty("zone", self.zone) - self.setProperty("vmcount", self.vmcount) - -
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVMs.py new file +86 -0
86 lines not on this page · raw message
Signature · 1 line
1.9.3
PATCH 7/7 ProvisionGoogleVMs.py: Delete VM before trying to provision it.
In reply to [email protected]From: California Sullivan <[email protected]>
In the case that somebody turned off a build before it completed, the buildstep would attempt and fail to procure a new VM, failing the build.
--- .../autobuilder/buildsteps/ProvisionGoogleVMs.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-)
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVMs.py +10 -6
@@ -19,6 +19,7 @@ class ProvisionGoogleVMs(ShellCommand): name = "Provision Google VMs" def __init__(self, factory, argdict=None, **kwargs): self.vmname=None + self.vmnames="" self.vmcount=1 self.startupscript=None self.metadata=None
@@ -38,15 +39,18 @@ class ProvisionGoogleVMs(ShellCommand): if self.vmname is None or self.vmname == "": self.finished(FAILURE) else: - self.command = "gcloud compute instances create" - self.vmnames = "" + self.command = "" + if self.zone is None or self.zone not in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]: + self.zone = "us-central1-a" for x in range(0, self.vmcount): self.vmnames += " " + self.vmname + "-" + str(x) + self.command += " gcloud compute instances delete" self.command += self.vmnames - if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]: - self.command += " --zone " + self.zone - else: - self.command += " --zone us-central1-a" + self.command += " --zone " + self.zone + " --quiet;" + self.command += " gcloud compute instances create" + self.command += self.vmnames + self.command += " --zone " + self.zone + if self.disksize is not None and self.disksize != "": self.command += " --boot-disk-size " + self.disksize else:
Signature · 1 line
1.9.3
Pulled into master, thanks!
-b
Quoted 104 lines · On Tue, Dec 2, 2014 at 3:31 PM, <[email protected]> wrote:
From: California Sullivan <[email protected]>
This build step provisions a Google VM using the gcloud command line tool.
Signed-off-by California Sullivan <[email protected]>--- .../autobuilder/buildsteps/ProvisionGoogleVM.py | 85 ++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.pylib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py new file +85
-1@@ -0,0 +1,85 @@ +''' +Created on Aug 13, 2014 + +__author__ = "California Sullivan" +__copyright__ = "Copyright 2014, Intel Corp." +__credits__ = ["California Sullivan"] +__license__ = "GPL" +__version__ = "2.0" +__maintainer__ = "Elizabeth Flanagan" +__email__ = "[email protected]" +''' + +from buildbot.steps.shell import ShellCommand + +class ProvisionGoogleVM(ShellCommand): + haltOnFailure = True + flunkOnFailure = True + name = "Provision Google VM" + def __init__(self, factory, argdict=None, **kwargs): + self.vmname=None + self.vmcount=1 + self.startupscript=None + self.metadata=None + self.zone="us-central1-a" + self.image=None + self.machine=None + self.network=None + self.disksize=None + self.factory = factory + self.description = "Provision Google VM" + for k, v in argdict.iteritems(): + if type(v) is bool: + setattr(self, k, str(v)) + else: + setattr(self, k, v) + + if self.vmname is None or self.vmname == "": + self.finished(FAILURE) + else: + self.command = "gcloud compute instances create" + self.vmnames = "" + for x in range(0, self.vmcount): + self.vmnames += " " + self.vmname + "-" + str(x) + self.command += self.vmnames + if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]: + self.command += " --zone " + self.zone + else: + self.command += " --zone us-central1-a" + if self.disksize is not None and self.disksize != "": + self.command += " --boot-disk-size " + self.disksize + else: + self.command += " --boot-disk-size 200GB" + if self.image is None: + self.command += " --image debian-7" + else: + self.command += " --image " + self.image + + if self.machine is not None and self.machine in ["g1-small", "f1-micro", "n1-standard-1", "n1-standard-2", "n1-standard-4", "n1-standard-8", "n1-standard-16", "n1-highmem-2", "n1-highmem-4"]: + self.command += " --machine-type " + self.machine + else: + self.command += " --machine-type n1-standard-1" + if self.network is not None and self.network != "": + self.command += " --network " + self.network + if self.startupscript is not None and self.startupscript != "": + self.command += " --metadata-from-file startup-script=" + self.startupscript + if self.metadata is not None and self.metadata != "": + self.command += " --metadata " + self.scriptargs + self.command += " 1> /dev/null"; + + # Timeout needs to be passed to LoggingBuildStep as a kwarg + self.timeout = 1000 + kwargs['timeout']=self.timeout + ShellCommand.__init__(self, **kwargs) + + def describe(self, done=False): + description = ShellCommand.describe(self,done) + return description + + def commandComplete(self, cmd): + if not cmd.didFail(): + self.setProperty("vmnames", self.vmnames) + self.setProperty("zone", self.zone) + self.setProperty("vmcount", self.vmcount) + + --1.9.3
Signature · 3 lines
Elizabeth Flanagan Yocto Project Build and Release