Skip to contentSkip to conversation

ProvisionGoogleVM.py: Initial commit

PATCH In review [email protected] 2 participants 8 messages Dec 2, 2014 – Dec 9, 2014

lore
Share & tools
lore
#0 lore raw

PATCH 1/7 ProvisionGoogleVM.py: Initial commit

From: California Sullivan <[email protected]>

This build step provisions a Google VM using the gcloud command line tool.

---
 .../autobuilder/buildsteps/ProvisionGoogleVM.py    | 85 ++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py new file +85 -0
new file mode 100644
index 0000000..fb667dd
@@ -0,0 +1,85 @@
+'''
+Created on Aug 13, 2014
+
+__author__ = "California Sullivan"
+__copyright__ = "Copyright 2014, Intel Corp."
+__credits__ = ["California Sullivan"]
+__license__ = "GPL"
+__version__ = "2.0"
+__maintainer__ = "Elizabeth Flanagan"
+__email__ = "[email protected]"
+'''
+
+from buildbot.steps.shell import ShellCommand
+
+class ProvisionGoogleVM(ShellCommand):
+    haltOnFailure = True
+    flunkOnFailure = True
+    name = "Provision Google VM"
+    def __init__(self, factory, argdict=None, **kwargs):
+        self.vmname=None
+        self.vmcount=1
+        self.startupscript=None
+        self.metadata=None
+        self.zone="us-central1-a"
+        self.image=None
+        self.machine=None
+        self.network=None
+        self.disksize=None
+        self.factory = factory
+        self.description = "Provision Google VM"
+        for k, v in argdict.iteritems():
+            if type(v) is bool:
+                setattr(self, k, str(v))
+            else:
+                setattr(self, k, v)
+
+        if self.vmname is None or self.vmname == "":
+            self.finished(FAILURE)
+        else:
+            self.command = "gcloud compute instances create"
+            self.vmnames = ""
+            for x in range(0, self.vmcount):
+                self.vmnames += " " + self.vmname + "-" + str(x)
+            self.command += self.vmnames
+            if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]:
+                self.command += " --zone " + self.zone
+            else:
+                self.command += " --zone us-central1-a"
+            if self.disksize is not None and self.disksize != "":
+                self.command += " --boot-disk-size " + self.disksize
+            else:
+                self.command += " --boot-disk-size 200GB"
+            if self.image is None:
+                self.command += " --image debian-7"
+            else:
+                self.command += " --image " + self.image
+
+            if self.machine is not None and self.machine in ["g1-small", "f1-micro", "n1-standard-1", "n1-standard-2", "n1-standard-4", "n1-standard-8", "n1-standard-16", "n1-highmem-2", "n1-highmem-4"]:
+                self.command += " --machine-type " + self.machine
+            else:
+                self.command += " --machine-type n1-standard-1"
+            if self.network is not None and self.network != "":
+                self.command += " --network " + self.network
+            if self.startupscript is not None and self.startupscript != "":
+                self.command += " --metadata-from-file startup-script=" + self.startupscript
+                if self.metadata is not None and self.metadata != "":
+                    self.command += " --metadata " + self.scriptargs
+            self.command += " 1> /dev/null";
+
+            # Timeout needs to be passed to LoggingBuildStep as a kwarg
+            self.timeout = 1000
+            kwargs['timeout']=self.timeout
+            ShellCommand.__init__(self, **kwargs)
+
+    def describe(self, done=False):
+        description = ShellCommand.describe(self,done)
+        return description
+
+    def commandComplete(self, cmd):
+        if not cmd.didFail():
+            self.setProperty("vmnames", self.vmnames)
+            self.setProperty("zone", self.zone)
+            self.setProperty("vmcount", self.vmcount)
+
+
Signature · 1 line
1.9.3
#1 lore raw

PATCH 2/7 DeleteGoogleVMs.py: Initial commit

From: California Sullivan <[email protected]>

This build step deletes the Google VMs created in the previous ProvisionGoogleVM build step using Google's gcloud command line tool.

---
 .../autobuilder/buildsteps/DeleteGoogleVMs.py      | 52 ++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/DeleteGoogleVMs.py
lib/python2.7/site-packages/autobuilder/buildsteps/DeleteGoogleVMs.py new file +52 -0
new file mode 100644
index 0000000..b0071cf
@@ -0,0 +1,52 @@
+'''
+Created on Aug 13, 2014
+
+__author__ = "California Sullivan"
+__copyright__ = "Copyright 2014, Intel Corp."
+__credits__ = ["California Sullivan"]
+__license__ = "GPL"
+__version__ = "2.0"
+__maintainer__ = "Elizabeth Flanagan"
+__email__ = "[email protected]"
+'''
+
+from buildbot.steps.shell import ShellCommand
+
+class DeleteGoogleVMs(ShellCommand):
+    haltOnFailure = True
+    flunkOnFailure = True
+    name = "Delete Google VMs"
+    def __init__(self, factory, argdict=None, **kwargs):
+
+        self.factory = factory
+        self.description = "Deleting Google VMs"
+        for k, v in argdict.iteritems():
+            if type(v) is bool:
+                setattr(self, k, str(v))
+            else:
+                setattr(self, k, v)
+        # Timeout needs to be passed to LoggingBuildStep as a kwarg
+        self.timeout = 1000
+        kwargs['timeout']=self.timeout
+        ShellCommand.__init__(self, **kwargs)
+
+
+    def start(self):
+        self.vmnames = self.getProperty("vmnames")
+        self.vmcount = self.getProperty("vmcount")
+        self.zone = self.getProperty("zone")
+
+        if self.vmnames is not None and self.vmnames != "":
+            self.command = "gcloud compute instances delete"
+            self.command += " " + self.vmnames
+
+        if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]:
+            self.command += " --zone " + self.zone
+        else:
+            self.command += " --zone us-central1-a"
+        self.command += " --quiet"
+        ShellCommand.start(self)
+
+    def describe(self, done=False):
+        description = ShellCommand.describe(self,done)
+        return description
Signature · 1 line
1.9.3
#2 lore raw

PATCH 3/7 bin/worker-init: Initial commit

In reply to [email protected]

From: California Sullivan <[email protected]>

This script takes a fresh debian VM, completely sets up software and disks, starts an autobuilder worker, and then connects it to the controller. It is to be used by the ProvisionGoogleVMs build step.

---
 bin/worker-init | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100755 bin/worker-init
bin/worker-init new file +39 -0
new file mode 100755
index 0000000..7c6f827
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+if [ ! -f /var/log/firstboot.log ]
+then
+  START_SECTOR=$(fdisk -l | grep sda1 | sed 's/\*//' | awk '{ print $2 }')
+  echo -e "d\nn\n\n\n$START_SECTOR\n\nw\n" | fdisk /dev/sda
+  touch /var/log/firstboot.log
+  reboot
+fi
+
+if [ ! -f /var/log/secondboot.log ]
+then
+  resize2fs /dev/sda1
+  sudo apt-get update
+  yes | sudo apt-get install make
+  yes | sudo apt-get install git
+  yes | sudo apt-get install gcc
+  yes | sudo apt-get install g++
+  yes | sudo apt-get install diffstat
+  yes | sudo apt-get install texinfo
+  yes | sudo apt-get install bzip2
+  yes | sudo apt-get install gawk
+  yes | sudo apt-get install chrpath
+  yes | sudo apt-get install libsdl1.2-dev
+  yes "dummyinfo" | adduser autobuilder
+  touch /var/log/secondboot.log
+fi
+
+WORKER_NUM=$(uname -n | grep -o [0-9])
+
+cd /home/autobuilder
+su autobuilder -c "yes 'yes' | git clone git://git.yoctoproject.org/yocto-autobuilder"
+
+cd yocto-autobuilder
+su autobuilder -c '. ./yocto-autobuilder-setup'
+su autobuilder -c "sed -i 's/example-worker/worker'$WORKER_NUM'/' yocto-worker/buildbot.tac"
+su autobuilder -c 'sed -i "s/passwd =.*/passwd = '"'"'<PASS>'"'"'/" yocto-worker/buildbot.tac'
+su autobuilder -c 'sed -i "s/buildmaster_host =.*/buildmaster_host = '"'"'<HOST_ADDR>'"'"'/" yocto-worker/buildbot.tac'
+su autobuilder -c '. ./yocto-autobuilder-setup; ./yocto-start-autobuilder worker &'
Signature · 1 line
1.9.3
#3 lore raw

PATCH 4/7 yocto-autobuilder-setup: Add worker init script setup

In reply to [email protected]

From: California Sullivan <[email protected]>

The password and address fields in the worker init script need to be set up.

---
 yocto-autobuilder-setup | 5 +++++
 1 file changed, 5 insertions(+)
yocto-autobuilder-setup +5 -0
index aee71ce..615deb4 100644
@@ -60,6 +60,7 @@ export OPTIMIZED_GIT_CLONE=False
 export OGIT_TRASH_DIR=/tmp/yocto-autobuilder/git/trash
 export OGIT_TRASH_CRON_TIME="0 0 * * *"
 export OGIT_TRASH_NICE_LEVEL="19"
+HOST_ADDR=$(hostname -I)
 
 ##########################
 #
@@ -153,6 +154,10 @@ if [ -n "${config_check:+x}" ]; then
     sed -i "s/<USER>/$username/g" ${PWD}/yocto-worker/buildbot.tac
     sed -i "s/<PASS>/$password/g" ${PWD}/yocto-worker/buildbot.tac
     echo ""
+    echo " Updating worker-init script used for google cloud building. "
+    sed -i "s/<HOST_ADDR>/$HOST_ADDR/" ${PWD}/bin/worker-init
+    sed -i "s/<PASS>/$password/" ${PWD}/bin/worker-init
+    echo ""
     echo " If you wish to use your own generated username and password please "
     echo " modify the above files as needed. Please see the README for more "
     echo " information. "
Signature · 1 line
1.9.3
#4 lore raw

PATCH 5/7 README-GOOGLE-CLOUD: initial commit

In reply to [email protected]

From: California Sullivan <[email protected]>

Goes over some limitations and gives an example of how to use it.

---
 README-GOOGLE-CLOUD | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 README-GOOGLE-CLOUD
README-GOOGLE-CLOUD new file +68 -0
new file mode 100644
index 0000000..34c80a9
@@ -0,0 +1,68 @@
+Limitations
+------------
+The current implementation of the Google Cloud building has a couple of
+limitations.
+
+First of all, Google's gcloud command line tool is required for
+all functions (see https://developers.google.com/cloud/sdk/gcloud/).
+
+Next, the controller machine is required to be authenticated with Google's cloud
+system (see https://developers.google.com/cloud/sdk/gcloud/reference/auth/).
+
+Third, the workers must be on a network which has the worker port
+(port 9989 by default) open. This can be done by setting up a network and
+using that network in the ProvisionGoogleVM builstep. Example commands to
+set up a network:
+
+gcloud compute networks create autobuilder-network
+gcloud compute firewall-rules autobuilder-allow --network autobuilder-network \
+    --allow tcp:8010 tcp:9989 tcp:80 tcp:22
+
+This sets up a network that is suitable for both controllers and workers, as
+it allows all default autobuilder connections, as well as ssh and http. The
+key:value pair 'network':'autobuilder-network' can then be added to your
+ProvisionGoogleVMs buildstep, allowing provisioned machines to connect to
+controllers.
+
+Fourth, additional workers must be added to yocto-controller/controller.cfg
+with the format:
+     BuildWorker("worker<N>", "<WORKER-PASSWORD>", max_builds=3)
+where N is 0...however many workers you want and WORKER-PASSWORD is the default
+password. For example, if I wanted to add three workers for the cloud builder:
+
+    c['workers'] = [BuildWorker("example-worker", "eeieHim8V", max_builds=3),]
+
+would be changed to this:
+
+    c['workers'] = [BuildWorker("example-worker", "eeieHim8V", max_builds=3),
+                    BuildWorker("worker0", "eeieHim8V", max_builds=3),
+                    BuildWorker("worker1", "eeieHim8V", max_builds=3),
+                    BuildWorker("worker2", "eeieHim8V", max_builds=3),]
+
+Finally, the worker-init script is only an example and has a couple
+limitations itself. It has only been tested for debian VMs and will likely not
+work on others. Also, the <HOST_ADDR> and <PASS> fields will need to be
+manually edited in it if your autobuilder was created before the addition of
+the GoogleVM buildsteps or if the controller machine is behind a firewall.
+
+HowTo
+----------
+
+Assuming that the limitations above have been dealt with, this will provision
+one new VM and completely set it up, including connections
+
+    {'ProvisionGoogleVMs':{
+        'vmname':'autobuilder-worker-startup-test',
+        'vmcount':1,
+        'zone':'us-central1-a',
+        'machine':'n1-standard-4',
+        'disksize':'230GB',
+        'startupscript':'/path/to/yocto-autobuilder/bin/worker-init',
+        'network':'autobuilder-network'}}
+
+And this will delete the entire VM when you are finished with it:
+
+    {'DeleteGoogleVMs': {}}
+
+Note that the current implementation of DeleteGoogleVM requires that its in the
+same buildset as CreateGoogleVM.
Signature · 1 line
1.9.3
#5 lore raw

PATCH 6/7 ProvisionGoogleVMs.py: startup script now looks in bin directory of autobuilder.

In reply to [email protected]

From: California Sullivan <[email protected]>

---
 .../autobuilder/buildsteps/ProvisionGoogleVM.py    | 85 ---------------------
 .../autobuilder/buildsteps/ProvisionGoogleVMs.py   | 86 ++++++++++++++++++++++
 2 files changed, 86 insertions(+), 85 deletions(-)
 delete mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py
 create mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVMs.py
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py deleted +0 -85
deleted file mode 100644
index fb667dd..0000000
@@ -1,85 +0,0 @@
-'''
-Created on Aug 13, 2014
-
-__author__ = "California Sullivan"
-__copyright__ = "Copyright 2014, Intel Corp."
-__credits__ = ["California Sullivan"]
-__license__ = "GPL"
-__version__ = "2.0"
-__maintainer__ = "Elizabeth Flanagan"
-__email__ = "[email protected]"
-'''
-
-from buildbot.steps.shell import ShellCommand
-
-class ProvisionGoogleVM(ShellCommand):
-    haltOnFailure = True
-    flunkOnFailure = True
-    name = "Provision Google VM"
-    def __init__(self, factory, argdict=None, **kwargs):
-        self.vmname=None
-        self.vmcount=1
-        self.startupscript=None
-        self.metadata=None
-        self.zone="us-central1-a"
-        self.image=None
-        self.machine=None
-        self.network=None
-        self.disksize=None
-        self.factory = factory
-        self.description = "Provision Google VM"
-        for k, v in argdict.iteritems():
-            if type(v) is bool:
-                setattr(self, k, str(v))
-            else:
-                setattr(self, k, v)
-
-        if self.vmname is None or self.vmname == "":
-            self.finished(FAILURE)
-        else:
-            self.command = "gcloud compute instances create"
-            self.vmnames = ""
-            for x in range(0, self.vmcount):
-                self.vmnames += " " + self.vmname + "-" + str(x)
-            self.command += self.vmnames
-            if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]:
-                self.command += " --zone " + self.zone
-            else:
-                self.command += " --zone us-central1-a"
-            if self.disksize is not None and self.disksize != "":
-                self.command += " --boot-disk-size " + self.disksize
-            else:
-                self.command += " --boot-disk-size 200GB"
-            if self.image is None:
-                self.command += " --image debian-7"
-            else:
-                self.command += " --image " + self.image
-
-            if self.machine is not None and self.machine in ["g1-small", "f1-micro", "n1-standard-1", "n1-standard-2", "n1-standard-4", "n1-standard-8", "n1-standard-16", "n1-highmem-2", "n1-highmem-4"]:
-                self.command += " --machine-type " + self.machine
-            else:
-                self.command += " --machine-type n1-standard-1"
-            if self.network is not None and self.network != "":
-                self.command += " --network " + self.network
-            if self.startupscript is not None and self.startupscript != "":
-                self.command += " --metadata-from-file startup-script=" + self.startupscript
-                if self.metadata is not None and self.metadata != "":
-                    self.command += " --metadata " + self.scriptargs
-            self.command += " 1> /dev/null";
-
-            # Timeout needs to be passed to LoggingBuildStep as a kwarg
-            self.timeout = 1000
-            kwargs['timeout']=self.timeout
-            ShellCommand.__init__(self, **kwargs)
-
-    def describe(self, done=False):
-        description = ShellCommand.describe(self,done)
-        return description
-
-    def commandComplete(self, cmd):
-        if not cmd.didFail():
-            self.setProperty("vmnames", self.vmnames)
-            self.setProperty("zone", self.zone)
-            self.setProperty("vmcount", self.vmcount)
-
-
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVMs.py new file +86 -0

86 lines not on this page · raw message

Signature · 1 line
1.9.3
#6 lore raw

PATCH 7/7 ProvisionGoogleVMs.py: Delete VM before trying to provision it.

In reply to [email protected]

From: California Sullivan <[email protected]>

In the case that somebody turned off a build before it completed, the buildstep would attempt and fail to procure a new VM, failing the build.

---
 .../autobuilder/buildsteps/ProvisionGoogleVMs.py         | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVMs.py +10 -6
index d0bc6eb..262526f 100644
@@ -19,6 +19,7 @@ class ProvisionGoogleVMs(ShellCommand):
     name = "Provision Google VMs"
     def __init__(self, factory, argdict=None, **kwargs):
         self.vmname=None
+        self.vmnames=""
         self.vmcount=1
         self.startupscript=None
         self.metadata=None
@@ -38,15 +39,18 @@ class ProvisionGoogleVMs(ShellCommand):
         if self.vmname is None or self.vmname == "":
             self.finished(FAILURE)
         else:
-            self.command = "gcloud compute instances create"
-            self.vmnames = ""
+            self.command = ""
+            if self.zone is None or self.zone not in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]:
+                self.zone = "us-central1-a"
             for x in range(0, self.vmcount):
                 self.vmnames += " " + self.vmname + "-" + str(x)
+            self.command += " gcloud compute instances delete"
             self.command += self.vmnames
-            if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]:
-                self.command += " --zone " + self.zone
-            else:
-                self.command += " --zone us-central1-a"
+            self.command += " --zone " + self.zone + " --quiet;"
+            self.command += " gcloud compute instances create"
+            self.command += self.vmnames
+            self.command += " --zone " + self.zone
+
             if self.disksize is not None and self.disksize != "":
                 self.command += " --boot-disk-size " + self.disksize
             else:
Signature · 1 line
1.9.3
#7 lore raw
In reply to [email protected]

Pulled into master, thanks!

-b

Quoted 104 lines · On Tue, Dec 2, 2014 at 3:31 PM, <[email protected]> wrote:

From: California Sullivan <[email protected]>

This build step provisions a Google VM using the gcloud command line tool.

---
 .../autobuilder/buildsteps/ProvisionGoogleVM.py    | 85 ++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py
lib/python2.7/site-packages/autobuilder/buildsteps/ProvisionGoogleVM.py new file +85 -1
new file mode 100644
index 0000000..fb667dd
@@ -0,0 +1,85 @@
+'''
+Created on Aug 13, 2014
+
+__author__ = "California Sullivan"
+__copyright__ = "Copyright 2014, Intel Corp."
+__credits__ = ["California Sullivan"]
+__license__ = "GPL"
+__version__ = "2.0"
+__maintainer__ = "Elizabeth Flanagan"
+__email__ = "[email protected]"
+'''
+
+from buildbot.steps.shell import ShellCommand
+
+class ProvisionGoogleVM(ShellCommand):
+    haltOnFailure = True
+    flunkOnFailure = True
+    name = "Provision Google VM"
+    def __init__(self, factory, argdict=None, **kwargs):
+        self.vmname=None
+        self.vmcount=1
+        self.startupscript=None
+        self.metadata=None
+        self.zone="us-central1-a"
+        self.image=None
+        self.machine=None
+        self.network=None
+        self.disksize=None
+        self.factory = factory
+        self.description = "Provision Google VM"
+        for k, v in argdict.iteritems():
+            if type(v) is bool:
+                setattr(self, k, str(v))
+            else:
+                setattr(self, k, v)
+
+        if self.vmname is None or self.vmname == "":
+            self.finished(FAILURE)
+        else:
+            self.command = "gcloud compute instances create"
+            self.vmnames = ""
+            for x in range(0, self.vmcount):
+                self.vmnames += " " + self.vmname + "-" + str(x)
+            self.command += self.vmnames
+            if self.zone is not None and self.zone in ["us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b", "asia-east1-a", "asia-east1-b", "asia-east1-b"]:
+                self.command += " --zone " + self.zone
+            else:
+                self.command += " --zone us-central1-a"
+            if self.disksize is not None and self.disksize != "":
+                self.command += " --boot-disk-size " + self.disksize
+            else:
+                self.command += " --boot-disk-size 200GB"
+            if self.image is None:
+                self.command += " --image debian-7"
+            else:
+                self.command += " --image " + self.image
+
+            if self.machine is not None and self.machine in ["g1-small", "f1-micro", "n1-standard-1", "n1-standard-2", "n1-standard-4", "n1-standard-8", "n1-standard-16", "n1-highmem-2", "n1-highmem-4"]:
+                self.command += " --machine-type " + self.machine
+            else:
+                self.command += " --machine-type n1-standard-1"
+            if self.network is not None and self.network != "":
+                self.command += " --network " + self.network
+            if self.startupscript is not None and self.startupscript != "":
+                self.command += " --metadata-from-file startup-script=" + self.startupscript
+                if self.metadata is not None and self.metadata != "":
+                    self.command += " --metadata " + self.scriptargs
+            self.command += " 1> /dev/null";
+
+            # Timeout needs to be passed to LoggingBuildStep as a kwarg
+            self.timeout = 1000
+            kwargs['timeout']=self.timeout
+            ShellCommand.__init__(self, **kwargs)
+
+    def describe(self, done=False):
+        description = ShellCommand.describe(self,done)
+        return description
+
+    def commandComplete(self, cmd):
+        if not cmd.didFail():
+            self.setProperty("vmnames", self.vmnames)
+            self.setProperty("zone", self.zone)
+            self.setProperty("vmcount", self.vmcount)
+
+
--

1.9.3

Signature · 3 lines
Elizabeth Flanagan
Yocto Project
Build and Release