Skip to contentSkip to conversation

0/6] Add support to handle layer in upgrade

PATCH New Aníbal Limón 1 participant 7 messages May 18, 2017

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 0/6] Add support to handle layer in upgrade

This series enables a layer mode to try make a recipe upgrades over a certain layer.

Aníbal Limón (6):
upgradehelper.py: Add support to load layer settings
README: Update with the new layer settings. upgradehelper.py: Add ability to run UniverseUpdater over certain
recipes
upgradehelper.py: Add support for sync layer master upgradehelper.py: Add layer name to upgrade email subject upgradehelper.py: Add _get_recipes_by_layer method

 README           |   6 +++
 upgradehelper.py | 118 +++++++++++++++++++++++++++++++++++++++++++++----------
 2 files changed, 104 insertions(+), 20 deletions(-)
Signature · 1 line
2.1.4
#1 lore raw

PATCH 1/1 1/6] upgradehelper.py: Add support to load layer settings

Those settings are to support layer recipe upgrades runs into the AUH.

[YOCTO #8962]

---
 upgradehelper.py | 42 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 8 deletions(-)
upgradehelper.py +34 -8
index df7d36d..f7b87c6 100755
@@ -131,10 +131,6 @@ class Updater(object):
     def __init__(self, auto_mode=False, send_email=False, skip_compilation=False):
         build_dir = get_build_dir()
 
-        self._make_dirs(build_dir)
-
-        self._add_file_logger()
-
         self.bb = Bitbake(build_dir)
 
         try:
@@ -145,18 +141,43 @@ class Updater(object):
             E( " Bitbake output:\n%s" % (e.stdout))
             exit(1)
 
+        self._set_options(auto_mode, send_email, skip_compilation)
+
+        self._make_dirs(build_dir)
+
+        self._add_file_logger()
+
         self.email_handler = Email(settings)
         self.statistics = Statistics()
-        # XXX: assume that the poky directory is the first entry in the PATH
-        self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
 
+    def _set_options(self, auto_mode, send_email, skip_compilation):
         self.opts = {}
+        self.opts['layer_mode'] = settings.get('layer_mode', '')
+        if self.opts['layer_mode'] == 'yes':
+            def _layer_settings_error(setting):
+                E(" In layer mode enable you need to specify %s.\n" % setting)
+                exit(1)
+
+            layer_settings = ('layer_name', 'layer_dir', 'layer_machines')
+            for s in layer_settings:
+                self.opts[s] = settings.get(s, '')
+                if not self.opts[s]:
+                    _layer_settings_error(s)
+
+            self.git = Git(self.opts['layer_dir'])
+            self.poky_git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
+            self.opts['machines'] = self.opts['layer_machines'].split()
+        else:
+            # XXX: assume that the poky directory is the first entry in the PATH
+            self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
+            self.poky_git = None
+            self.opts['machines'] = settings.get('machines',
+                'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
+
         self.opts['interactive'] = not auto_mode
         self.opts['send_email'] = send_email
         self.opts['author'] = "Upgrade Helper <%s>" % \
                 settings.get('from', '[email protected]')
-        self.opts['machines'] = settings.get('machines',
-                'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
         self.opts['skip_compilation'] = skip_compilation
         self.opts['buildhistory'] = self._buildhistory_is_enabled()
         self.opts['testimage'] = self._testimage_is_enabled()
@@ -168,6 +189,11 @@ class Updater(object):
         self.uh_base_work_dir = settings.get('workdir', '')
         if not self.uh_base_work_dir:
             self.uh_base_work_dir = self.uh_dir
+        if self.opts['layer_mode'] == 'yes':
+            self.uh_base_work_dir = os.path.join(self.uh_base_work_dir,
+                    self.opts['layer_name'])
+        if not os.path.exists(self.uh_base_work_dir):
+            os.mkdir(self.uh_base_work_dir)
         self.uh_work_dir = os.path.join(self.uh_base_work_dir, "%s" % \
                 datetime.now().strftime("%Y%m%d%H%M%S"))
         os.mkdir(self.uh_work_dir)
Signature · 1 line
2.1.4
#2 lore raw

PATCH 1/1 2/6] README: Update with the new layer settings.

In reply to Aníbal Limón

[YOCTO #8962]

---
 README | 6 ++++++
 1 file changed, 6 insertions(+)
README +6 -0
index 1b73034..e6b0dc2 100644
@@ -67,6 +67,12 @@ buildhistory=no
 testimage=no
 testimage_name=image-custom # defaults to core-image-sato
 
+# to enable upgrade recipes in a layer example for meta-intel
+layer_mode=False
+layer_name=meta-intel
+layer_dir=DIR/meta-intel
+layer_machines=intel-core2-32 intel-corei7-64 intel-quark
+
 --------------- snip ---------------
 
 3. Enable distrodata and supply appropriate additional metadata. For
Signature · 1 line
2.1.4
#3 lore raw

PATCH 1/1 3/6] upgradehelper.py: Add ability to run UniverseUpdater over certain recipes

In reply to Aníbal Limón

[YOCTO #8962]

---
 upgradehelper.py | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)
upgradehelper.py +21 -10
index f7b87c6..6732fb2 100755
@@ -655,9 +655,12 @@ class Updater(object):
                 self.send_status_mail(statistics_summary)
 
 class UniverseUpdater(Updater):
-    def __init__(self):
+    def __init__(self, recipes=None):
         Updater.__init__(self, True, True)
 
+        # to filter recipes in upgrade
+        self.recipes = recipes
+
         # read history file
         self.history_file = os.path.join(get_build_dir(), "upgrade-helper", "history.uh")
         self.history = dict()
@@ -694,11 +697,16 @@ class UniverseUpdater(Updater):
             I(" Removing tmp directory ...")
             shutil.rmtree(os.path.join(get_build_dir(), "tmp"))
 
-    def _check_upstream_versions(self, packages=[("universe", None, None)]):
+    def _check_upstream_versions(self):
         I(" Fetching upstream version(s) ...")
 
+        if self.recipes:
+            recipe = " ".join(self.recipes)
+        else:
+            recipe = 'universe'
+
         try:
-            self.bb.checkpkg(" ".join([p[0] for p in packages]))
+            self.bb.checkpkg(recipe)
         except Error as e:
             for line in e.stdout.split('\n'):
                 if line.find("ERROR: Task do_checkpkg does not exist") == 0:
@@ -876,16 +884,14 @@ if __name__ == "__main__":
                     level=debug_levels[args.debug_level - 1])
     settings, maintainer_override = parse_config_file(args.config_file)
 
-    if args.recipe == "all":
+    recipes = args.recipe.split()
+
+    if len(recipes) == 1 and recipes[0] == "all":
         updater = UniverseUpdater()
         updater.run()
-    else:
-        if not args.to_version:
-            E(" For upgrade only one recipe you must specify --to_version\n")
-            exit(1)
-
+    elif len(recipes) == 1 and args.to_version:
         if not args.maintainer and args.send_emails:
-            E(" For upgrade only one recipe and send email you must specify --maintainer\n")
+            E(" For upgrade one recipe and send email you must specify --maintainer\n")
             exit(1)
 
         if not args.maintainer:
@@ -895,3 +901,8 @@ if __name__ == "__main__":
         pkg_list = [(args.recipe, args.to_version, args.maintainer)]
         updater = Updater(args.auto_mode, args.send_emails, args.skip_compilation)
         updater.run(pkg_list)
+    else:
+        updater = UniverseUpdater(recipes)
+        updater.run()
+
+
Signature · 1 line
2.1.4
#4 lore raw

PATCH 1/1 4/6] upgradehelper.py: Add support for sync layer master

In reply to Aníbal Limón

[YOCTO #8962]

---
 upgradehelper.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)
upgradehelper.py +8 -1
index 6732fb2..41f81c1 100755
@@ -674,6 +674,10 @@ class UniverseUpdater(Updater):
                                                         line.split(',')[4]]
 
     def _update_master(self):
+        if self.opts['layer_mode'] == 'yes':
+            I(" Sync poky master ...")
+            self.poky_git.pull()
+
         I(" Drop all uncommited changes (including untracked) ...")
         self.git.reset_hard()
         self.git.clean_untracked()
@@ -683,7 +687,10 @@ class UniverseUpdater(Updater):
             self.git.delete_branch("upgrades")
         except Error:
             pass
-        I(" Sync master ...")
+        if self.opts['layer_mode'] == 'yes':
+            I(" Sync %s master ..." % self.opts['layer_name'])
+        else:
+            I(" Sync poky master ...")
         self.git.pull()
         self.git.create_branch("upgrades")
 
Signature · 1 line
2.1.4
#5 lore raw

PATCH 1/1 5/6] upgradehelper.py: Add layer name to upgrade email subject

In reply to Aníbal Limón

[YOCTO #8962]

---
 upgradehelper.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
upgradehelper.py +5 -1
index 41f81c1..be4282f 100755
@@ -440,7 +440,11 @@ class Updater(object):
 
         to_list = settings["status_recipients"].split()
 
-        subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
+        if self.opts['layer_mode'] == 'yes':
+            subject = "[AUH] Upgrade status %s: %s" \
+                    % (self.opts['layer_name'], date.isoformat(date.today()))
+        else:
+            subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
 
         if self.statistics.total_attempted:
             self.email_handler.send_email(to_list, subject, statistics_summary)
Signature · 1 line
2.1.4
#6 lore raw

PATCH 1/1 6/6] upgradehelper.py: Add _get_recipes_by_layer method

In reply to Aníbal Limón

When the layer mode is enabled and the upgradehelper is run with all, there is a need to discover what recipes are provides the layer to only try to upgrade those.

[YOCTO #8962]

---
 upgradehelper.py | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)
upgradehelper.py +31 -1
index be4282f..5ce2575 100755
@@ -663,7 +663,13 @@ class UniverseUpdater(Updater):
         Updater.__init__(self, True, True)
 
         # to filter recipes in upgrade
-        self.recipes = recipes
+        if not recipes and self.opts['layer_mode'] == 'yes':
+            # when layer mode is enabled and no recipes are specified
+            # we need to figure out what recipes are provided by the
+            # layer to try upgrade
+            self.recipes = self._get_recipes_by_layer()
+        else:
+            self.recipes = recipes
 
         # read history file
         self.history_file = os.path.join(get_build_dir(), "upgrade-helper", "history.uh")
@@ -676,6 +682,30 @@ class UniverseUpdater(Updater):
                                                         line.split(',')[2],
                                                         line.split(',')[3],
                                                         line.split(',')[4]]
+    def _get_recipes_by_layer(self):
+        recipes = []
+
+        recipe_regex = re.compile('^(?P<name>.*):$')
+        layer_regex = re.compile('^  (?P<name>.*) +')
+
+        layers = False
+        name = ''
+
+        output = subprocess.check_output('bitbake-layers show-recipes',
+                shell=True)
+        for line in output.decode("utf-8") .split('\n'):
+            s = recipe_regex.search(line)
+            if s:
+                name = s.group('name')
+                continue
+
+            if not 'skipped' in line:
+                s = layer_regex.search(line)
+                if s:
+                    if s.group('name').strip() == self.opts['layer_name']:
+                        recipes.append(name)
+
+        return recipes
 
     def _update_master(self):
         if self.opts['layer_mode'] == 'yes':
Signature · 1 line
2.1.4