Skip to contentSkip to conversation

bitbake-setup: pass only the registry settting into obtain_config(), not the whole settings

PATCH New Alexander Kanavin 1 participant 7 messages Jan 16

lore
Share & tools
lore
#0 lore raw

PATCH 1/7 bitbake-setup: pass only the registry settting into obtain_config(), not the whole settings

From: Alexander Kanavin <[email protected]>

This will help the following commit.

---
 bin/bitbake-setup | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
bin/bitbake-setup +7 -7
index 4d709ed8c..4496cb61c 100755
@@ -523,7 +523,7 @@ def choose_fragments(possibilities, parameters, non_interactive, skip_selection)
         choices[k] = options_enumerated[option_n][1]
     return choices
 
-def obtain_config(top_dir, settings, args, source_overrides, d):
+def obtain_config(top_dir, registry, args, source_overrides, d):
     if args.config:
         config_id = args.config[0]
         config_parameters = args.config[1:]
@@ -541,20 +541,20 @@ def obtain_config(top_dir, settings, args, source_overrides, d):
               upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json.load(f)}
         else:
             logger.info("Looking up config {} in configuration registry".format(config_id))
-            registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
+            registry_path = update_registry(registry, cache_dir(top_dir), d)
             registry_configs = list_registry(registry_path, with_expired=True)
             if config_id not in registry_configs:
                 raise Exception("Config {} not found in configuration registry, re-run 'init' without parameters to choose from available configurations.".format(config_id))
-            upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
+            upstream_config = {'type':'registry','registry':registry,'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
         expiry_date = upstream_config['data'].get("expires", None)
         if has_expired(expiry_date):
             logger.warning("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date))
     else:
-        registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
+        registry_path = update_registry(registry, cache_dir(top_dir), d)
         registry_configs = list_registry(registry_path, with_expired=True)
         config_id = choose_config(registry_configs, args.non_interactive)
         config_parameters = []
-        upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
+        upstream_config = {'type':'registry','registry':registry,'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
 
     upstream_config['bitbake-config'] = choose_bitbake_config(upstream_config['data']['bitbake-setup']['configurations'], config_parameters, args.non_interactive)
     upstream_config['bitbake-config']['oe-fragment-choices'] = choose_fragments(upstream_config['bitbake-config'].get('oe-fragments-one-of',{}), config_parameters[1:], args.non_interactive, args.skip_selection)
@@ -594,7 +594,7 @@ def init_config(top_dir, settings, args):
         logger.handlers[0].terminator = '\n'
 
     source_overrides = obtain_overrides(args)
-    upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
+    upstream_config = obtain_config(top_dir, settings["default"]["registry"], args, source_overrides, d)
     logger.info("Run 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options'])))
 
     if args.setup_dir_name:
@@ -708,7 +708,7 @@ def build_status(top_dir, settings, args, d, update=False):
     args.non_interactive = True
     args.skip_selection = current_upstream_config['skip-selection']
     source_overrides = current_upstream_config["source-overrides"]
-    new_upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
+    new_upstream_config = obtain_config(top_dir, settings["default"]["registry"], args, source_overrides, d)
 
     write_upstream_config(confdir, new_upstream_config)
     config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]
Signature · 1 line
2.47.3
#1 lore raw

PATCH 2/7 bitbake-setup: use correct registry in status/update

From: Alexander Kanavin <[email protected]>

status/update operations should be using the registry that was used in initializing a setup, but they've been using the registry from settings. This is now corrected.

---
 bin/bitbake-setup | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
bin/bitbake-setup +2 -1
index 4496cb61c..b61436272 100755
@@ -708,7 +708,8 @@ def build_status(top_dir, settings, args, d, update=False):
     args.non_interactive = True
     args.skip_selection = current_upstream_config['skip-selection']
     source_overrides = current_upstream_config["source-overrides"]
-    new_upstream_config = obtain_config(top_dir, settings["default"]["registry"], args, source_overrides, d)
+    registry = current_upstream_config.get("registry")
+    new_upstream_config = obtain_config(top_dir, registry, args, source_overrides, d)
 
     write_upstream_config(confdir, new_upstream_config)
     config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]
Signature · 1 line
2.47.3
#2 lore raw

PATCH 3/7 bitbake-setup: do not print a premature notice about renaming bitbake configuration

In reply to Alexander Kanavin

From: Alexander Kanavin <[email protected]>

This was done before setting up a new bitbake configuration, so that it does not write over the old one, comparing the old and the new bitbake configurations, and taking one of three possible actions:

- no changes, no need to keep a backup;
- changes exist, do not replace existing config
- changes exist, replace existing config.

All three options correctly print the eventual outcome, so this early print is unnecessary and confusing.

---
 bin/bitbake-setup     | 1 -
 lib/bb/tests/setup.py | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)
bin/bitbake-setup +0 -1
index b61436272..8b5cb37ba 100755
@@ -309,7 +309,6 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c
 
     if os.path.exists(bitbake_confdir):
         os.rename(bitbake_confdir, backup_bitbake_confdir)
-        logger.info("Existing bitbake configuration directory renamed to {}".format(backup_bitbake_confdir))
 
     if layers:
         filerelative_layers = bitbake_config.get("bb-layers-file-relative") or []
lib/bb/tests/setup.py +2 -2
index 834d09854..802f017fb 100644
@@ -362,7 +362,7 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
             self.assertIn("Layer repository file://{} checked out into {}/layers/test-repo updated revision master from".format(self.testrepopath, setuppath), out[0])
             out = self.runbbsetup("update --update-bb-conf='yes'")
             if c in ('gadget', 'gizmo'):
-                self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(setuppath), out[0])
+                self.assertIn("Leaving the previous configuration in {}/build/conf-backup.".format(setuppath), out[0])
                 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
             self.check_setupdir_files(setuppath, test_file_content)
 
@@ -386,7 +386,7 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
             self.assertIn('-                    "rev": "master"\n+                    "rev": "another-branch"', out[0])
             out = self.runbbsetup("update --update-bb-conf='yes'")
             if c in ('gadget', 'gizmo'):
-                self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(setuppath), out[0])
+                self.assertIn("Leaving the previous configuration in {}/build/conf-backup.".format(setuppath), out[0])
                 self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
             self.check_setupdir_files(setuppath, test_file_content)
 
Signature · 1 line
2.47.3
#3 lore raw

PATCH 4/7 bitbake-setup: sort configurations before printing a list of them

In reply to Alexander Kanavin

From: Alexander Kanavin <[email protected]>

The order was non-deterministic which did not help finding the number for the right one in the list.

---
 bin/bitbake-setup | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
bin/bitbake-setup +2 -2
index 8b5cb37ba..776ab15f0 100755
@@ -474,7 +474,7 @@ def choose_bitbake_config(configs, parameters, non_interactive):
     return flattened_configs[config_n]
 
 def choose_config(configs, non_interactive):
-    not_expired_configs = [k for k in configs.keys() if not has_expired(configs[k].get("expires", None))]
+    not_expired_configs = [k for k in sorted(configs.keys()) if not has_expired(configs[k].get("expires", None))]
     config_list = list(enumerate(not_expired_configs, 1))
     if len(config_list) == 1:
         only_config = config_list[0][1]
@@ -792,7 +792,7 @@ def list_configs(settings, args):
     shutil.rmtree(top_dir)
 
     logger.plain("Available configurations:")
-    for config_name, config_data in json_data.items():
+    for config_name, config_data in sorted(json_data.items()):
         expiry_date = config_data.get("expires", None)
         config_desc = config_data["description"]
         if expiry_date:
Signature · 1 line
2.47.3
#4 lore raw

PATCH 5/7 bitbake-setup: tests: avoid repeating the list of config variants

In reply to Alexander Kanavin

From: Alexander Kanavin <[email protected]>

---
 lib/bb/tests/setup.py | 28 +++++++---------------------
 1 file changed, 7 insertions(+), 21 deletions(-)
lib/bb/tests/setup.py +7 -21
index 802f017fb..ef331c8a2 100644
@@ -324,17 +324,12 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         self.add_file_to_testrepo('test-file', test_file_content)
 
         # test-config-1 is tested as a registry config, test-config-2 as a local file
+        variants = ('gadget','gizmo','gizmo-env-passthrough','gizmo-no-fragment','gadget-notemplate','gizmo-notemplate')
+        variants_local = variants + ('gizmo-notemplate-with-filerelative-layers',)
         test_configurations = {'test-config-1': {'cmdline': 'test-config-1',
-                                                 'buildconfigs':('gadget','gizmo',
-                                                                 'gizmo-env-passthrough',
-                                                                 'gizmo-no-fragment',
-                                                                 'gadget-notemplate','gizmo-notemplate')},
+                                                 'buildconfigs': variants},
                                'test-config-2': {'cmdline': os.path.join(self.registrypath,'config-2/test-config-2.conf.json'),
-                                                 'buildconfigs': ('gadget','gizmo',
-                                                                  'gizmo-env-passthrough',
-                                                                  'gizmo-no-fragment',
-                                                                  'gadget-notemplate','gizmo-notemplate',
-                                                                  'gizmo-notemplate-with-filerelative-layers')}
+                                                 'buildconfigs': variants_local}
                                }
         for cf, v in test_configurations.items():
             for c in v['buildconfigs']:
@@ -352,10 +347,7 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         prev_test_file_content = test_file_content
         test_file_content = 'modified\n'
         self.add_file_to_testrepo('test-file', test_file_content)
-        for c in ('gadget', 'gizmo',
-                  'gizmo-env-passthrough',
-                  'gizmo-no-fragment',
-                  'gadget-notemplate', 'gizmo-notemplate'):
+        for c in variants:
             setuppath = self.get_setup_path('test-config-1', c)
             os.environ['BBPATH'] = os.path.join(setuppath, 'build')
             out = self.runbbsetup("status")
@@ -375,10 +367,7 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         self.git('checkout -b {}'.format(branch), cwd=self.testrepopath)
         self.add_file_to_testrepo('test-file', test_file_content)
         json_1 = self.add_json_config_to_registry('test-config-1.conf.json', branch, branch)
-        for c in ('gadget', 'gizmo',
-                  'gizmo-env-passthrough',
-                  'gizmo-no-fragment',
-                  'gadget-notemplate', 'gizmo-notemplate'):
+        for c in variants:
             setuppath = self.get_setup_path('test-config-1', c)
             os.environ['BBPATH'] = os.path.join(setuppath, 'build')
             out = self.runbbsetup("status")
@@ -408,10 +397,7 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         self.git('checkout -b {}'.format(branch), cwd=self.testrepopath)
         self.add_file_to_testrepo('test-file', test_file_content)
         json_1 = self.add_json_config_to_registry('test-config-1.conf.json', branch, branch)
-        for c in ('gadget', 'gizmo',
-                  'gizmo-env-passthrough',
-                  'gizmo-no-fragment',
-                  'gadget-notemplate', 'gizmo-notemplate'):
+        for c in variants:
             setuppath = self.get_setup_path('test-config-1', c)
             os.environ['BBPATH'] = os.path.join(setuppath, 'build')
             # write something in local.conf and bblayers.conf
Signature · 1 line
2.47.3
#5 lore raw

PATCH 6/7 bitbake-setup: tests: use a tuple for iterating over configurations

In reply to Alexander Kanavin

From: Alexander Kanavin <[email protected]>

This prepares the data for the next commit.

---
 lib/bb/tests/setup.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
lib/bb/tests/setup.py +5 -5
index ef331c8a2..41aae2d36 100644
@@ -326,15 +326,15 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         # test-config-1 is tested as a registry config, test-config-2 as a local file
         variants = ('gadget','gizmo','gizmo-env-passthrough','gizmo-no-fragment','gadget-notemplate','gizmo-notemplate')
         variants_local = variants + ('gizmo-notemplate-with-filerelative-layers',)
-        test_configurations = {'test-config-1': {'cmdline': 'test-config-1',
+        test_configurations = ({'name':'test-config-1','cmdline': 'test-config-1',
                                                  'buildconfigs': variants},
-                               'test-config-2': {'cmdline': os.path.join(self.registrypath,'config-2/test-config-2.conf.json'),
+                               {'name':'test-config-2','cmdline': os.path.join(self.registrypath,'config-2/test-config-2.conf.json'),
                                                  'buildconfigs': variants_local}
-                               }
-        for cf, v in test_configurations.items():
+                               )
+        for v in test_configurations:
             for c in v['buildconfigs']:
                 out = self.runbbsetup("init --non-interactive {} {}".format(v['cmdline'], c))
-                setuppath = self.get_setup_path(cf, c)
+                setuppath = self.get_setup_path(v['name'], c)
                 self.check_setupdir_files(setuppath, test_file_content)
 
         # install buildtools
Signature · 1 line
2.47.3
#6 lore raw

PATCH 7/7 bitbake-setup: tests: add a test for getting configs over http

In reply to Alexander Kanavin

From: Alexander Kanavin <[email protected]>

Configs from registry, and configs on local filesystem were tested, but not configs obtained over http URIs.

---
 lib/bb/tests/setup.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
lib/bb/tests/setup.py +8 -2
index 41aae2d36..a98160f4a 100644
@@ -8,6 +8,7 @@ from bb.tests.fetch import FetcherTest
 import json
 import hashlib
 import glob
+from bb.tests.support.httpserver import HTTPService
 
 class BitbakeSetupTest(FetcherTest):
     def setUp(self):
@@ -323,19 +324,24 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         test_file_content = 'initial\n'
         self.add_file_to_testrepo('test-file', test_file_content)
 
-        # test-config-1 is tested as a registry config, test-config-2 as a local file
+        # test-config-1 is tested as a registry config and over http, test-config-2 as a local file
+        server = HTTPService(self.registrypath, host="127.0.0.1")
+        server.start()
         variants = ('gadget','gizmo','gizmo-env-passthrough','gizmo-no-fragment','gadget-notemplate','gizmo-notemplate')
         variants_local = variants + ('gizmo-notemplate-with-filerelative-layers',)
         test_configurations = ({'name':'test-config-1','cmdline': 'test-config-1',
                                                  'buildconfigs': variants},
                                {'name':'test-config-2','cmdline': os.path.join(self.registrypath,'config-2/test-config-2.conf.json'),
-                                                 'buildconfigs': variants_local}
+                                                 'buildconfigs': variants_local},
+                               {'name':'test-config-1','cmdline':'http://127.0.0.1:{}/test-config-1.conf.json'.format(server.port),
+                                                 'buildconfigs': variants}
                                )
         for v in test_configurations:
             for c in v['buildconfigs']:
                 out = self.runbbsetup("init --non-interactive {} {}".format(v['cmdline'], c))
                 setuppath = self.get_setup_path(v['name'], c)
                 self.check_setupdir_files(setuppath, test_file_content)
+        server.stop()
 
         # install buildtools
         out = self.runbbsetup("install-buildtools --setup-dir {}".format(setuppath))
Signature · 1 line
2.47.3