bitbake-setup: move the local source tests to the end
PATCH 1/10 bitbake-setup: move the local source tests to the end
From: Alexander Kanavin <[email protected]>
They are not a part of the main testing sequence, and logically should not be in the middle of it.
--- lib/bb/tests/setup.py | 56 +++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-)
lib/bb/tests/setup.py +28 -28
@@ -320,34 +320,6 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) out = self.runbbsetup("update --update-bb-conf='yes'") self.assertIn("Configuration in {} has not changed".format(setuppath), out[0]) - # check source overrides, local sources provided with symlinks, and custom setup dir name - source_override_content = """ -{ - "sources": { - "test-repo": { - "local": { - "path": "." - } - } - } -}""" - override_filename = 'source-overrides.json' - custom_setup_dir = 'special-setup-dir' - self.add_file_to_testrepo(override_filename, source_override_content) - out = self.runbbsetup("init --non-interactive --source-overrides {} --setup-dir-name {} test-config-1 gadget".format(os.path.join(self.testrepopath, override_filename), custom_setup_dir)) - custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir) - custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo') - self.assertTrue(os.path.islink(custom_layer_path)) - self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) - - # same but use command line options to specify local overrides - custom_setup_dir = 'special-setup-dir-with-cmdline-overrides' - out = self.runbbsetup("init --non-interactive -L test-repo {} --setup-dir-name {} test-config-1 gadget".format(self.testrepopath, custom_setup_dir)) - custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir) - custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo') - self.assertTrue(os.path.islink(custom_layer_path)) - self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) - # install buildtools out = self.runbbsetup("install-buildtools") self.assertIn("Buildtools installed into", out[0])
@@ -428,3 +400,31 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) out = self.runbbsetup("update --update-bb-conf='no'") sums_after = _conf_chksum(f"{setuppath}/build/conf") self.assertEqual(sums_before, sums_after) + + # check source overrides, local sources provided with symlinks, and custom setup dir name + source_override_content = """ +{ + "sources": { + "test-repo": { + "local": { + "path": "." + } + } + } +}""" + override_filename = 'source-overrides.json' + custom_setup_dir = 'special-setup-dir' + self.add_file_to_testrepo(override_filename, source_override_content) + out = self.runbbsetup("init --non-interactive --source-overrides {} --setup-dir-name {} test-config-1 gadget".format(os.path.join(self.testrepopath, override_filename), custom_setup_dir)) + custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir) + custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo') + self.assertTrue(os.path.islink(custom_layer_path)) + self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) + + # same but use command line options to specify local overrides + custom_setup_dir = 'special-setup-dir-with-cmdline-overrides' + out = self.runbbsetup("init --non-interactive -L test-repo {} --setup-dir-name {} test-config-1 gadget".format(self.testrepopath, custom_setup_dir)) + custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir) + custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo') + self.assertTrue(os.path.islink(custom_layer_path)) + self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path))
Signature · 1 line
2.47.3
PATCH 2/10 bitbake-setup: consistently add configuration commits at the end of updates
From: Alexander Kanavin <[email protected]>
bitbake-setup tracks configuration history under setup directory in config/.
Making commits in there wasn't consistently done in all init/update scenarios sometimes leaving uncommitted files. Move the commit creation to the end of update_build() function, ensuring it always happens.
--- bin/bitbake-setup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
bin/bitbake-setup +1 -2
@@ -342,6 +342,7 @@ def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt" thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf) write_sources_fixed_revisions(confdir, sources_fixed_revisions) + commit_config(confdir) def int_input(allowed_values, prompt=''): n = None
@@ -581,7 +582,6 @@ def init_config(top_dir, settings, args): write_upstream_config(confdir, upstream_config) update_build(upstream_config, confdir, setupdir, layerdir, d, update_bb_conf="yes") - commit_config(confdir) bb.event.remove("bb.build.TaskProgress", None)
@@ -648,7 +648,6 @@ def build_status(top_dir, settings, args, d, update=False): if config_diff: logger.plain('\nConfiguration in {} has changed:\n{}'.format(setupdir, config_diff)) if update: - commit_config(confdir) update_build(new_upstream_config, confdir, setupdir, layerdir, d, update_bb_conf=args.update_bb_conf) else:
Signature · 1 line
2.47.3
PATCH 3/10 bitbake-setup: test after every update that config hasn't changed
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This would catch situations where update hasn't run through to completion even though bitbake-setup finished without error, such as the one fixed in the previous commit.
Also unset BBPATH from environment so that it doesn't leak out of the function where it's used. Adjust buildtools test to pass setup dir explicitly, as it was previously relying on BBPATH being set.
--- lib/bb/tests/setup.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)
lib/bb/tests/setup.py +11 -6
@@ -180,6 +180,14 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) self.git('add {}'.format(name), cwd=self.testrepopath) self.git('commit -m "Adding {}"'.format(name), cwd=self.testrepopath) + def config_is_unchanged(self, setuppath): + os.environ['BBPATH'] = os.path.join(setuppath, 'build') + out = self.runbbsetup("status") + self.assertIn("Configuration in {} has not changed".format(setuppath), out[0]) + out = self.runbbsetup("update --update-bb-conf='yes'") + self.assertIn("Configuration in {} has not changed".format(setuppath), out[0]) + del os.environ['BBPATH'] + def check_setupdir_files(self, setuppath, test_file_content): with open(os.path.join(setuppath, 'config', "config-upstream.json")) as f: config_upstream = json.load(f)
@@ -228,6 +236,8 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) self.assertTrue('BUILD_SERVER' in init_build_env) # a more throrough test could be to initialize a bitbake build-env, export FOO to the shell environment, set the env-passthrough on it and finally check against 'bitbake-getvar FOO' + self.config_is_unchanged(setuppath) + def get_setup_path(self, cf, c): if c == 'gizmo': return os.path.join(self.tempdir, 'bitbake-builds', 'this-is-a-custom-gizmo-build')
@@ -314,14 +324,9 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) out = self.runbbsetup("init --non-interactive {} {}".format(v['cmdline'], c)) setuppath = self.get_setup_path(cf, c) self.check_setupdir_files(setuppath, test_file_content) - os.environ['BBPATH'] = os.path.join(setuppath, 'build') - out = self.runbbsetup("status") - self.assertIn("Configuration in {} has not changed".format(setuppath), out[0]) - out = self.runbbsetup("update --update-bb-conf='yes'") - self.assertIn("Configuration in {} has not changed".format(setuppath), out[0]) # install buildtools - out = self.runbbsetup("install-buildtools") + out = self.runbbsetup("install-buildtools --setup-dir {}".format(setuppath)) self.assertIn("Buildtools installed into", out[0]) self.assertTrue(os.path.exists(os.path.join(setuppath, 'buildtools')))
Signature · 1 line
2.47.3
PATCH 4/10 bitbake-setup: consider overrides when checking whether layers have changed in 'status'/'update'
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This was erroneusly not done, and the original layer spec was used.
Make a function out of the code that overlayes overrides onto the base layer spec, so the code doesn't have to be repeated.
--- bin/bitbake-setup | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-)
bin/bitbake-setup +10 -6
@@ -331,12 +331,15 @@ def get_registry_config(registry_path, id): return os.path.join(root, f) raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id)) +def merge_overrides_into_sources(sources, overrides): + layers = copy.deepcopy(sources) + for k,v in overrides.items(): + if k in layers: + layers[k] = v + return layers + def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt"): - layer_config = copy.deepcopy(config["data"]["sources"]) - layer_overrides = config["source-overrides"]["sources"] - for k,v in layer_overrides.items(): - if k in layer_config: - layer_config[k] = v + layer_config = merge_overrides_into_sources(config["data"]["sources"], config["source-overrides"]["sources"]) sources_fixed_revisions = checkout_layers(layer_config, layerdir, d) bitbake_config = config["bitbake-config"] thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
@@ -654,7 +657,8 @@ def build_status(top_dir, settings, args, d, update=False): bb.process.run('git -C {} restore config-upstream.json'.format(confdir)) return - if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d): + layer_config = merge_overrides_into_sources(current_upstream_config["data"]["sources"], current_upstream_config["source-overrides"]["sources"]) + if are_layers_changed(layer_config, layerdir, d): if update: update_build(current_upstream_config, confdir, setupdir, layerdir, d, update_bb_conf=args.update_bb_conf)
Signature · 1 line
2.47.3
PATCH 5/10 bitbake-setup: fix are_layers_changed when use local source
In reply to Alexander KanavinFrom: Anibal Limon <[email protected]>
When use local source the git-remote isn't set causing exception because takes repodir as default value.
Fixes:
```
NOTE: Looking up config ls-master in configuration registry
Traceback (most recent call last):
File "/home/workspaces/ls/bitbake/bin/bitbake-setup", line 1042, in <module>
main()
~~~~^^
File "/home/workspaces/ls/bitbake/bin/bitbake-setup", line 1035, in main
args.func(top_dir, all_settings, args, d)
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/workspaces/ls/bitbake/bin/bitbake-setup", line 667, in build_update
build_status(top_dir, settings, args, d, update=True)
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/workspaces/ls/bitbake/bin/bitbake-setup", line 658, in build_status
if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d):
~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/workspaces/ls/bitbake/bin/bitbake-setup", line 627, in are_layers_changed
changed = changed | _is_git_remote_changed(git_remote, repodir)
~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^
File "/home/workspaces/ls/bitbake/bin/bitbake-setup", line 601, in _is_git_remote_changed
rev = r_remote['rev']
~~~~~~~~^^^^^^^
TypeError: string indices must be integers, not 'str'
```Alex: remove the default '' value from get() as None is used otherwise, which is also suitable.
--- bin/bitbake-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
bin/bitbake-setup +1 -1
@@ -625,7 +625,7 @@ def are_layers_changed(layers, layerdir, d): r_data = layers[r_name] repodir = r_data.get("path", r_name) - git_remote = r_data.get('git-remote', repodir) + git_remote = r_data.get('git-remote') if git_remote: changed = changed | _is_git_remote_changed(git_remote, repodir)
Signature · 1 line
2.47.3
PATCH 6/10 bitbake-setup: extend the tests for local sources with status/update sequence
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This checks for both issues fixed in two previous commits.
Split common local source checks into a function and extend it with a call to status/update checker.
--- lib/bb/tests/setup.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-)
lib/bb/tests/setup.py +9 -8
@@ -407,6 +407,13 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) self.assertEqual(sums_before, sums_after) # check source overrides, local sources provided with symlinks, and custom setup dir name + def _check_local_sources(custom_setup_dir): + custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir) + custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo') + self.assertTrue(os.path.islink(custom_layer_path)) + self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) + self.config_is_unchanged(custom_setup_path) + source_override_content = """ { "sources": {
@@ -421,15 +428,9 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) custom_setup_dir = 'special-setup-dir' self.add_file_to_testrepo(override_filename, source_override_content) out = self.runbbsetup("init --non-interactive --source-overrides {} --setup-dir-name {} test-config-1 gadget".format(os.path.join(self.testrepopath, override_filename), custom_setup_dir)) - custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir) - custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo') - self.assertTrue(os.path.islink(custom_layer_path)) - self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) + _check_local_sources(custom_setup_dir) # same but use command line options to specify local overrides custom_setup_dir = 'special-setup-dir-with-cmdline-overrides' out = self.runbbsetup("init --non-interactive -L test-repo {} --setup-dir-name {} test-config-1 gadget".format(self.testrepopath, custom_setup_dir)) - custom_setup_path = os.path.join(self.tempdir, 'bitbake-builds', custom_setup_dir) - custom_layer_path = os.path.join(custom_setup_path, 'layers', 'test-repo') - self.assertTrue(os.path.islink(custom_layer_path)) - self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) + _check_local_sources(custom_setup_dir)
Signature · 1 line
2.47.3
PATCH 7/10 bitbake-setup: in tests, refactor the config writer to allow variable sources
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This will be used to make configs that use local sources instead of git-remotes.
--- lib/bb/tests/setup.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-)
lib/bb/tests/setup.py +20 -13
@@ -89,21 +89,12 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) bbsetup = os.path.abspath(os.path.dirname(__file__) + "/../../../bin/bitbake-setup") return bb.process.run("{} --global-settings {} {}".format(bbsetup, os.path.join(self.tempdir, 'global-config'), cmd)) - def add_json_config_to_registry(self, name, rev, branch): + + def _add_json_config_to_registry_helper(self, name, sources): config = """ { "sources": { - "test-repo": { - "git-remote": { - "remotes": { - "origin": { - "uri": "file://%s" - } - }, - "branch": "%s", - "rev": "%s" - } - } +%s }, "description": "Test configuration", "bitbake-setup": {
@@ -160,7 +151,7 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) }, "version": "1.0" } -""" % (self.testrepopath, branch, rev) +""" % (sources) os.makedirs(os.path.join(self.registrypath, os.path.dirname(name)), exist_ok=True) with open(os.path.join(self.registrypath, name), 'w') as f: f.write(config)
@@ -168,6 +159,22 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) self.git('commit -m "Adding {}"'.format(name), cwd=self.registrypath) return json.loads(config) + def add_json_config_to_registry(self, name, rev, branch): + sources = """ + "test-repo": { + "git-remote": { + "remotes": { + "origin": { + "uri": "file://%s" + } + }, + "branch": "%s", + "rev": "%s" + } + } +""" % (self.testrepopath, branch, rev) + return self._add_json_config_to_registry_helper(name, sources) + def add_file_to_testrepo(self, name, content, script=False): fullname = os.path.join(self.testrepopath, name) os.makedirs(os.path.join(self.testrepopath, os.path.dirname(name)), exist_ok=True)
Signature · 1 line
2.47.3
PATCH 8/10 bitbake-setup: symlink json with fixed revisions into layers/
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
This makes the file more visible for users, and allows easier checks of layers current content against what was checked out by bitbake-setup (to ensure local modifications aren't wiped out).
--- bin/bitbake-setup | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
bin/bitbake-setup +6 -2
@@ -69,11 +69,15 @@ def write_upstream_config(config_dir, config_data): with open(os.path.join(config_dir, "config-upstream.json"),'w') as s: json.dump(config_data, s, sort_keys=True, indent=4) -def write_sources_fixed_revisions(config_dir, config_data): +def write_sources_fixed_revisions(config_dir, layer_dir, config_data): + json_path = os.path.join(config_dir, "sources-fixed-revisions.json") + json_link = os.path.join(layer_dir, "sources-fixed-revisions.json") sources = {} sources['sources'] = config_data with open(os.path.join(config_dir, "sources-fixed-revisions.json"),'w') as s: json.dump(sources, s, sort_keys=True, indent=4) + if not os.path.lexists(json_link): + os.symlink(os.path.relpath(json_path ,layer_dir), json_link) def commit_config(config_dir): bb.process.run("git -C {} add .".format(config_dir))
@@ -344,7 +348,7 @@ def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt" bitbake_config = config["bitbake-config"] thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf) - write_sources_fixed_revisions(confdir, sources_fixed_revisions) + write_sources_fixed_revisions(confdir, layerdir, sources_fixed_revisions) commit_config(confdir) def int_input(allowed_values, prompt=''):
Signature · 1 line
2.47.3
Re: [bitbake-devel] [PATCH 08/10] bitbake-setup: symlink json with fixed revisions into layers/
Just hit the case fixed by this commit,
...
File "/workspaces/ls/bitbake/bin/bitbake-setup", line 121, in_has_local_modifications
fixed_revisions = json.load(open(os.path.join(layerdir,
"sources-fixed-revisions.json")))
FileNotFoundError: [Errno 2] No such file or directory:
'/workspaces/ls/bitbake-setup/ls-master/stm32-sdk-stm32h503/la yers/sources-fixed-revisions.json
...
Thanks!,
Anibal
Quoted 61 lines · Alexander Kanavin via lists.openembedded.org
From: Alexander Kanavin <[email protected]>
This makes the file more visible for users, and allows easier checks of layers current content against what was checked out by bitbake-setup (to ensure local modifications aren't wiped out).
Signed-off-by Alexander Kanavin <[email protected]>--- bin/bitbake-setup | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)bin/bitbake-setup +3
-1@@ -69,11 +69,15 @@ def write_upstream_config(config_dir, config_data): with open(os.path.join(config_dir, "config-upstream.json"),'w') as s: json.dump(config_data, s, sort_keys=True, indent=4) -def write_sources_fixed_revisions(config_dir, config_data): +def write_sources_fixed_revisions(config_dir, layer_dir, config_data): + json_path = os.path.join(config_dir, "sources-fixed-revisions.json") + json_link = os.path.join(layer_dir, "sources-fixed-revisions.json") sources = {} sources['sources'] = config_data with open(os.path.join(config_dir,"sources-fixed-revisions.json"),'w') as s:
json.dump(sources, s, sort_keys=True, indent=4)bin/bitbake-setup +2
-0+ if not os.path.lexists(json_link): + os.symlink(os.path.relpath(json_path ,layer_dir), json_link) def commit_config(config_dir): bb.process.run("git -C {} add .".format(config_dir))@@ -344,7 +348,7 @@ def update_build(config, confdir, setupdir, layerdir,d, update_bb_conf="prompt"
bitbake_config= config["bitbake-config"]
thisdir =os.path.dirname(config["path"])if config["type"] ==
'local' else None
setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir,update_bb_conf)bin/bitbake-setup +1
-1- write_sources_fixed_revisions(confdir, sources_fixed_revisions) + write_sources_fixed_revisions(confdir, layerdir,
sources_fixed_revisions)
commit_config(confdir)def
int_input(allowed_values, prompt=''):
--
2.47.3-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#18678):
https://lists.openembedded.org/g/bitbake-devel/message/18678
Mute This Topic: https://lists.openembedded.org/mt/117046548/8181911
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [bitbake-devel] [PATCH 08/10] bitbake-setup: symlink json with fixed revisions into layers/
Quoted 9 lines · Anibal Limon
Just hit the case fixed by this commit,
...
File "/workspaces/ls/bitbake/bin/bitbake-setup", line 121, in_has_local_modifications
fixed_revisions=json.load(open(os.path.join(layerdir, "sources-fixed-revisions.json")))
FileNotFoundError: [Errno 2] No such file or directory: '/workspaces/ls/bitbake-setup/ls-master/stm32-sdk-stm32h503/layers/sources-fixed-revisions.json
...
No, this is an issue with the patchset. The _has_local_modifications() function expects the file in layers/ but the file is only there if the build was set up by a version of bitbake-setup that has the patchset. So bitbake-setup needs to read the file from config/ to maintain compatibility with existing version in master. I'll come up with a fixup.
Alex
PATCH 9/10 doc: document fixed revisions override in bitbake-setup manual
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
--- .../bitbake-user-manual-environment-setup.rst | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+)
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +46 -0
@@ -938,3 +938,49 @@ Here is an example file that overrides the branch of the BitBake repository to - The ``sources`` section contains the same options as the ``sources`` option of a :term:`Generic Configuration` file. See the :ref:`ref-bbsetup-section-config-reference` section for more information. + +.. _ref-bbsetup-fixed-revisions + +Fixed source revisions +====================== + +:term:`Generic Configuration` can set source revisions in ``rev`` to a tag or a branch. +Bitbake-setup will make sure the actual revision will match the tag or branch when performing +initializations or updates, and will capture the revisions in a :ref:`_ref-bbsetup-source-overrides` +file. + +The file is named ``sources-fixed-revisions.json`` and is available in ``layers/`` +or ``config/`` under :term:`Setup` directory. It can be used to keep record of +what was checked out when using configurations that do not specify exact revisions. + +Also, as it is an override file, it can be combined with the original configurations +to initialize a setup in a reproducible way that guarantees an exact, never-changing +set of revisions. + +For example if the original configuration had specified only a master branch +for a source:: + + "bitbake": { + "git-remote": { + "remotes": { + "origin": { + "uri": "https://git.openembedded.org/bitbake" + } + }, + "branch": "master", + "rev": "master" + } + + +the override will contain the exact revisions:: + + "bitbake": { + "git-remote": { + "remotes": { + "origin": { + "uri": "https://git.openembedded.org/bitbake" + } + }, + "branch": "master", + "rev": "720df1a53452983c1c832f624490e255cf389204" + }
Signature · 1 line
2.47.3
Re: [bitbake-devel] [PATCH 09/10] doc: document fixed revisions override in bitbake-setup manual
Hi,
Quoted 12 lines · Alexander Kanavin via lists.openembedded.org
From: Alexander Kanavin <[email protected]>
Signed-off-by Alexander Kanavin <[email protected]>--- .../bitbake-user-manual-environment-setup.rst | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+)doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +0
-0
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst Here is an example file that overrides the branch of the BitBake repository to · line 942
Quoted 5 lines
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +2
-0@@ -938,3 +938,49 @@ Here is an example file that overrides the branch of the BitBake repository to - The ``sources`` section contains the same options as the ``sources`` option of a :term:`Generic Configuration` file. See the :ref:`ref-bbsetup-section-config-reference` section for more information. + +.. _ref-bbsetup-fixed-revisions
Missing colon at the end:
.. _ref-bbsetup-fixed-revisions:
otherwise this is treated as a comment.
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst Here is an example file that overrides the branch of the BitBake repository to · line 952
Quoted 10 lines
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +10
-0+ +Fixed source revisions +====================== + +:term:`Generic Configuration` can set source revisions in ``rev`` to a tag or a branch. +Bitbake-setup will make sure the actual revision will match the tag or branch when performing +initializations or updates, and will capture the revisions in a :ref:`_ref-bbsetup-source-overrides` +file. + +The file is named ``sources-fixed-revisions.json`` and is available in ``layers/``
s/The/This/ ?
if this is a continuation to your previous sentence
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst Here is an example file that overrides the branch of the BitBake repository to · line 953
Quoted 1 line
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +1
-0+or ``config/`` under :term:`Setup` directory. It can be used to keep record of
s/under :term:Setup directory/under the :term:Setup directory/
Why layers/ _or_ config/? It's ambiguous for users reading here what the difference between the two might be. If this is a compatibility thing (because bitbake-setup used to do it one way, then it was changed to another location), I would just keep the final one documented here (or explain that it's for compatibility).
Quoted 23 lines
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +23
-0+what was checked out when using configurations that do not specify exact revisions. + +Also, as it is an override file, it can be combined with the original configurations +to initialize a setup in a reproducible way that guarantees an exact, never-changing +set of revisions. + +For example if the original configuration had specified only a master branch +for a source:: + + "bitbake": { + "git-remote": { + "remotes": { + "origin": { + "uri": "https://git.openembedded.org/bitbake" + } + }, + "branch": "master", + "rev": "master" + } + + +the override will contain the exact revisions:: +
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst Here is an example file that overrides the branch of the BitBake repository to · line 986
Quoted 10 lines
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +10
-0+ "bitbake": { + "git-remote": { + "remotes": { + "origin": { + "uri": "https://git.openembedded.org/bitbake" + } + }, + "branch": "master", + "rev": "720df1a53452983c1c832f624490e255cf389204" + }
"""
This file can later be used with the ``--source-overrides`` option of the :ref:ref-bbsetup-command-init command.
"""
?
Thanks,
Antonin
Signature · 3 lines
Antonin Godard, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Re: [bitbake-devel] [PATCH 09/10] doc: document fixed revisions override in bitbake-setup manual
Thanks, I corrected all of these and sent a v2.
Alex
Quoted 102 lines · Antonin Godard
Hi,
Quoted 12 lines · Alexander Kanavin via lists.openembedded.org
From: Alexander Kanavin <[email protected]>
Signed-off-by Alexander Kanavin <[email protected]>--- .../bitbake-user-manual-environment-setup.rst | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+)doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +0
-0doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst Here is an example file that overrides the branch of the BitBake repository to · line 942
Quoted 5 lines
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +2
-0@@ -938,3 +938,49 @@ Here is an example file that overrides the branch of the BitBake repository to - The ``sources`` section contains the same options as the ``sources`` option of a :term:`Generic Configuration` file. See the :ref:`ref-bbsetup-section-config-reference` section for more information. + +.. _ref-bbsetup-fixed-revisionsMissing colon at the end:
.. _ref-bbsetup-fixed-revisions:
otherwise this is treated as a comment.
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst Here is an example file that overrides the branch of the BitBake repository to · line 952
Quoted 10 lines
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +10
-0+ +Fixed source revisions +====================== + +:term:`Generic Configuration` can set source revisions in ``rev`` to a tag or a branch. +Bitbake-setup will make sure the actual revision will match the tag or branch when performing +initializations or updates, and will capture the revisions in a :ref:`_ref-bbsetup-source-overrides` +file. + +The file is named ``sources-fixed-revisions.json`` and is available in ``layers/``s/The/This/ ?
if this is a continuation to your previous sentence
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst Here is an example file that overrides the branch of the BitBake repository to · line 953
Quoted 1 line
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +1
-0+or ``config/`` under :term:`Setup` directory. It can be used to keep record ofs/under :term:
Setupdirectory/under the :term:Setupdirectory/Why layers/ _or_ config/? It's ambiguous for users reading here what the difference between the two might be. If this is a compatibility thing (because bitbake-setup used to do it one way, then it was changed to another location), I would just keep the final one documented here (or explain that it's for compatibility).
Quoted 23 lines
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +23
-0+what was checked out when using configurations that do not specify exact revisions. + +Also, as it is an override file, it can be combined with the original configurations +to initialize a setup in a reproducible way that guarantees an exact, never-changing +set of revisions. + +For example if the original configuration had specified only a master branch +for a source:: + + "bitbake": { + "git-remote": { + "remotes": { + "origin": { + "uri": "https://git.openembedded.org/bitbake" + } + }, + "branch": "master", + "rev": "master" + } + + +the override will contain the exact revisions:: +doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst Here is an example file that overrides the branch of the BitBake repository to · line 986
Quoted 10 lines
doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +10
-0+ "bitbake": { + "git-remote": { + "remotes": { + "origin": { + "uri": "https://git.openembedded.org/bitbake" + } + }, + "branch": "master", + "rev": "720df1a53452983c1c832f624490e255cf389204" + }"""
This file can later be used with the ``--source-overrides`` option of the :ref:ref-bbsetup-command-initcommand.
"""?
Thanks,
Antonin--
Antonin Godard, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
PATCH 10/10 bitbake-setup: correct (and test) several scenarios in layer checkout
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
There were untested scenarios in layer updates which did not work properly.
The most serious of them is updating a remote layer which had user-made local modifications (e.g. newly made local commits or simply edited files without a commit). Bitbake-setup relies on git fetcher's unpack operation which simply wipes it all out. Yes, the user should've used -L option to symlink a local clone from elsewhere; this doesn't help when their work is gone.
The commit adds a function that checks if such modifications have been made, and a code that moves the layer into a backup location if the function returns true. I considered asking the user what to do (similar to bitbake config handling), but this would've resulted in a half-updated, possibly broken layer set which ultimately just would be more confusing.
The above check is a part of a broader logic block that handles already existing sources, and in particular addresses another issue: when a local source replaces a git remote or another local source, this scenario failed with 'file exists' errors (thanks to Anibal Limon for spotting that). In this case the original symlink or source tree is removed before the new one is unpacked from git or symlinked as a local source.
Adjust commit_config() to allow updates with no changes to configs: this can happen when local modifications to a layer are undone and the original layer commit is unpacked.
Add tests for all of this: both transitions between remote and local sources, and ensuring that layer backups do happen when that is expected.
--- bin/bitbake-setup | 30 ++++++++++++++++++-- lib/bb/tests/setup.py | 65 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 3 deletions(-)
bin/bitbake-setup +28 -2
@@ -81,7 +81,7 @@ def write_sources_fixed_revisions(config_dir, layer_dir, config_data): def commit_config(config_dir): bb.process.run("git -C {} add .".format(config_dir)) - bb.process.run("git -C {} commit --no-verify -a -m 'Configuration at {}'".format(config_dir, time.asctime())) + bb.process.run("git -C {} commit --allow-empty --no-verify -a -m 'Configuration at {}'".format(config_dir, time.asctime())) def _write_layer_list(dest, repodirs): layers = []
@@ -117,6 +117,17 @@ def checkout_layers(layers, layerdir, d): logger.plain("Making a symbolic link {} pointing to {}".format(dst, src)) os.symlink(src, dst) + def _has_local_modifications(r_name, r_path): + fixed_revisions = json.load(open(os.path.join(layerdir, "sources-fixed-revisions.json"))) + rev = fixed_revisions['sources'][r_name]['git-remote']['rev'] + status = bb.process.run('git -C {} status --porcelain'.format(r_path))[0] + if status: + return True + diff = bb.process.run('git -C {} diff {}'.format(r_path, rev))[0] + if diff: + return True + return False + layers_fixed_revisions = copy.deepcopy(layers) repodirs = [] oesetupbuild = None
@@ -130,10 +141,25 @@ def checkout_layers(layers, layerdir, d): r_local = r_data.get('local') if r_remote and r_local: raise Exception("Source {} contains both git-remote and local properties.".format(r_name)) + + repodir_path = os.path.join(layerdir, repodir) + if os.path.lexists(repodir_path): + if os.path.islink(repodir_path): + os.remove(repodir_path) + elif _has_local_modifications(r_name, repodir_path): + timestamp = time.strftime("%Y%m%d%H%M%S") + backup_path = repodir_path + '-backup.{}'.format(timestamp) + logger.warning("""Source {} in {} contains local modifications. Renaming to {} to preserve them. +For local development work it is recommended to clone the needed layers separately and use 'bitbake-setup init' with -L option to use them.""".format( + r_name, repodir_path, backup_path)) + os.rename(repodir_path, backup_path) + else: + shutil.rmtree(repodir_path) + if r_remote: _checkout_git_remote(r_remote, repodir, layers_fixed_revisions) if r_local: - _symlink_local(os.path.expanduser(r_local["path"]), os.path.join(layerdir,repodir)) + _symlink_local(os.path.expanduser(r_local["path"]), repodir_path) if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')): oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build')
lib/bb/tests/setup.py +64 -1
83 lines not on this page · raw message
Signature · 1 line
2.47.3
Re: [bitbake-devel] [PATCH 10/10] bitbake-setup: correct (and test) several scenarios in layer checkout
I tested the whole series + last patch to fix revisions path and it is working as expected.
Cheers!,
Anibal
Quoted 241 lines · Alexander Kanavin via lists.openembedded.org
From: Alexander Kanavin <[email protected]>
There were untested scenarios in layer updates which did not work properly.
The most serious of them is updating a remote layer which had user-made local
modifications (e.g. newly made local commits or simply edited files without a commit). Bitbake-setup relies on git fetcher's unpack operation which simply wipes it all out. Yes, the user should've used -L option to symlink a local clone from elsewhere; this doesn't help when their work is gone.The commit adds a function that checks if such modifications have been made, and a code that moves the layer into a backup location if the function
returns true. I considered asking the user what to do (similar to bitbake config handling), but this would've resulted in a half-updated, possibly broken layer set which ultimately just would be more confusing.The above check is a part of a broader logic block that handles already existing sources, and in particular addresses another issue: when a local source replaces a git remote or another local source, this scenario failed with 'file exists' errors (thanks to Anibal Limon for spotting that). In this case the original symlink or source tree is removed before the new one is unpacked from git or symlinked as a local source.
Adjust
commit_config()to allow updates with no changes to configs: this can
happen when local modifications to a layer are undone and the original layer
commit is unpacked.Add tests for all of this: both transitions between remote and local sources,
and ensuring that layer backups do happen when that is expected.Signed-off-by Alexander Kanavin <[email protected]>--- bin/bitbake-setup | 30 ++++++++++++++++++-- lib/bb/tests/setup.py | 65 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 3 deletions(-)bin/bitbake-setup +0
-0@@ -81,7 +81,7 @@ def write_sources_fixed_revisions(config_dir, layer_dir,
config_data):def
commit_config(config_dir):bb.process.run("git -C {} add .".format(config_dir))- bb.process.run("git -C {} commit --no-verify -a -m 'Configuration at
{}'".format(config_dir,time.asctime()))
+ bb.process.run("git -C {} commit --allow-empty --no-verify -a -m
'Configuration at {}'".format(config_dir,time.asctime()))def
_write_layer_list(dest, repodirs):layers = []bin/bitbake-setup +20
-1@@ -117,6 +117,17 @@ def checkout_layers(layers, layerdir, d): logger.plain("Making a symbolic link {} pointing to {}".format(dst, src)) os.symlink(src, dst) + def _has_local_modifications(r_name, r_path): + fixed_revisions = json.load(open(os.path.join(layerdir, "sources-fixed-revisions.json"))) + rev = fixed_revisions['sources'][r_name]['git-remote']['rev'] + status = bb.process.run('git -C {} status --porcelain'.format(r_path))[0] + if status: + return True + diff = bb.process.run('git -C {} diff {}'.format(r_path, rev))[0] + if diff: + return True + return False + layers_fixed_revisions = copy.deepcopy(layers) repodirs = [] oesetupbuild = None@@ -130,10 +141,25 @@ def checkout_layers(layers, layerdir, d): r_local = r_data.get('local') if r_remote and r_local: raise Exception("Source {} contains both git-remote and local properties.".format(r_name)) + + repodir_path = os.path.join(layerdir, repodir) + if os.path.lexists(repodir_path): + if os.path.islink(repodir_path): + os.remove(repodir_path) + elif _has_local_modifications(r_name, repodir_path): + timestamp = time.strftime("%Y%m%d%H%M%S") + backup_path = repodir_path + '-backup.{}'.format(timestamp) + logger.warning("""Source {} in {} contains localmodifications. Renaming to {} to preserve them.
+For local development work it is recommended to clone the needed layers separately and use 'bitbake-setup init' with -L option to use them.""".format(bin/bitbake-setup +5
-0+ r_name, repodir_path, backup_path)) + os.rename(repodir_path, backup_path) + else: + shutil.rmtree(repodir_path) + if r_remote: _checkout_git_remote(r_remote, repodir,
layers_fixed_revisions)
ifr_local:
- _symlink_local(os.path.expanduser(r_local["path"]),os.path.join(layerdir,repodir))
+ _symlink_local(os.path.expanduser(r_local["path"]),repodir_path)if os.path.exists(os.path.join(layerdir, repodir,
'scripts/oe-setup-build')):
oesetupbuild = os.path.join(layerdir, repodir,
'scripts/oe-setup-build')lib/bb/tests/setup.py +10
-0@@ -175,6 +175,16 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) """ % (self.testrepopath, branch, rev) return self._add_json_config_to_registry_helper(name, sources) + def add_local_json_config_to_registry(self, name, path): + sources = """ + "test-repo": { + "local": { + "path": "%s" + } + } +""" % (path) + return self._add_json_config_to_registry_helper(name, sources) + def add_file_to_testrepo(self, name, content, script=False): fullname = os.path.join(self.testrepopath, name) os.makedirs(os.path.join(self.testrepopath,
os.path.dirname(name)), exist_ok=True)lib/bb/tests/setup.py +0
-1@@ -413,7 +423,6 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) sums_after = _conf_chksum(f"{setuppath}/build/conf") self.assertEqual(sums_before, sums_after) - # check source overrides, local sources provided with symlinks,and custom setup dir name
def_check_local_sources(custom_setup_dir):
custom_setup_path= os.path.join(self.tempdir,
'bitbake-builds',custom_setup_dir)
custom_layer_path= os.path.join(custom_setup_path, 'layers',
'test-repo')lib/bb/tests/setup.py +1
-0@@ -421,6 +430,60 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"])) self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) self.config_is_unchanged(custom_setup_path) + # Change the configuration to refer to a local source, then toanother local source, then back to a git remote
+ # Run status/update after each change and verify that nothing
breakslib/bb/tests/setup.py +5
-0+ c = 'gadget' + setuppath = self.get_setup_path('test-config-1', c) + self.config_is_unchanged(setuppath) + + json_1 =
self.add_local_json_config_to_registry('test-config-1.conf.json', self.testrepopath)lib/bb/tests/setup.py +7
-0+ os.environ['BBPATH'] = os.path.join(setuppath, 'build') + out = self.runbbsetup("update --update-bb-conf='yes'") + _check_local_sources(setuppath) + + prev_path = self.testrepopath + self.testrepopath = prev_path + "-2" + self.git("clone {} {}".format(prev_path, self.testrepopath),cwd=self.tempdir)
+json_1=self.add_local_json_config_to_registry('test-config-1.conf.json', self.testrepopath)lib/bb/tests/setup.py +6
-0+ os.environ['BBPATH'] = os.path.join(setuppath, 'build') + out = self.runbbsetup("update --update-bb-conf='yes'") + _check_local_sources(setuppath) + + self.testrepopath = prev_path + json_1 =
self.add_json_config_to_registry('test-config-1.conf.json', branch, branch)lib/bb/tests/setup.py +5
-0+ os.environ['BBPATH'] = os.path.join(setuppath, 'build') + out = self.runbbsetup("update --update-bb-conf='yes'") + self.check_setupdir_files(setuppath, test_file_content) + + # Also check that there are no layer backups up to this point,then make a change that should
lib/bb/tests/setup.py +5
-0+ # result in a layer backup, and check that it does happen. + def _check_layer_backups(layer_path, expected_backups): + files = os.listdir(layer_path) + backups = len([f for f in files if 'backup' in f]) + self.assertEqual(backups, expected_backups, msg = "Expected{} layer backups, got {}, directory listing: {}".format(expected_backups, backups, files))
lib/bb/tests/setup.py +7
-0+ + layers_path = os.path.join(setuppath, 'layers') + layer_path = os.path.join(layers_path, 'test-repo') + _check_layer_backups(layers_path, 0) + + ## edit a file without making a commit + with open(os.path.join(layer_path, 'local-modification'), 'w') asf:
lib/bb/tests/setup.py +9
-0+ f.write('locally-modified\n') + test_file_content = "modified-again\n" + self.add_file_to_testrepo('test-file', test_file_content) + os.environ['BBPATH'] = os.path.join(setuppath, 'build') + out = self.runbbsetup("update --update-bb-conf='yes'") + _check_layer_backups(layers_path, 1) + + ## edit a file and make a commit + with open(os.path.join(layer_path, 'local-modification'), 'w') asf:
lib/bb/tests/setup.py +3
-0+ f.write('locally-modified-again\n') + self.git('add .', cwd=layer_path) + self.git('commit -m "Adding a local modification"',cwd=layer_path)
lib/bb/tests/setup.py +4
-0+ out = self.runbbsetup("update --update-bb-conf='yes'") + _check_layer_backups(layers_path, 2) + + # check source overrides, local sources provided with symlinks,and custom setup dir name
source_override_content= """{ "sources": {--
2.47.3-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#18680):
https://lists.openembedded.org/g/bitbake-devel/message/18680
Mute This Topic: https://lists.openembedded.org/mt/117046550/8181911
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-