package_rpm/archiver: Apply bandaid to src.rpm creation
PATCH 1/2 package_rpm/archiver: Apply bandaid to src.rpm creation
| error: create archive failed on file /media/build1/poky/build/tmp/work/all-poky-linux/xcursor-transparent-theme/0.1.1+gitAUTOINC+23c8af5ba4-r0/deploy-sources/allarch-poky-linux/xcursor-transparent-theme-0.1.1+gitAUTOINC+23c8af5ba4-r0/xcursor-transparent-theme-0.1.1+git0+23c8af5ba4-r0.src.rpm: cpio: read failed - No such file or directory | Building target platforms: noarch-poky-linux | Building for target noarch-poky-linux
This is caused by:
$ cat log.task_order
do_cleansstate (24289): log.do_cleansstate.24289 do_deploy_archives_setscene (24395): log.do_deploy_archives_setscene.24395 do_fetch (24407): log.do_fetch.24407
[..]do_package_write_rpm (25448): log.do_package_write_rpm.25448 do_package_qa (25451): log.do_package_qa.25451
So do_deploy_archives can run from sstate, created a .src.rpm in WORKDIR/deploy-sources, then it was removed when rpm was running. This leads to a broken Source line in the spec file as the original file was found by the os.listdir().
This fix is just a bandaid over much more fundamental problems sadly.
--- meta/classes/package_rpm.bbclass | 4 ++++ 1 file changed, 4 insertions(+)
meta/classes/package_rpm.bbclass +4 -0
@@ -113,6 +113,10 @@ python write_specfile () { source_list = os.listdir(ar_outdir) source_number = 0 for source in source_list: + # do_deploy_archives may have already run (from sstate) meaning a .src.rpm may already + # exist in ARCHIVER_OUTDIR so skip if present. + if source.endswith(".src.rpm"): + continue # The rpmbuild doesn't need the root permission, but it needs # to know the file's user and group name, the only user and # group in fakeroot is "root" when working in fakeroot.
Signature · 1 line
2.20.1
PATCH 2/2 archiver/package_rpm: Drop srpm source archiver mode
Creating these 'srpms' misleads users into thinking there is a way of rebuilding the package when there isn't. They amount to a glorified tarball. Worse, the code is full of races. In particular, do_package_write_rpm is writing out files outside of its sstate meaning the source archives may sometimes appear and sometimes disappear.
In my view the simplest way of dealing with this is to admit these are not source rpms and stop creating them. The tarballs from the source archiver are perfectly reasonable.
--- meta/classes/archiver.bbclass | 24 +---------- meta/classes/package_rpm.bbclass | 43 -------------------- meta/lib/oeqa/selftest/cases/archiver.py | 13 ------ meta/lib/oeqa/selftest/cases/buildoptions.py | 15 ------- 4 files changed, 2 insertions(+), 93 deletions(-)
meta/classes/archiver.bbclass +2 -22
@@ -12,14 +12,12 @@ # 5) The environment data, similar to 'bitbake -e recipe': # ARCHIVER_MODE[dumpdata] = "1" # 6) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1" -# 7) Whether output the .src.rpm package: -# ARCHIVER_MODE[srpm] = "1" -# 8) Filter the license, the recipe whose license in +# 7) Filter the license, the recipe whose license in # COPYLEFT_LICENSE_INCLUDE will be included, and in # COPYLEFT_LICENSE_EXCLUDE will be excluded. # COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*' # COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary' -# 9) The recipe type that will be archived: +# 8) The recipe type that will be archived: # COPYLEFT_RECIPE_TYPES = 'target' #
@@ -27,7 +25,6 @@ COPYLEFT_RECIPE_TYPES ?= 'target native nativesdk cross crosssdk cross-canadian' inherit copyleft_filter -ARCHIVER_MODE[srpm] ?= "0" ARCHIVER_MODE[src] ?= "patched" ARCHIVER_MODE[diff] ?= "0" ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
@@ -112,23 +109,6 @@ python () { if ar_recipe == "1": d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_recipe' % pn) - - # Output the SRPM package - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == "1" and d.getVar('PACKAGES'): - if "package_rpm" in d.getVar('PACKAGE_CLASSES'): - d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_package_write_rpm' % pn) - if ar_dumpdata == "1": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_dumpdata' % pn) - if ar_recipe == "1": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_recipe' % pn) - if ar_src == "original": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_original' % pn) - elif ar_src == "patched": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_patched' % pn) - elif ar_src == "configured": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_configured' % pn) - else: - bb.fatal("ARCHIVER_MODE[srpm] needs package_rpm in PACKAGE_CLASSES") } # Take all the sources for a recipe and puts them in WORKDIR/archiver-work/.
meta/classes/package_rpm.bbclass +0 -43
73 lines not on this page · raw message
meta/lib/oeqa/selftest/cases/archiver.py +0 -13
@@ -116,16 +116,3 @@ class Archiver(OESelftestTestCase): excluded_present = len(glob.glob(src_path_target + '/%s-*' % target_recipes[1])) self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % target_recipes[1]) - - - - def test_archiver_srpm_mode(self): - """ - Test that in srpm mode, the added recipe dependencies at least exist/work [YOCTO #11121] - """ - - features = 'INHERIT += "archiver"\n' - features += 'ARCHIVER_MODE[srpm] = "1"\n' - self.write_config(features) - - bitbake('-n core-image-sato')
meta/lib/oeqa/selftest/cases/buildoptions.py +0 -15
@@ -152,21 +152,6 @@ class BuildhistoryTests(BuildhistoryBase): self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True) self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True, error_regex=error) -class ArchiverTest(OESelftestTestCase): - @OETestID(926) - def test_arch_work_dir_and_export_source(self): - """ - Test for archiving the work directory and exporting the source files. - """ - self.write_config("INHERIT += \"archiver\"\nARCHIVER_MODE[src] = \"original\"\nARCHIVER_MODE[srpm] = \"1\"") - res = bitbake("xcursor-transparent-theme", ignore_status=True) - self.assertEqual(res.status, 0, "\nCouldn't build xcursortransparenttheme.\nbitbake output %s" % res.output) - deploy_dir_src = get_bb_var('DEPLOY_DIR_SRC') - pkgs_path = g.glob(str(deploy_dir_src) + "/allarch*/xcurs*") - src_file_glob = str(pkgs_path[0]) + "/xcursor*.src.rpm" - tar_file_glob = str(pkgs_path[0]) + "/xcursor*.tar.gz" - self.assertTrue((g.glob(src_file_glob) and g.glob(tar_file_glob)), "Couldn't find .src.rpm and .tar.gz files under %s/allarch*/xcursor*" % deploy_dir_src) - class ToolchainOptions(OESelftestTestCase): def test_toolchain_fortran(self):
Signature · 1 line
2.20.1
Re: [PATCH 2/2] archiver/package_rpm: Drop srpm source archiver mode
Quoted 9 lines · Richard Purdie
Creating these 'srpms' misleads users into thinking there is a way of rebuilding the package when there isn't. They amount to a glorified tarball. Worse, the code is full of races. In particular,
do_package_write_rpmis writing out files outside of its sstate meaning the source archives may sometimes appear and sometimes disappear.In my view the simplest way of dealing with this is to admit these are not source rpms and stop creating them. The tarballs from the source archiver are perfectly reasonable.
To start with, yes SRPMs are absolutely a glorified archive and are NOT expected to be buildable out of the build system.
However, we do have users who want SRMS output format for some reason. I suspect the reason for this is that they have additional tooling that processing both the RPMS and SRPMS to ensure that they have meet license terms by releasing corresponding sources to any binary packages they may be publishing for package-based upgrade situations.
Most of the users of this that I know of are using it specifically for:
- package metadata in the RPM format
- include original unmodified source and patches
- include run files for each 'operation'
- include log files for the output of each operation
What I thought was happening was:
do_fetchdo_unpack
...do_packagedo_archiverdo_package_write_rpmdo_deploy_archives
So in the sstate case you could end up skipping all the way to the do_package (which would have the problem of the build instructions and build output being missing.
So even in an sstate case you'd end up with something like:
do_<setscene operations>
do_<archiver setscene operations>
<some task here that has to be re-run>do_packagedo_archiver (and maybe scene operations here to restore logs/run files) do_package_write_rpmdo_deploy_archives
Clearly that is not happening based on your previous message.
If we can generate the archiver tarball with a consistent set of contents in parallel with the do_package operation, then we SHOULD be able to adjust it to also be fed into the package generation in do_package_write_rpm as well.
I don't think it matters if the input to the do_package_write_rpm is an archiver tarball or the individual items BTW. In the end, if the archiver works and can be run -before- the do_package_write_rpm, then we should be able to continue to do this for those people who need it. (I see no other purpose for the SRPM other then external tooling of some kind trying to either automatically download matching SRPMs, or doing validation that each binary package has a corresponding SRPM.)
We can take this offline if you'd like. I won't have time in the short term (next two weeks) to help with the archiver unfortunately.. but if the issues with that can be corrected, then I can certainly help get the SRPM part re-working.
--Mark
Quoted 191 lines
Signed-off-by Richard Purdie <[email protected]>--- meta/classes/archiver.bbclass | 24 +---------- meta/classes/package_rpm.bbclass | 43 -------------------- meta/lib/oeqa/selftest/cases/archiver.py | 13 ------ meta/lib/oeqa/selftest/cases/buildoptions.py | 15 ------- 4 files changed, 2 insertions(+), 93 deletions(-)meta/classes/archiver.bbclass +2
-22@@ -12,14 +12,12 @@ # 5) The environment data, similar to 'bitbake -e recipe': # ARCHIVER_MODE[dumpdata] = "1" # 6) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1" -# 7) Whether output the .src.rpm package: -# ARCHIVER_MODE[srpm] = "1" -# 8) Filter the license, the recipe whose license in +# 7) Filter the license, the recipe whose license in # COPYLEFT_LICENSE_INCLUDE will be included, and in # COPYLEFT_LICENSE_EXCLUDE will be excluded. # COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*' # COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary' -# 9) The recipe type that will be archived: +# 8) The recipe type that will be archived: # COPYLEFT_RECIPE_TYPES = 'target' #@@ -27,7 +25,6 @@ COPYLEFT_RECIPE_TYPES ?= 'target native nativesdk cross crosssdk cross-canadian' inherit copyleft_filter -ARCHIVER_MODE[srpm] ?= "0" ARCHIVER_MODE[src] ?= "patched" ARCHIVER_MODE[diff] ?= "0" ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"@@ -112,23 +109,6 @@ python () { if ar_recipe == "1": d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_recipe' % pn) - - # Output the SRPM package - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == "1" and d.getVar('PACKAGES'): - if "package_rpm" in d.getVar('PACKAGE_CLASSES'): - d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_package_write_rpm' % pn) - if ar_dumpdata == "1": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_dumpdata' % pn) - if ar_recipe == "1": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_recipe' % pn) - if ar_src == "original": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_original' % pn) - elif ar_src == "patched": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_patched' % pn) - elif ar_src == "configured": - d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_configured' % pn) - else: - bb.fatal("ARCHIVER_MODE[srpm] needs package_rpm in PACKAGE_CLASSES") } # Take all the sources for a recipe and puts them in WORKDIR/archiver-work/.meta/classes/package_rpm.bbclass +0
-43@@ -97,34 +97,6 @@ def write_rpm_perfiledata(srcname, d): python write_specfile () { import oe.packagedata - # append information for logs and patches to %prep - def add_prep(d,spec_files_bottom): - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): - spec_files_bottom.append('%%prep -n %s' % d.getVar('PN') ) - spec_files_bottom.append('%s' % "echo \"include logs and patches, Please check them in SOURCES\"") - spec_files_bottom.append('') - - # append the name of tarball to key word 'SOURCE' in xxx.spec. - def tail_source(d): - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): - ar_outdir = d.getVar('ARCHIVER_OUTDIR') - if not os.path.exists(ar_outdir): - return - source_list = os.listdir(ar_outdir) - source_number = 0 - for source in source_list: - # do_deploy_archives may have already run (from sstate) meaning a .src.rpm may already - # exist in ARCHIVER_OUTDIR so skip if present. - if source.endswith(".src.rpm"): - continue - # The rpmbuild doesn't need the root permission, but it needs - # to know the file's user and group name, the only user and - # group in fakeroot is "root" when working in fakeroot. - f = os.path.join(ar_outdir, source) - os.chown(f, 0, 0) - spec_preamble_top.append('Source%s: %s' % (source_number, source)) - source_number += 1 - # In RPM, dependencies are of the format: pkg <>= Epoch:Version-Release # This format is similar to OE, however there are restrictions on the # characters that can be in a field. In the Version field, "-"@@ -386,11 +358,6 @@ python write_specfile () { # Gather special src/first package data if srcname == splitname: - archiving = d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and \ - bb.data.inherits_class('archiver', d) - if archiving and srclicense != splitlicense: - bb.warn("The SRPM produced may not have the correct overall source license in the License tag. This is due to the LICENSE for the primary package and SRPM conflicting.") - srclicense = splitlicense srcrdepends = splitrdepends srcrrecommends = splitrrecommends@@ -515,7 +482,6 @@ python write_specfile () { del localdata - add_prep(d,spec_files_bottom) spec_preamble_top.append('Summary: %s' % srcsummary) spec_preamble_top.append('Name: %s' % srcname) spec_preamble_top.append('Version: %s' % srcversion)@@ -529,7 +495,6 @@ python write_specfile () { spec_preamble_top.append('URL: %s' % srchomepage) if srccustomtagschunk: spec_preamble_top.append(srccustomtagschunk) - tail_source(d) # Replaces == Obsoletes && Provides robsoletes = bb.utils.explode_dep_versions2(srcrobsoletes or "")@@ -692,14 +657,6 @@ python do_package_rpm () { cmd = cmd + " --define '_unpackaged_files_terminate_build 0'" cmd = cmd + " --define 'debug_package %{nil}'" cmd = cmd + " --define '_tmppath " + workdir + "'" - if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d): - cmd = cmd + " --define '_sourcedir " + d.getVar('ARCHIVER_OUTDIR') + "'" - cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_OUTDIR') + "'" - cmdsrpm = cmdsrpm + " -bs " + outspecfile - # Build the .src.rpm - d.setVar('SBUILDSPEC', cmdsrpm + "\n") - d.setVarFlag('SBUILDSPEC', 'func', '1') - bb.build.exec_func('SBUILDSPEC', d) cmd = cmd + " -bb " + outspecfile # rpm 4 creates various empty directories in _topdir, let's clean them upmeta/lib/oeqa/selftest/cases/archiver.py +0
-13@@ -116,16 +116,3 @@ class Archiver(OESelftestTestCase): excluded_present = len(glob.glob(src_path_target + '/%s-*' % target_recipes[1])) self.assertFalse(excluded_present, 'Recipe %s was not excluded.' % target_recipes[1]) - - - - def test_archiver_srpm_mode(self): - """ - Test that in srpm mode, the added recipe dependencies at least exist/work [YOCTO #11121] - """ - - features = 'INHERIT += "archiver"\n' - features += 'ARCHIVER_MODE[srpm] = "1"\n' - self.write_config(features) - - bitbake('-n core-image-sato')meta/lib/oeqa/selftest/cases/buildoptions.py +0
-15@@ -152,21 +152,6 @@ class BuildhistoryTests(BuildhistoryBase): self.run_buildhistory_operation(target, target_config="PR = \"r1\"", change_bh_location=True) self.run_buildhistory_operation(target, target_config="PR = \"r0\"", change_bh_location=False, expect_error=True, error_regex=error) -class ArchiverTest(OESelftestTestCase): - @OETestID(926) - def test_arch_work_dir_and_export_source(self): - """ - Test for archiving the work directory and exporting the source files. - """ - self.write_config("INHERIT += \"archiver\"\nARCHIVER_MODE[src] = \"original\"\nARCHIVER_MODE[srpm] = \"1\"") - res = bitbake("xcursor-transparent-theme", ignore_status=True) - self.assertEqual(res.status, 0, "\nCouldn't build xcursortransparenttheme.\nbitbake output %s" % res.output) - deploy_dir_src = get_bb_var('DEPLOY_DIR_SRC') - pkgs_path = g.glob(str(deploy_dir_src) + "/allarch*/xcurs*") - src_file_glob = str(pkgs_path[0]) + "/xcursor*.src.rpm" - tar_file_glob = str(pkgs_path[0]) + "/xcursor*.tar.gz" - self.assertTrue((g.glob(src_file_glob) and g.glob(tar_file_glob)), "Couldn't find .src.rpm and .tar.gz files under %s/allarch*/xcursor*" % deploy_dir_src) - class ToolchainOptions(OESelftestTestCase): def test_toolchain_fortran(self):