package.bbclass: decouple splitting and stripping
PATCH 1/1 package.bbclass: decouple splitting and stripping
Fix logic in split_and_strip_files() to allow splitting or stripping independently. We also return quickly from this function if we have nothing to do. We seek the following behavior:
Strip / Split Behavior yes / yes binaries stripped; debug info and source in -dbg no / yes debug info and source in -dbg yes / no binaries stripped; -dbg packages empty no / no -dbg packages empty (not a very useful case)
Currently, no/yes does not work and is the same as no/no.
--- meta/classes/package.bbclass | 108 ++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 51 deletions(-)
meta/classes/package.bbclass +57 -51
@@ -812,6 +812,12 @@ python fixup_perms () { } python split_and_strip_files () { + + stripping = d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1' + splitting = d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT', True) != '1' + if not (stripping or splitting): + return + import stat, errno dvar = d.getVar('PKGD', True)
@@ -879,64 +885,64 @@ python split_and_strip_files () { kernmods = [] libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True)) baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True)) - if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'): - for root, dirs, files in cpath.walk(dvar): - for f in files: - file = os.path.join(root, f) - if file.endswith(".ko") and file.find("/lib/modules/") != -1: - kernmods.append(file) - continue - # Skip debug files - if debugappend and file.endswith(debugappend): - continue - if debugdir and debugdir in os.path.dirname(file[len(dvar):]): - continue + for root, dirs, files in cpath.walk(dvar): + for f in files: + file = os.path.join(root, f) + if file.endswith(".ko") and file.find("/lib/modules/") != -1: + kernmods.append(file) + continue - try: - ltarget = cpath.realpath(file, dvar, False) - s = cpath.lstat(ltarget) - except OSError as e: - (err, strerror) = e.args - if err != errno.ENOENT: - raise - # Skip broken symlinks - continue - if not s: + # Skip debug files + if debugappend and file.endswith(debugappend): + continue + if debugdir and debugdir in os.path.dirname(file[len(dvar):]): + continue + + try: + ltarget = cpath.realpath(file, dvar, False) + s = cpath.lstat(ltarget) + except OSError as e: + (err, strerror) = e.args + if err != errno.ENOENT: + raise + # Skip broken symlinks + continue + if not s: + continue + # Check its an excutable + if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \ + or ((file.startswith(libdir) or file.startswith(baselibdir)) and ".so" in f): + # If it's a symlink, and points to an ELF file, we capture the readlink target + if cpath.islink(file): + target = os.readlink(file) + if isELF(ltarget): + #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget))) + symlinks[file] = target continue - # Check its an excutable - if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \ - or ((file.startswith(libdir) or file.startswith(baselibdir)) and ".so" in f): - # If it's a symlink, and points to an ELF file, we capture the readlink target - if cpath.islink(file): - target = os.readlink(file) - if isELF(ltarget): - #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget))) - symlinks[file] = target + # It's a file (or hardlink), not a link + # ...but is it ELF, and is it already stripped? + elf_file = isELF(file) + if elf_file & 1: + if elf_file & 2: + if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn, True) or "").split(): + bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn)) + else: + msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn) + package_qa_handle_error("already-stripped", msg, d) continue - # It's a file (or hardlink), not a link - # ...but is it ELF, and is it already stripped? - elf_file = isELF(file) - if elf_file & 1: - if elf_file & 2: - if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn, True) or "").split(): - bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn)) - else: - msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn) - package_qa_handle_error("already-stripped", msg, d) - continue - # Check if it's a hard link to something else - if s.st_nlink > 1: - file_reference = "%d_%d" % (s.st_dev, s.st_ino) - # Hard link to something else - hardlinks[file] = file_reference - continue - elffiles[file] = elf_file + # Check if it's a hard link to something else + if s.st_nlink > 1: + file_reference = "%d_%d" % (s.st_dev, s.st_ino) + # Hard link to something else + hardlinks[file] = file_reference + continue + elffiles[file] = elf_file # # First lets process debug splitting # - if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT', True) != '1'): + if splitting: hardlinkmap = {} # For hardlinks, process only one of the files for file in hardlinks:
@@ -1008,7 +1014,7 @@ python split_and_strip_files () { # # Now lets go back over things and strip them # - if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'): + if stripping: strip = d.getVar("STRIP", True) sfiles = [] for file in elffiles:
Signature · 1 line
1.7.9.5
Quoted 41 lines · Joe Slater
Fix logic in
split_and_strip_files()to allow splitting or stripping independently. We also return quickly from this function if we have nothing to do. We seek the following behavior:Strip / Split Behavior yes / yes binaries stripped; debug info and source in -dbg no / yes debug info and source in -dbg yes / no binaries stripped; -dbg packages empty no / no -dbg packages empty (not a very useful case)Currently, no/yes does not work and is the same as no/no.
Signed-off-by Joe Slater <[email protected]>---
meta/classes/package.bbclass | 108
++++++++++++++++++++++--------------------1 file changed, 57 insertions(+), 51 deletions(-)meta/classes/package.bbclass +10
-1@@ -812,6 +812,12 @@ python fixup_perms () { } python split_and_strip_files () { + for root, dirs, files in cpath.walk(dvar): + for f in files: + file = os.path.join(root, f) + if file.endswith(".ko") and file.find("/lib/modules/") != -1: + kernmods.append(file) + continue + # Skip debug files + if debugappend and file.endswith(debugappend): + continue + if debugdir and debugdir inos.path.dirname(file[len(dvar):]):
meta/classes/package.bbclass
Quoted 2 lines
meta/classes/package.bbclass +2
-0+ continue +
It's a pity to first construct the files just to throw them away right afterwards. Maybe there are other cpath.walk spots that would benefit from file_not_endswith and dir filters?
Thanks,
Quoted 102 lines
meta/classes/package.bbclass +13
-0+ try: + ltarget = cpath.realpath(file, dvar, False) + s = cpath.lstat(ltarget) + except OSError as e: + (err, strerror) = e.args + if err != errno.ENOENT: + raise + # Skip broken symlinks + continue + if not s: + continue + # Check its an excutable + if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
+ or ((file.startswith(libdir)orfile.startswith(baselibdir)) and ".so" in f):
+ # If it's a symlink, and points to an ELF file, we
capture the readlink targetmeta/classes/package.bbclass +4
-0+ if cpath.islink(file): + target = os.readlink(file) + if isELF(ltarget): + #bb.note("Sym: %s (%d)" % (ltarget,isELF(ltarget)))
meta/classes/package.bbclass +1
-2+ symlinks[file] = target continue - # Check its an excutable - if (s[stat.ST_MODE] & stat.S_IXUSR) or(s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
- or ((file.startswith(libdir)orfile.startswith(baselibdir)) and ".so" in f):
- # If it's a symlink, and points to an ELF file, we
capture the readlink targetmeta/classes/package.bbclass +0
-4- if cpath.islink(file): - target = os.readlink(file) - if isELF(ltarget): - #bb.note("Sym: %s (%d)" % (ltarget,isELF(ltarget)))
meta/classes/package.bbclass +6
-1- symlinks[file] = target + # It's a file (or hardlink), not a link + # ...but is it ELF, and is it already stripped? + elf_file = isELF(file) + if elf_file & 1: + if elf_file & 2: + if 'already-stripped' in(
d.getVar('INSANE_SKIP_' + pn, True)or "").split():
+ bb.note("Skipping file %s from %s for
already-stripped QA test" % (file[len(dvar):], pn))meta/classes/package.bbclass +2
-0+ else: + msg = "File '%s' from %s was alreadystripped, this will prevent future debugging!" % (file[len(dvar):], pn) +
package_qa_handle_error("already-stripped", msg, d)
continuemeta/classes/package.bbclass +0
-6- # It's a file (or hardlink), not a link - # ...but is it ELF, and is it already stripped? - elf_file = isELF(file) - if elf_file & 1: - if elf_file & 2: - if 'already-stripped' in(
d.getVar('INSANE_SKIP_' + pn, True)or "").split():
- bb.note("Skipping file %s from %s for
already-stripped QA test" % (file[len(dvar):], pn))meta/classes/package.bbclass +0
-2- else: - msg = "File '%s' from %s was alreadystripped, this will prevent future debugging!" % (file[len(dvar):], pn) -
package_qa_handle_error("already-stripped", msg, d)meta/classes/package.bbclass +0
-4- continue - # Check if it's a hard link to something else - if s.st_nlink > 1: - file_reference = "%d_%d" % (s.st_dev,s.st_ino)
meta/classes/package.bbclass +3
-4- # Hard link to something else - hardlinks[file] = file_reference - continue - elffiles[file] = elf_file + # Check if it's a hard link to something else + if s.st_nlink > 1: + file_reference = "%d_%d" % (s.st_dev,s.st_ino)
meta/classes/package.bbclass +6
-2+ # Hard link to something else + hardlinks[file] = file_reference + continue + elffiles[file] = elf_file # # First lets process debug splitting # - if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT', True) != '1'): + if splitting: hardlinkmap = {} # For hardlinks, process only one of the files for file in hardlinks:@@ -1008,7 +1014,7 @@ python split_and_strip_files () { # # Now lets go back over things and strip them # - if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'): + if stripping: strip = d.getVar("STRIP", True) sfiles = [] for file in elffiles:
Quoted 45 lines · Bernhard Reutner-Fischer
Quoted 41 lines · Joe Slater
Fix logic in
split_and_strip_files()to allow splitting or stripping independently. We also return quickly from this function if we have nothing to do. We seek the following behavior:Strip / Split Behavior yes / yes binaries stripped; debug info and source in -dbg no / yes debug info and source in -dbg yes / no binaries stripped; -dbg packages empty no / no -dbg packages empty (not a very useful case)Currently, no/yes does not work and is the same as no/no.
Signed-off-by Joe Slater <[email protected]>---
meta/classes/package.bbclass | 108
++++++++++++++++++++++--------------------1 file changed, 57 insertions(+), 51 deletions(-)meta/classes/package.bbclass +10
-1@@ -812,6 +812,12 @@ python fixup_perms () { } python split_and_strip_files () { + for root, dirs, files in cpath.walk(dvar): + for f in files: + file = os.path.join(root, f) + if file.endswith(".ko") and file.find("/lib/modules/") != -1: + kernmods.append(file) + continue + # Skip debug files + if debugappend and file.endswith(debugappend): + continue + if debugdir and debugdir inos.path.dirname(file[len(dvar):]):
meta/classes/package.bbclass
Quoted 2 lines
meta/classes/package.bbclass +2
-0+ continue +
Quoted 2 lines
It's a pity to first construct the files just to throw them away right afterwards. Maybe there are other cpath.walk spots that would benefit from
file_not_endswithand dir filters?
FWIW cpath is actually a caching wrapper around os.path so there is some amount of caching going on here behind the scenes. The current code has been profiled and the worst hot spots worked around with the cache...
Its not perfect and I don't doubt more could be done but these pieces do seem to help a lot. The best thing we did is avoid a lot of syscall overhead.
Cheers,
Richard
I just found out that this patch will break qa for ltp packaging. The ltp recipe inhibits stripping which used to inhibit splitting. If splitting is enabled, there are several .debug directories in places packages.bbclass doesn't look, so they wind up in ltp, not ltp-dbg.
It's tempting just to inhibit splitting since that was never really done before, anyhow, but I suppose that's not the right way to fix it.
Joe
Quoted 69 lines
-----Original Message-----
From: Richard Purdie [mailto:[email protected]]
Sent: Tuesday, March 17, 2015 2:20 PM
To: Bernhard Reutner-Fischer
Cc: Slater, Joseph; [email protected]
Subject: Re: [OE-core] [oe-core][PATCH 1/1] package.bbclass: decouple splitting and strippingQuoted 45 lines · Bernhard Reutner-Fischer
Quoted 41 lines · Joe Slater
Fix logic in
split_and_strip_files()to allow splitting or stripping independently. We also return quickly from this function if we have nothing to do. We seek the following behavior:Strip / Split Behavior yes / yes binaries stripped; debug info and source in -dbg no / yes debug info and source in -dbg yes / no binaries stripped; -dbg packages empty no / no -dbg packages empty (not a very useful case)Currently, no/yes does not work and is the same as no/no.
Signed-off-by Joe Slater <[email protected]>---
meta/classes/package.bbclass | 108
++++++++++++++++++++++--------------------1 file changed, 57 insertions(+), 51 deletions(-)meta/classes/package.bbclass +10
-1@@ -812,6 +812,12 @@ python fixup_perms () { } python split_and_strip_files () { + for root, dirs, files in cpath.walk(dvar): + for f in files: + file = os.path.join(root, f) + if file.endswith(".ko") and file.find("/lib/modules/") != -1: + kernmods.append(file) + continue + # Skip debug files + if debugappend and file.endswith(debugappend): + continue + if debugdir and debugdir inos.path.dirname(file[len(dvar):]):
meta/classes/package.bbclass
Quoted 2 lines
meta/classes/package.bbclass +2
-0+ continue +Quoted 2 lines
It's a pity to first construct the files just to throw them away right afterwards. Maybe there are other cpath.walk spots that would benefit from
file_not_endswithand dirfilters?
FWIW cpath is actually a caching wrapper around os.path so there is some amount of caching going on here behind the scenes. The current code has been profiled and the worst hot spots worked around with the cache...
Its not perfect and I don't doubt more could be done but these pieces do seem to help a lot. The best thing we did is avoid a lot of syscall overhead.
Cheers,
Richard
PATCH 1/1 gcc-cross-canadian: Add inhibit of split as well was Re: [PATCH 1/1] package.bbclass: decouple splitting and stripping
Quoted 7 lines · Joseph
I just found out that this patch will break qa for ltp packaging. The ltp recipe inhibits stripping which used to inhibit splitting. If splitting is enabled, there are several .debug directories in places packages.bbclass doesn't look, so they wind up in ltp, not ltp-dbg.
It's tempting just to inhibit splitting since that was never really done before, anyhow, but I suppose that's not the right way to fix it.
In addition to ltp, I recently found this breaks gcc-cross-canadian, below is a patch that should probably be applied even if this one does not make it in.
gcc-cross-canadian: Add inhibit of split as well
With the recent change to allow strip and split of packages to be controlled seperately, gcc-cross-canadian will sometimes fail to build properly. So in addition to the existing inhibit strip, we also want to inhibit split.
--- meta/recipes-devtools/gcc/gcc-cross-canadian.inc | 1 + 1 file changed, 1 insertion(+)
(quoted diff) +0 -0
b/meta/recipes-devtools/gcc/gcc-cross-canadian.inc index 195b465..ad4b08f 100644
meta/recipes-devtools/gcc/gcc-cross-canadian.inc +1 -0
@@ -64,6 +64,7 @@ do_compile () { } INHIBIT_PACKAGE_STRIP = "1" +INHIBIT_PACKAGE_DEBUG_SPLIT = "1" # Having anything auto depending on gcc-cross-sdk is a really bad idea... EXCLUDE_FROM_SHLIBS = "1"
Quoted 73 lines
Joe
Quoted 69 lines
-----Original Message-----
From: Richard Purdie [mailto:[email protected]]
Sent: Tuesday, March 17, 2015 2:20 PM
To: Bernhard Reutner-Fischer
Cc: Slater, Joseph; [email protected]
Subject: Re: [OE-core] [oe-core][PATCH 1/1] package.bbclass: decouple splitting and strippingQuoted 45 lines · Bernhard Reutner-Fischer
Quoted 41 lines · Joe Slater
Fix logic in
split_and_strip_files()to allow splitting or stripping independently. We also return quickly from this function if we have nothing to do. We seek the following behavior:Strip / Split Behavior yes / yes binaries stripped; debug info and source in -dbg no / yes debug info and source in -dbg yes / no binaries stripped; -dbg packages empty no / no -dbg packages empty (not a very useful case)Currently, no/yes does not work and is the same as no/no.
Signed-off-by Joe Slater <[email protected]>---
meta/classes/package.bbclass | 108
++++++++++++++++++++++--------------------1 file changed, 57 insertions(+), 51 deletions(-)meta/classes/package.bbclass +10
-1@@ -812,6 +812,12 @@ python fixup_perms () { } python split_and_strip_files () { + for root, dirs, files in cpath.walk(dvar): + for f in files: + file = os.path.join(root, f) + if file.endswith(".ko") and file.find("/lib/modules/") != -1: + kernmods.append(file) + continue + # Skip debug files + if debugappend and file.endswith(debugappend): + continue + if debugdir and debugdir inos.path.dirname(file[len(dvar):]):
meta/classes/package.bbclass
Quoted 2 lines
meta/classes/package.bbclass +2
-0+ continue +Quoted 2 lines
It's a pity to first construct the files just to throw them away right afterwards. Maybe there are other cpath.walk spots that would benefit from
file_not_endswithand dirfilters?
FWIW cpath is actually a caching wrapper around os.path so there is some amount of caching going on here behind the scenes. The current code has been profiled and the worst hot spots worked around with the cache...
Its not perfect and I don't doubt more could be done but these pieces do seem to help a lot. The best thing we did is avoid a lot of syscall overhead.
Cheers,
Richard
Re: [PATCH] gcc-cross-canadian: Add inhibit of split as well was Re: [PATCH 1/1] package.bbclass: decouple splitting and stripping
Quoted 30 lines · Mark Hatle
Quoted 7 lines · Joseph
I just found out that this patch will break qa for ltp packaging. The ltp recipe inhibits stripping which used to inhibit splitting. If splitting is enabled, there are several .debug directories in places packages.bbclass doesn't look, so they wind up in ltp, not ltp-dbg.
It's tempting just to inhibit splitting since that was never really done before, anyhow, but I suppose that's not the right way to fix it.
In addition to ltp, I recently found this breaks gcc-cross-canadian, below is a patch that should probably be applied even if this one does not make it in.
gcc-cross-canadian: Add inhibit of split as well
With the recent change to allow strip and split of packages to be controlled seperately, gcc-cross-canadian will sometimes fail to build properly. So in addition to the existing inhibit strip, we also want to inhibit split.
Signed-off-by Mark Hatle <[email protected]>--- meta/recipes-devtools/gcc/gcc-cross-canadian.inc | 1 + 1 file changed, 1 insertion(+)meta/recipes-devtools/gcc/gcc-cross-canadian.inc +0
-0
meta/recipes-devtools/gcc/gcc-cross-canadian.inc do_compile () { · line 70
Quoted 7 lines
meta/recipes-devtools/gcc/gcc-cross-canadian.inc +1
-0@@ -64,6 +64,7 @@ do_compile () { } INHIBIT_PACKAGE_STRIP = "1" +INHIBIT_PACKAGE_DEBUG_SPLIT = "1" # Having anything auto depending on gcc-cross-sdk is a really bad idea... EXCLUDE_FROM_SHLIBS = "1"
Not sure which revision of fido or master this against but:
http://git.yoctoproject.org/cgit.cgi/poky/commit/?id=92739edc9da0dea10ccc9d153226cacd6a18ee33
i.e. we fixed this the other way...
Cheers,
Richard
Re: [PATCH] gcc-cross-canadian: Add inhibit of split as well was Re: [PATCH 1/1] package.bbclass: decouple splitting and stripping
Quoted 36 lines · Richard Purdie
Quoted 35 lines · Mark Hatle
Quoted 7 lines · Joseph
I just found out that this patch will break qa for ltp packaging. The ltp recipe inhibits stripping which used to inhibit splitting. If splitting is enabled, there are several .debug directories in places packages.bbclass doesn't look, so they wind up in ltp, not ltp-dbg.
It's tempting just to inhibit splitting since that was never really done before, anyhow, but I suppose that's not the right way to fix it.
In addition to ltp, I recently found this breaks gcc-cross-canadian, below is a patch that should probably be applied even if this one does not make it in.
gcc-cross-canadian: Add inhibit of split as well
With the recent change to allow strip and split of packages to be controlled seperately, gcc-cross-canadian will sometimes fail to build properly. So in addition to the existing inhibit strip, we also want to inhibit split.
Signed-off-by Mark Hatle <[email protected]>--- meta/recipes-devtools/gcc/gcc-cross-canadian.inc | 1 + 1 file changed, 1 insertion(+)meta/recipes-devtools/gcc/gcc-cross-canadian.inc +1
-0@@ -64,6 +64,7 @@ do_compile () { } INHIBIT_PACKAGE_STRIP = "1" +INHIBIT_PACKAGE_DEBUG_SPLIT = "1"
Quoted 8 lines
meta/recipes-devtools/gcc/gcc-cross-canadian.inc do_compile () { · line 70
Quoted 2 lines
meta/recipes-devtools/gcc/gcc-cross-canadian.inc +0
-0# Having anything auto depending on gcc-cross-sdk is a really bad idea... EXCLUDE_FROM_SHLIBS = "1"Not sure which revision of fido or master this against but:
http://git.yoctoproject.org/cgit.cgi/poky/commit/?id=92739edc9da0dea10ccc9d153226cacd6a18ee33
i.e. we fixed this the other way...
You are right.. I thought I was in a different directory, but I was back on a pull from Feb 20th. Sorry for the confusion.
--Mark
Quoted 4 lines
Cheers,
Richard