split_and_strip_files: regroup hardlinks to make build deterministic

PATCH New Ed Bartosh 2 participants 5 messages Apr 22, 2015 – Apr 28, 2015

lore

Share & tools

lore

Ed Bartosh

author

2015-04-22 12:11 UTC #0 lore raw

PATCH 1/1 split_and_strip_files: regroup hardlinks to make build deterministic

Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf

Created separate group of hardlinks for the files inside the same package. This should prevent stripped files to be populated outside of package directories.

[YOCTO #7586]

Signed-off-by Ed Bartosh <[email protected]>

---
 meta/classes/package.bbclass | 15 ++++++++++++++-
 meta/lib/oe/package.py       |  3 +--
 2 files changed, 15 insertions(+), 3 deletions(-)

meta/classes/package.bbclass +14 -1

index f1b966d..e54c366 100644
@@ -812,7 +812,7 @@ python fixup_perms () {
 }
 
 python split_and_strip_files () {
-    import stat, errno
+    import stat, errno, subprocess
 
     dvar = d.getVar('PKGD', True)
     pn = d.getVar('PN', True)

@@ -877,6 +877,7 @@ python split_and_strip_files () {
     symlinks = {}
     hardlinks = {}
     kernmods = []
+    inodes = {}
     libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True))
     baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
     if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):

@@ -914,6 +915,18 @@ python split_and_strip_files () {
                             #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget)))
                             symlinks[file] = target
                         continue
+
+                    if s.st_ino in inodes:
+                        os.unlink(file)
+                        os.link(inodes[s.st_ino], file)
+                    elif s.st_nlink > 1:
+                        inodes[s.st_ino] = file
+                        cmd = "cp -fa %s %s.tmp && mv -f %s.tmp %s" % (file, file, file, file)
+                        bb.debug(1, "split_and_strip_files: breaking hardlink: %s" % cmd)
+                        ret = subprocess.call(cmd, shell=True)
+                        if ret:
+                            bb.error("split_and_strip_files: '%s' command failed" % cmd)
+
                     # It's a file (or hardlink), not a link
                     # ...but is it ELF, and is it already stripped?
                     elf_file = isELF(file)

meta/lib/oe/package.py +1 -2

index 8bc56c6..ea6feaa 100644
@@ -30,8 +30,7 @@ def runstrip(arg):
     elif elftype & 8 or elftype & 4:
         extraflags = "--remove-section=.comment --remove-section=.note"
 
-    # Use mv to break hardlinks
-    stripcmd = "'%s' %s '%s' -o '%s.tmp' && chown --reference='%s' '%s.tmp' && mv '%s.tmp' '%s'" % (strip, extraflags, file, file, file, file, file, file)
+    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
     bb.debug(1, "runstrip: %s" % stripcmd)
 
     ret = subprocess.call(stripcmd, shell=True)

Signature · 1 line

2.1.4

Ed Bartosh

author

2015-04-22 19:57 UTC #1 lore raw

PATCH 1/1 split_and_strip_files: regroup hardlinks to make build deterministic

Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf

Created separate group of hardlinks for the files inside the same package. This should prevent stripped files to be populated outside of package directories.

[YOCTO #7586]

Signed-off-by Ed Bartosh <[email protected]>

---
 meta/classes/package.bbclass | 10 ++++++++++
 meta/lib/oe/package.py       |  3 +--
 2 files changed, 11 insertions(+), 2 deletions(-)

meta/classes/package.bbclass +10 -0

index f1b966d..b9748e7 100644
@@ -877,6 +877,7 @@ python split_and_strip_files () {
     symlinks = {}
     hardlinks = {}
     kernmods = []
+    inodes = {}
     libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True))
     baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
     if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):

@@ -914,6 +915,15 @@ python split_and_strip_files () {
                             #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget)))
                             symlinks[file] = target
                         continue
+
+                    if s.st_ino in inodes:
+                        os.unlink(file)
+                        os.link(inodes[s.st_ino], file)
+                    elif s.st_nlink > 1:
+                        inodes[s.st_ino] = file
+                        # break hardlink
+                        bb.utils.copyfile(file, file)
+
                     # It's a file (or hardlink), not a link
                     # ...but is it ELF, and is it already stripped?
                     elf_file = isELF(file)

meta/lib/oe/package.py +1 -2

index 8bc56c6..ea6feaa 100644
@@ -30,8 +30,7 @@ def runstrip(arg):
     elif elftype & 8 or elftype & 4:
         extraflags = "--remove-section=.comment --remove-section=.note"
 
-    # Use mv to break hardlinks
-    stripcmd = "'%s' %s '%s' -o '%s.tmp' && chown --reference='%s' '%s.tmp' && mv '%s.tmp' '%s'" % (strip, extraflags, file, file, file, file, file, file)
+    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
     bb.debug(1, "runstrip: %s" % stripcmd)
 
     ret = subprocess.call(stripcmd, shell=True)

Signature · 1 line

2.1.4

Richard Purdie

2015-04-27 21:02 UTC #2 lore raw

Quoted 6 lines · Ed Bartosh

Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf
Created separate group of hardlinks for the files inside the same package. This should prevent stripped files to be populated outside of package directories.

Quoted 7 lines

[YOCTO #7586]

Signed-off-by Ed Bartosh <[email protected]>

---
 meta/classes/package.bbclass | 10 ++++++++++
 meta/lib/oe/package.py       |  3 +--
 2 files changed, 11 insertions(+), 2 deletions(-)

I ran this on the autobuilder. I noticed:

https://autobuilder.yoctoproject.org/main/builders/build-appliance/builds/269/steps/BuildImages_1/logs/stdio

which would seem related to this patch somehow but I'm not sure what is happening to cause it as yet.

Cheers,

Richard

Ed Bartosh

author

2015-04-28 16:07 UTC #3 lore raw

PATCH 1/1 split_and_strip_files: regroup hardlinks to make build deterministic

Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf

Created separate group of hardlinks for the files inside the same package. This should prevent stripped files to be populated outside of package directories.

[YOCTO #7586]

Signed-off-by Ed Bartosh <[email protected]>

---
 meta/classes/package.bbclass | 12 ++++++++++++
 meta/lib/oe/package.py       |  3 +--
 2 files changed, 13 insertions(+), 2 deletions(-)

meta/classes/package.bbclass +12 -0

index f1b966d..4019f12 100644
@@ -877,6 +877,7 @@ python split_and_strip_files () {
     symlinks = {}
     hardlinks = {}
     kernmods = []
+    inodes = {}
     libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True))
     baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
     if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):

@@ -914,6 +915,17 @@ python split_and_strip_files () {
                             #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget)))
                             symlinks[file] = target
                         continue
+
+                    if s.st_ino in inodes:
+                        os.unlink(file)
+                        os.link(inodes[s.st_ino], file)
+                        cpath.updatecache(file)
+                    elif s.st_nlink > 1:
+                        inodes[s.st_ino] = file
+                        # break hardlink
+                        bb.utils.copyfile(file, file)
+                        cpath.updatecache(file)
+
                     # It's a file (or hardlink), not a link
                     # ...but is it ELF, and is it already stripped?
                     elf_file = isELF(file)

meta/lib/oe/package.py +1 -2

index 8bc56c6..ea6feaa 100644
@@ -30,8 +30,7 @@ def runstrip(arg):
     elif elftype & 8 or elftype & 4:
         extraflags = "--remove-section=.comment --remove-section=.note"
 
-    # Use mv to break hardlinks
-    stripcmd = "'%s' %s '%s' -o '%s.tmp' && chown --reference='%s' '%s.tmp' && mv '%s.tmp' '%s'" % (strip, extraflags, file, file, file, file, file, file)
+    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
     bb.debug(1, "runstrip: %s" % stripcmd)
 
     ret = subprocess.call(stripcmd, shell=True)

Signature · 1 line

2.1.4

Ed Bartosh

author

2015-04-28 16:46 UTC #4 lore raw

PATCH 1/1 split_and_strip_files: regroup hardlinks to make build deterministic

In reply to Richard Purdie

Reverted 7c0fd561bad0250a00cef63e3d787573112a59cf

Created separate group of hardlinks for the files inside the same package. This should prevent stripped files to be populated outside of package directories.

[YOCTO #7586]

Signed-off-by Ed Bartosh <[email protected]>

---
 meta/classes/package.bbclass | 12 ++++++++++++
 meta/lib/oe/package.py       |  3 +--
 2 files changed, 13 insertions(+), 2 deletions(-)

meta/classes/package.bbclass +12 -0

index f1b966d..dc87206 100644
@@ -877,6 +877,7 @@ python split_and_strip_files () {
     symlinks = {}
     hardlinks = {}
     kernmods = []
+    inodes = {}
     libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir", True))
     baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
     if (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):

@@ -914,6 +915,17 @@ python split_and_strip_files () {
                             #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget)))
                             symlinks[file] = target
                         continue
+                    if s.st_nlink > 1:
+                        if s.st_ino in inodes:
+                            os.unlink(file)
+                            os.link(inodes[s.st_ino], file)
+                        elif s.st_nlink > 1:
+                            inodes[s.st_ino] = file
+                            # break hardlink
+                            bb.utils.copyfile(file, file)
+                        cpath.updatecache(file)
+                        s = cpath.lstat(file)
+
                     # It's a file (or hardlink), not a link
                     # ...but is it ELF, and is it already stripped?
                     elf_file = isELF(file)

meta/lib/oe/package.py +1 -2

index 8bc56c6..ea6feaa 100644
@@ -30,8 +30,7 @@ def runstrip(arg):
     elif elftype & 8 or elftype & 4:
         extraflags = "--remove-section=.comment --remove-section=.note"
 
-    # Use mv to break hardlinks
-    stripcmd = "'%s' %s '%s' -o '%s.tmp' && chown --reference='%s' '%s.tmp' && mv '%s.tmp' '%s'" % (strip, extraflags, file, file, file, file, file, file)
+    stripcmd = "'%s' %s '%s'" % (strip, extraflags, file)
     bb.debug(1, "runstrip: %s" % stripcmd)
 
     ret = subprocess.call(stripcmd, shell=True)

Signature · 1 line

2.1.4