Skip to contentSkip to conversation

support/scripts: add fix_rpaths

PATCH New Samuel Martin 4 participants 12 messages Jul 13, 2015 – Jan 6, 2016 Kbuild

lore
Share & tools
lore
#0 lore raw

PATCH v5 1/5 support/scripts: add fix_rpaths

This pyhton script leverages patchelf program to fix the RPATH of binaries.

It offers 3 actions:
- clear the RPATH;
- set the RPATH using relative paths between every single binary and the libraries directories;
- sanitize the RPATH, removing sysroot and build locations.

---
changes v4->v5:
- add verbose support
- rename shrink_rpath -> clear_rpath
- add sanitize_rpath function

changes v3->v4:
- fix typos and license (Baruch)

changes v2->v3:
- no change

---
 support/scripts/fix_rpaths | 338 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 338 insertions(+)
 create mode 100755 support/scripts/fix_rpaths
support/scripts/fix_rpaths new file +338 -0
new file mode 100755
index 0000000..86f0d53
@@ -0,0 +1,338 @@
+#!/usr/bin/env python
+##
+## Author(s):
+##  - Samuel Martin <[email protected]>
+##
+## Copyright (C) 2013 Samuel Martin
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+""" This script scans a direcotry for ELF files and fix their RPATH, making
+them relative.
+"""
+
+from __future__ import print_function, with_statement
+import os
+import stat
+import subprocess
+import sys
+import re
+
+PATCHELF_BIN = "patchelf"
+
+
+# pylint: disable=too-few-public-methods
+class PreservedTime(object):
+    """ With-class ensuring the file's times are preserved
+
+    :param path: File path
+    """
+    # pylint: disable=redefined-outer-name
+    def __init__(self, path):
+        self.path = path
+        self.times = None
+
+    # pylint: disable=invalid-name
+    def __enter__(self):
+        st = os.lstat(self.path)
+        self.times = (st.st_atime, st.st_mtime)
+        return self.path
+
+    # pylint: disable=redefined-builtin
+    def __exit__(self, type, value, traceback):
+        os.utime(self.path, self.times)
+
+
+# pylint: disable=redefined-outer-name
+def is_elf_binary(path):
+    """ Return True if path points to a valid ELF file.
+
+    :param path: File path
+    """
+    if not stat.S_ISREG(os.lstat(path).st_mode):
+        return False
+    with PreservedTime(path):
+        # pylint: disable=invalid-name
+        with open(path, "rb") as fp:
+            data = fp.read(4)
+    return data == b"\x7fELF"
+
+
+def has_rpath(elf_file, patchelf_bin=PATCHELF_BIN):
+    """ Return True if the given ELF file accept a RPATH.
+
+    :param elf_file: ELF file path
+    :param patchelf_bin: patchelf program path
+    """
+    cmd = [patchelf_bin, "--print-rpath", elf_file]
+    with PreservedTime(elf_file):
+        try:
+            subprocess.check_call(cmd, stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE)
+            elf_with_rpath = True
+        except subprocess.CalledProcessError as _:
+            elf_with_rpath = False
+    return elf_with_rpath
+
+
+def compute_rpath(binary, libdirs):
+    """ Return the RPATH value (with relative paths to the given libdirs).
+
+    :param binary: ELF binary path
+    :param libdirs: List of library directory paths
+    """
+    bindir = os.path.dirname(binary)
+    relpaths = [os.path.relpath(libdir, bindir) for libdir in libdirs]
+    # reduce the list, but keep its original order
+    # pylint: disable=unnecessary-lambda
+    sorted(set(relpaths), key=lambda x: relpaths.index(x))
+    rpaths = [os.path.join("$ORIGIN", relpath) for relpath in relpaths]
+    rpath = ":".join(rpaths)
+    return rpath
+
+
+def set_rpath(elf_file, rpath, patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Fix the ELF file RPATH.
+
+    :param elf_file: ELF file patch
+    :param rpath: New RPATH value
+    :param patchelf_bin: patchelf program path
+    """
+    cmd = [patchelf_bin, "--set-rpath", rpath, elf_file]
+    with PreservedTime(elf_file):
+        subprocess.check_call(cmd, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+    if verbose:
+        print("FILE: %s" % elf_file)
+        print("RPATH set to: '%s'" % dump_rpath(elf_file, patchelf_bin))
+
+
+def clear_rpath(elf_file, patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Shrink the ELF file's RPATH.
+
+    :param elf_file: ELF file patch
+    :param patchelf_bin: patchelf program path
+    """
+    set_rpath(elf_file, "", patchelf_bin=patchelf_bin, verbose=False)
+    if verbose:
+        print("FILE: %s\nRPATH cleared" % elf_file)
+
+
+def dump_rpath(elf_file, patchelf_bin=PATCHELF_BIN):
+    """ Return the ELF file's RPATH.
+
+    :param elf_file: ELF file patch
+    :param patchelf_bin: patchelf program path
+    """
+    cmd = [patchelf_bin, "--print-rpath", elf_file]
+    with PreservedTime(elf_file):
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        proc.wait()
+        rpath = proc.communicate()[0].strip()
+    if sys.version_info.major >= 3:
+        def _decode(txt):
+            """ Decode function """
+            return txt.decode()
+    else:
+        def _decode(txt):
+            """ Decode function """
+            return txt
+    return _decode(rpath)
+
+
+def update_rpath(elf_file, libdirs, patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Set the ELF file's RPATH with relative paths to the libdirs.
+
+    :param elf_file: ELF file patch
+    :param libdirs: List of library directory paths
+    :param patchelf_bin: patchelf program path
+    """
+    rpath = compute_rpath(elf_file, libdirs)
+    set_rpath(elf_file, rpath, patchelf_bin=patchelf_bin, verbose=verbose)
+
+
+def sanitize_rpath(elf_file, matcher, patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Remove all paths matching patterns in the ELF file's RPATH.
+
+    :param elf_file: ELF file patch
+    :param matcher: Regex objects matching the paths to be removed
+    :param patchelf_bin: patchelf program path
+    """
+    rpath = dump_rpath(elf_file, patchelf_bin=PATCHELF_BIN)
+    rpath = [path for path in rpath.split(":") if not matcher.match(path)]
+    if rpath:
+        set_rpath(elf_file, ":".join(rpath), patchelf_bin=patchelf_bin,
+                verbose=verbose)
+    else:
+        clear_rpath(elf_file, patchelf_bin=patchelf_bin, verbose=verbose)
+
+
+def find_files(root, file_filter_func=None, exclude_dirs=None):
+    """ Generator returning files from the root location.
+
+    :param root: Root path to be scan
+    :param file_filter_func: Filter function returning a boolean whether the
+                             file path should be yielded or not
+    :param exclude_dirs: List of directories to be prune from the scan
+    """
+    def dummy_filter(_):
+        """ Dummy filter function. Always return True.
+        """
+        return True
+    if not file_filter_func:
+        file_filter_func = dummy_filter
+    for parent, dirs, files in os.walk(root):
+        for xdir in exclude_dirs:
+            if xdir in dirs:
+                del dirs[dirs.index(xdir)]
+                continue
+            for idx, a_dir in enumerate(dirs):
+                if os.path.join(parent, a_dir).endswith(xdir):
+                    del dirs[idx]
+                    continue
+        for a_file in files:
+            full_path = os.path.join(parent, a_file)
+            if not file_filter_func(full_path):
+                continue
+            yield full_path
+
+
+def scan_and_apply(root, rpath_func, exclude_dirs=None,
+                   patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Scan and update RPATH on ELF files under the root location.
+
+    The new RPATH value is computed from the binaries's and the libraries
+    directories.
+
+    :param root: Root path to be scan
+    :param libdirs: List of library directory paths
+    :param patchelf_bin: patchelf program path
+    """
+    def file_filter(path):
+        """ Return True if the path points to a valid ELF file accepting RPATH.
+        """
+        # check for valid file (discard non-ELF files and broken symlinks)
+        if not is_elf_binary(path):
+            return False
+        return has_rpath(path)
+    exclude_dirs = exclude_dirs if exclude_dirs else list()
+    for filepath in find_files(root, file_filter_func=file_filter,
+                               exclude_dirs=exclude_dirs):
+        rpath_func(filepath, patchelf_bin=patchelf_bin, verbose=verbose)
+
+
+def main():
+    """ Main function
+    """
+    import argparse
+    parser = argparse.ArgumentParser(description="""\
+            Update the RPATH in all ELF files in ROOT.
+
+            It can perform 3 types of actions on the ELF files, preserving
+            their times:
+            1) 'set' the RPATH, with relative paths between ELF files and
+              the library directories;
+            2) or 'sanitize' the RPATH by removing any path matching some
+              given patterns;
+            3) or 'clear' the RPATH.
+
+            """)
+    parser.add_argument("action", choices=["set", "clear", "sanitize"],
+                        help="""\
+            Action processed on RPATH.
+            'set' updates the RPATH with relative path between each binary and
+                library directories passed via the required --libdirs option.
+            'sanitize' strips path matching patterns passed via the required
+                --patterns option.
+            'clear' empties the RPATH of the binaries.
+            """)
+    parser.add_argument("rootdir", metavar="ROOT",
+                        help="Root path to scan for RPATH fixup")
+    parser.add_argument("--libdirs", nargs="+", default=list(),
+                        help="""\
+            List of library directory paths (must be sub-location of ROOT)""")
+    parser.add_argument("--patterns", nargs="+", default=list(),
+                        help="""\
+            List of path patterns to be remove from RPATH""")
+    parser.add_argument("--exclude-dirs", nargs="+", default=list(),
+                        help="List of directories skipped for RPATH update")
+    parser.add_argument("--patchelf-program", dest="patchelf_bin",
+                        default=PATCHELF_BIN,
+                        help="Path to patchelf program to be used")
+    parser.add_argument("-v", "--verbose", action="store_true", default=False,
+                        help="Turn on verbose outputs")
+    args = parser.parse_args()
+    # sanitizing arguments
+    action = args.action
+    root = os.path.abspath(args.rootdir)
+    libdirs = [os.path.abspath(l) for l in args.libdirs if os.path.isdir(l)]
+    patterns = [x for x in args.patterns]
+    exclude_dirs = [x for x in args.exclude_dirs]
+    patchelf_bin = os.path.abspath(args.patchelf_bin)
+    # sanity checks
+    if action == "set" and not libdirs:
+        msg = "\nERROR: Setting RPATH requires non-empty --libdirs option\n\n"
+        msg += parser.format_help()
+        raise Exception(msg)
+    if action == "sanitize" and not patterns:
+        msg = "\nERROR: Sanitizing RPATH requires non-empty --patterns option\n\n"
+        msg += parser.format_help()
+        raise Exception(msg)
+    if not os.path.exists(root):
+        msg = "\nERROR: ROOT must be an existing path.\n"
+        msg += "\troot: %s\n\n" % root
+        msg += parser.format_help()
+        raise Exception(msg)
+    for libdir in libdirs:
+        if not libdir.startswith(root):
+            msg = "\nERROR: each libdirs must be under the root location.\n"
+            msg += "\troot: %s\n" % root
+            msg += "\tfaulty libdir: %s\n\n" % libdir
+            msg += parser.format_help()
+            raise Exception(msg)
+    if not os.path.exists(patchelf_bin):
+        patchelf_found = False
+        for path in os.environ.get("PATH", "").split(":"):
+            if not path:
+                continue
+            if PATCHELF_BIN in os.listdir(path):
+                patchelf_found = True
+                break
+        if not patchelf_found:
+            msg = "\nERROR: no '%s' program found on the host system.\n\n" % \
+                    PATCHELF_BIN
+            msg += parser.format_help()
+            raise Exception(msg)
+    if action == "set":
+        def _update_rpath(elf_file, patchelf_bin, verbose):
+            """ Update RPATH
+            """
+            return update_rpath(elf_file, libdirs, patchelf_bin=patchelf_bin,
+                                verbose=verbose)
+        action = _update_rpath
+    elif action == "clear":
+        action = clear_rpath
+    elif action == "sanitize":
+        regex = r"(" + "|".join(patterns) + ")"
+        matcher = re.compile(regex)
+        def _sanitize_rpath(elf_file, patchelf_bin, verbose):
+            """ Sanitize RPATH
+            """
+            return sanitize_rpath(elf_file, matcher, patchelf_bin=patchelf_bin,
+                                  verbose=verbose)
+        action = _sanitize_rpath
+    scan_and_apply(root, action, exclude_dirs=exclude_dirs,
+                   patchelf_bin=args.patchelf_bin, verbose=args.verbose)
+
+
+if __name__ == "__main__":
+    main()
Signature · 1 line
2.4.5
#1 lore raw

PATCH v5 2/5 Makefile: add HOST_FIX_RPATH_HOOK to TARGET_FINALIZE_HOOKS

This patch adds host-patchelf as a target-finalize dependency, and introduces the HOST_FIX_RPATH_HOOK hook fixing the ELF files' RPATH of the HOST_DIR location (excluding the sysroot).

After running this hook, the RPATH from any host ELF files is relative to the binary location itself.

Note that, we avoid to fix RPATH in the sysroot, and in the external toolcahin installation location.

As a first step toward a fully relocatable SDK, this change allows to get the toolchain relocatable, but not yet the whole SDK.

---
changes v4->v5:
- add verbose support

changes v3->v4:
- add host-patchelf to PACKAGES instead of target-finalize (Baruch)
- add comment

changes v2->v3:
- move hook in Makefile (Baruch)

---
 Makefile | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
Makefile +16 -0
index 546bcd3..4b88116 100644
@@ -555,6 +555,22 @@ endef
 TARGET_FINALIZE_HOOKS += PURGE_LOCALES
 endif
 
+# RPATH fixing
+# - The host hook sets RPATH in host ELF binaries, using relative paths to the
+#   library locations.
+PACKAGES += host-patchelf
+
+define HOST_FIX_RPATH_HOOK
+	$(TOPDIR)/support/scripts/fix_rpaths \
+		set $(HOST_DIR) \
+		$(if $(V),--verbose) \
+		--patchelf-program $(HOST_DIR)/usr/bin/patchelf \
+		--libdirs $(HOST_DIR)/usr/lib \
+		--exclude-dirs sysroot opt/ext-toolchain
+endef
+
+TARGET_FINALIZE_HOOKS += HOST_FIX_RPATH_HOOK
+
 $(TARGETS_ROOTFS): target-finalize
 
 target-finalize: $(PACKAGES)
Signature · 1 line
2.4.5
#2 lore raw

PATCH v5 2/5 Makefile: add HOST_FIX_RPATH_HOOK to TARGET_FINALIZE_HOOKS

Quoted 33 lines · Samuel Martin

This patch adds host-patchelf as a target-finalize dependency, and introduces the HOST_FIX_RPATH_HOOK hook fixing the ELF files' RPATH of the HOST_DIR location (excluding the sysroot).

After running this hook, the RPATH from any host ELF files is relative to the binary location itself.

Note that, we avoid to fix RPATH in the sysroot, and in the external toolcahin installation location.

As a first step toward a fully relocatable SDK, this change allows to get the toolchain relocatable, but not yet the whole SDK.

---
changes v4->v5:
- add verbose support

changes v3->v4:
- add host-patchelf to PACKAGES instead of target-finalize (Baruch)
- add comment

changes v2->v3:
- move hook in Makefile (Baruch)

---
  Makefile | 16 ++++++++++++++++
  1 file changed, 16 insertions(+)
Makefile +0 -0
index 546bcd3..4b88116 100644

Makefile endef · line 561

Quoted 7 lines
Makefile +4 -0
@@ -555,6 +555,22 @@ endef
  TARGET_FINALIZE_HOOKS += PURGE_LOCALES
  endif
 
+# RPATH fixing
+# - The host hook sets RPATH in host ELF binaries, using relative paths to the
+#   library locations.
+PACKAGES += host-patchelf

This should be TARGETS, not PACKAGES.

Signature · 2 lines
Sam Thursfield, Codethink Ltd.
Office telephone: +44 161 236 5575
#3 lore raw

PATCH v5 2/5 Makefile: add HOST_FIX_RPATH_HOOK to TARGET_FINALIZE_HOOKS

Sam, All,

On 2016-01-06 17:18 +0000, Sam Thursfield spake thusly:

Quoted 37 lines · Samuel Martin
Quoted 36 lines

This patch adds host-patchelf as a target-finalize dependency, and introduces the HOST_FIX_RPATH_HOOK hook fixing the ELF files' RPATH of the HOST_DIR location (excluding the sysroot).

After running this hook, the RPATH from any host ELF files is relative to the binary location itself.

Note that, we avoid to fix RPATH in the sysroot, and in the external toolcahin installation location.

As a first step toward a fully relocatable SDK, this change allows to get the toolchain relocatable, but not yet the whole SDK.

---
changes v4->v5:
- add verbose support

changes v3->v4:
- add host-patchelf to PACKAGES instead of target-finalize (Baruch)
- add comment

changes v2->v3:
- move hook in Makefile (Baruch)

---
 Makefile | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
Makefile +0 -0
index 546bcd3..4b88116 100644
@@ -555,6 +555,22 @@ endef
 TARGET_FINALIZE_HOOKS += PURGE_LOCALES
 endif
 
Quoted 6 lines

Makefile endef · line 561

Quoted 4 lines
Makefile +4 -0
+# RPATH fixing
+# - The host hook sets RPATH in host ELF binaries, using relative paths to the
+#   library locations.
+PACKAGES += host-patchelf

This should be TARGETS, not PACKAGES.

Nope, PACKAGES is the correct variable. It has been renamed from TARGETS to PACKAGES in April2015:

commit 8a58e0238ef3099d31fdd01cbbcca6ae1421eea5
Author: Thomas Petazzoni <[email protected]>
Date: Sun Apr 12 18:37:48 2015 +0200

Makefile: rename TARGETS to PACKAGES

For clarity, this commit renames the TARGETS variable to the more meaningful PACKAGES variable. Indeed, only packages (handled by one of the package infrastructures) should be listed in this variable, and not other random non-package targets.

Regards,
Yann E. MORIN.

Signature · 6 lines
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 223 225 172 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'
#4 lore raw

PATCH v5 3/5 Makefile: add TARGET_SANITIZE_RPATH_HOOK to TARGET_FINALIZE_HOOKS

In reply to Samuel Martin

This patch introduces the TARGET_SANITIZE_RPATH_HOOK hook fixing the ELF files' RPATH of the TARGET_DIR (excluding the sysroot).

After running this hook, the RPATH from any target ELF files won't contain any occurence of the sysroot or some build locations.

---
changes v4->v5:
- target hook now sanitizes the rpath (Baruch)
- add verbose support
- update comment

changes v3->v4:
- rebase
- add comment

changes v2->v3:
- move hook in Makefile (Baruch)

---
 Makefile | 13 +++++++++++++
 1 file changed, 13 insertions(+)
Makefile +13 -0
index 4b88116..69c1105 100644
@@ -558,6 +558,8 @@ endif
 # RPATH fixing
 # - The host hook sets RPATH in host ELF binaries, using relative paths to the
 #   library locations.
+# - The target hook sanitizes RPATH in target ELF binaries, removing pathes
+#   pointing to package's build directories or the sysroot's libdirs.
 PACKAGES += host-patchelf
 
 define HOST_FIX_RPATH_HOOK
@@ -571,6 +573,17 @@ endef
 
 TARGET_FINALIZE_HOOKS += HOST_FIX_RPATH_HOOK
 
+define TARGET_SANITIZE_RPATH_HOOK
+	$(TOPDIR)/support/scripts/fix_rpaths \
+		sanitize $(TARGET_DIR) \
+		$(if $(V),--verbose) \
+		--patchelf-program $(HOST_DIR)/usr/bin/patchelf \
+		--patterns "($(STAGING_DIR)|`readlink -f $(STAGING_DIR)`)/(usr/)?lib$$" \
+			"($(BUILD_DIR)|`readlink -f $(BUILD_DIR)`)/.*"
+endef
+
+TARGET_FINALIZE_HOOKS += TARGET_SANITIZE_RPATH_HOOK
+
 $(TARGETS_ROOTFS): target-finalize
 
 target-finalize: $(PACKAGES)
Signature · 1 line
2.4.5
#5 lore raw

PATCH v5 3/5 Makefile: add TARGET_SANITIZE_RPATH_HOOK to TARGET_FINALIZE_HOOKS

Hi Samuel,

Quoted 2 lines · Samuel Martin

This patch introduces the TARGET_SANITIZE_RPATH_HOOK hook fixing the ELF files' RPATH of the TARGET_DIR (excluding the sysroot).

Are you sure sysroot is excluded?

Quoted 4 lines

After running this hook, the RPATH from any target ELF files won't contain any occurence of the sysroot or some build locations.

[ Build test only: verified that /sbin/ebtables preserves its RPATH
(/lib/ebtables), and that /usr/sbin/visudo also preserves its target RPATH (/usr/libexec/sudo), while dropping the staging part ($(@D)/lib/util/.libs) ]

baruch

Signature · 3 lines
     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -
#6 lore raw

PATCH v5 3/5 Makefile: add TARGET_SANITIZE_RPATH_HOOK to TARGET_FINALIZE_HOOKS

Hi Baruch,

Quoted 7 lines · Baruch Siach

Hi Samuel,

Quoted 2 lines · Samuel Martin

This patch introduces the TARGET_SANITIZE_RPATH_HOOK hook fixing the ELF files' RPATH of the TARGET_DIR (excluding the sysroot).

Are you sure sysroot is excluded?

Yes, via $(STAGING_DIR).

During my tests, I only saw /path/to/sysroot/usr/lib in the RPATH. So, I only remove paths similar to this (i.e. standard libdirs from the sysroot) in RPATH.

Quoted 13 lines
Quoted 5 lines

After running this hook, the RPATH from any target ELF files won't contain any occurence of the sysroot or some build locations.

[ Build test only: verified that /sbin/ebtables preserves its RPATH
(/lib/ebtables), and that /usr/sbin/visudo also preserves its target RPATH (/usr/libexec/sudo), while dropping the staging part ($(@D)/lib/util/.libs) ]

Quoted 6 lines

baruch

--
http://baruch.siach.name/blog/ ~. .~ Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
- baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -

Thanks a lot for your careful review. :-)

Regards,

Signature · 1 line
Samuel
#7 lore raw

PATCH v5 4/5 Makefile: staging symlink uses a relative path

In reply to Samuel Martin

A step forward toward a relocatable SDK.

---
changes v3->v4:
- no change

changes v2->v3:
- no change

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
Makefile +1 -1
index 69c1105..570cbf2 100644
@@ -460,7 +460,7 @@ $(STAGING_DIR):
 	@ln -snf lib $(STAGING_DIR)/usr/$(LIB_SYMLINK)
 	@mkdir -p $(STAGING_DIR)/usr/include
 	@mkdir -p $(STAGING_DIR)/usr/bin
-	@ln -snf $(STAGING_DIR) $(BASE_DIR)/staging
+	@ln -snf $(subst $(BASE_DIR)/,,$(STAGING_DIR)) $(BASE_DIR)/staging
 
 ifeq ($(BR2_ROOTFS_SKELETON_CUSTOM),y)
 TARGET_SKELETON = $(BR2_ROOTFS_SKELETON_CUSTOM_PATH)
Signature · 1 line
2.4.5
#8 lore raw

PATCH v5 5/5 package/speex: remove no longer needed hook

In reply to Samuel Martin

Remove the LIBTOOL_FIXUP hook since RPATH are now sanitized in target-finalize hooks.

---
changes v4->v5:
- new patch (suggested by Baruch)

---
 package/speex/speex.mk | 5 -----
 1 file changed, 5 deletions(-)
package/speex/speex.mk +0 -5
index a34bfac..24782ec 100644
@@ -24,11 +24,6 @@ ifeq ($(BR2_PACKAGE_SPEEX_ARM5E),y)
 SPEEX_CONF_OPTS += --enable-arm5e-asm
 endif
 
-define SPEEX_LIBTOOL_FIXUP
-	$(SED) 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' $(@D)/libtool
-	$(SED) 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' $(@D)/libtool
-endef
-
 SPEEX_POST_CONFIGURE_HOOKS += SPEEX_LIBTOOL_FIXUP
 
 define SPEEX_BUILD_CMDS
Signature · 1 line
2.4.5
#9 lore raw

PATCH v5 5/5 package/speex: remove no longer needed hook

Hi Samuel,

Quoted 16 lines · Samuel Martin

Remove the LIBTOOL_FIXUP hook since RPATH are now sanitized in target-finalize hooks.

---
changes v4->v5:
- new patch (suggested by Baruch)

---
 package/speex/speex.mk | 5 -----
 1 file changed, 5 deletions(-)
package/speex/speex.mk +0 -0
index a34bfac..24782ec 100644

package/speex/speex.mk ifeq ($(BR2_PACKAGE_SPEEX_ARM5E),y) · line 27

Quoted 9 lines
package/speex/speex.mk +0 -5
@@ -24,11 +24,6 @@ ifeq ($(BR2_PACKAGE_SPEEX_ARM5E),y)
 SPEEX_CONF_OPTS += --enable-arm5e-asm
 endif
 
-define SPEEX_LIBTOOL_FIXUP
-	$(SED) 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' $(@D)/libtool
-	$(SED) 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' $(@D)/libtool
-endef
-
 SPEEX_POST_CONFIGURE_HOOKS += SPEEX_LIBTOOL_FIXUP

You can remove this line as well.

baruch

Quoted 1 line
package/speex/speex.mk +0 -0
 define SPEEX_BUILD_CMDS
Signature · 3 lines
     http://baruch.siach.name/blog/                  ~. .~   Tk Open Systems
=}------------------------------------------------ooO--U--Ooo------------{=
   - baruch at tkos.co.il - tel: +972.2.679.5364, http://www.tkos.co.il -
#10 lore raw

PATCH v5 1/5 support/scripts: add fix_rpaths

In reply to Samuel Martin

Hi!

Thanks for this patch series, I'm trying it out at the moment.

If it works I will be very happy, as I've implement an artifact caching mechanism that uses Buildroot build results, and of course hardcoded paths in the host tools are causing things to break! I'm hopeful that relative rpaths will solve all my problems :-)

I had a few problems in the fix_rpaths script, I think it's because I didn't have patchelf installed system-wide. Fixes described below:

Quoted 8 lines · Samuel Martin
---
  support/scripts/fix_rpaths | 338 +++++++++++++++++++++++++++++++++++++++++++++
  1 file changed, 338 insertions(+)

create mode 100755 support/scripts/fix_rpaths

support/scripts/fix_rpaths new file +0 -0
new file mode 100755
index 0000000..86f0d53

...

support/scripts/fix_rpaths line 170

Quoted 8 lines
support/scripts/fix_rpaths new file +8 -0
+def sanitize_rpath(elf_file, matcher, patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Remove all paths matching patterns in the ELF file's RPATH.
+
+    :param elf_file: ELF file patch
+    :param matcher: Regex objects matching the paths to be removed
+    :param patchelf_bin: patchelf program path
+    """
+    rpath = dump_rpath(elf_file, patchelf_bin=PATCHELF_BIN)

This should be:

rpath = dump_rpath(elf_file, patchelf_bin=patchelf_bin)

Otherwise it breaks when patchelf is not available in PATH.

Quoted 7 lines
support/scripts/fix_rpaths new file +7 -0
+    rpath = [path for path in rpath.split(":") if not matcher.match(path)]
+    if rpath:
+        set_rpath(elf_file, ":".join(rpath), patchelf_bin=patchelf_bin,
+                verbose=verbose)
+    else:
+        clear_rpath(elf_file, patchelf_bin=patchelf_bin, verbose=verbose)
+

...

Quoted 8 lines
support/scripts/fix_rpaths new file +8 -0
+def scan_and_apply(root, rpath_func, exclude_dirs=None,
+                   patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Scan and update RPATH on ELF files under the root location.
+
+    The new RPATH value is computed from the binaries's and the libraries
+    directories.
+
+    :param root: Root path to be scan

support/scripts/fix_rpaths line 226

Quoted 10 lines
support/scripts/fix_rpaths new file +10 -0
+    :param libdirs: List of library directory paths
+    :param patchelf_bin: patchelf program path
+    """
+    def file_filter(path):
+        """ Return True if the path points to a valid ELF file accepting RPATH.
+        """
+        # check for valid file (discard non-ELF files and broken symlinks)
+        if not is_elf_binary(path):
+            return False
+        return has_rpath(path)

This needs to pass on the value of patchelf_bin, otherwise it will break when patchelf is not available in PATH.

return has_rpath(path, patchelf_bin=patchelf_bin)

Quoted 71 lines
support/scripts/fix_rpaths new file +71 -0
+    exclude_dirs = exclude_dirs if exclude_dirs else list()
+    for filepath in find_files(root, file_filter_func=file_filter,
+                               exclude_dirs=exclude_dirs):
+        rpath_func(filepath, patchelf_bin=patchelf_bin, verbose=verbose)
+
+
+def main():
+    """ Main function
+    """
+    import argparse
+    parser = argparse.ArgumentParser(description="""\
+            Update the RPATH in all ELF files in ROOT.
+
+            It can perform 3 types of actions on the ELF files, preserving
+            their times:
+            1) 'set' the RPATH, with relative paths between ELF files and
+              the library directories;
+            2) or 'sanitize' the RPATH by removing any path matching some
+              given patterns;
+            3) or 'clear' the RPATH.
+
+            """)
+    parser.add_argument("action", choices=["set", "clear", "sanitize"],
+                        help="""\
+            Action processed on RPATH.
+            'set' updates the RPATH with relative path between each binary and
+                library directories passed via the required --libdirs option.
+            'sanitize' strips path matching patterns passed via the required
+                --patterns option.
+            'clear' empties the RPATH of the binaries.
+            """)
+    parser.add_argument("rootdir", metavar="ROOT",
+                        help="Root path to scan for RPATH fixup")
+    parser.add_argument("--libdirs", nargs="+", default=list(),
+                        help="""\
+            List of library directory paths (must be sub-location of ROOT)""")
+    parser.add_argument("--patterns", nargs="+", default=list(),
+                        help="""\
+            List of path patterns to be remove from RPATH""")
+    parser.add_argument("--exclude-dirs", nargs="+", default=list(),
+                        help="List of directories skipped for RPATH update")
+    parser.add_argument("--patchelf-program", dest="patchelf_bin",
+                        default=PATCHELF_BIN,
+                        help="Path to patchelf program to be used")
+    parser.add_argument("-v", "--verbose", action="store_true", default=False,
+                        help="Turn on verbose outputs")
+    args = parser.parse_args()
+    # sanitizing arguments
+    action = args.action
+    root = os.path.abspath(args.rootdir)
+    libdirs = [os.path.abspath(l) for l in args.libdirs if os.path.isdir(l)]
+    patterns = [x for x in args.patterns]
+    exclude_dirs = [x for x in args.exclude_dirs]
+    patchelf_bin = os.path.abspath(args.patchelf_bin)
+    # sanity checks
+    if action == "set" and not libdirs:
+        msg = "\nERROR: Setting RPATH requires non-empty --libdirs option\n\n"
+        msg += parser.format_help()
+        raise Exception(msg)
+    if action == "sanitize" and not patterns:
+        msg = "\nERROR: Sanitizing RPATH requires non-empty --patterns option\n\n"
+        msg += parser.format_help()
+        raise Exception(msg)
+    if not os.path.exists(root):
+        msg = "\nERROR: ROOT must be an existing path.\n"
+        msg += "\troot: %s\n\n" % root
+        msg += parser.format_help()
+        raise Exception(msg)
+    for libdir in libdirs:
+        if not libdir.startswith(root):
+            msg = "\nERROR: each libdirs must be under the root location.\n"

support/scripts/fix_rpaths line 307

Quoted 10 lines
support/scripts/fix_rpaths new file +10 -0
+            msg += "\troot: %s\n" % root
+            msg += "\tfaulty libdir: %s\n\n" % libdir
+            msg += parser.format_help()
+            raise Exception(msg)
+    if not os.path.exists(patchelf_bin):
+        patchelf_found = False
+        for path in os.environ.get("PATH", "").split(":"):
+            if not path:
+                continue
+            if PATCHELF_BIN in os.listdir(path):

This can abort with an exception if there is a directory in PATH that isn't a readable directory.

Better to write it as:

if not path or not os.path.isdir(path):
continue
if PATCHELF_BIN in os.listdir(path):

Signature · 2 lines
Sam Thursfield, Codethink Ltd.
Office telephone: +44 161 236 5575
#11 lore raw

PATCH v5 1/5 support/scripts: add fix_rpaths

Sam, All,

On 2016-01-06 17:17 +0000, Sam Thursfield spake thusly:

Quoted 1 line

Thanks for this patch series, I'm trying it out at the moment.

Be aware that Samuel has been reworking this series quite intensively, and what you tested may have changed since the original submission.

I guess Samuel will send an updated series when it is ready, but this is a tricky topic, which needs quite an attentive eye to the details...

Regards,
Yann E. MORIN.

Quoted 166 lines

If it works I will be very happy, as I've implement an artifact caching mechanism that uses Buildroot build results, and of course hardcoded paths in the host tools are causing things to break! I'm hopeful that relative rpaths will solve all my problems :-)

I had a few problems in the fix_rpaths script, I think it's because I didn't have patchelf installed system-wide. Fixes described below:

Quoted 8 lines · Samuel Martin
---
 support/scripts/fix_rpaths | 338 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 338 insertions(+)
 create mode 100755 support/scripts/fix_rpaths
support/scripts/fix_rpaths new file +0 -0
new file mode 100755
index 0000000..86f0d53

...

support/scripts/fix_rpaths line 170

Quoted 8 lines
support/scripts/fix_rpaths new file +8 -0
+def sanitize_rpath(elf_file, matcher, patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Remove all paths matching patterns in the ELF file's RPATH.
+
+    :param elf_file: ELF file patch
+    :param matcher: Regex objects matching the paths to be removed
+    :param patchelf_bin: patchelf program path
+    """
+    rpath = dump_rpath(elf_file, patchelf_bin=PATCHELF_BIN)

This should be:

rpath = dump_rpath(elf_file, patchelf_bin=patchelf_bin)

Otherwise it breaks when patchelf is not available in PATH.

Quoted 7 lines
support/scripts/fix_rpaths new file +7 -0
+    rpath = [path for path in rpath.split(":") if not matcher.match(path)]
+    if rpath:
+        set_rpath(elf_file, ":".join(rpath), patchelf_bin=patchelf_bin,
+                verbose=verbose)
+    else:
+        clear_rpath(elf_file, patchelf_bin=patchelf_bin, verbose=verbose)
+

...

Quoted 8 lines
support/scripts/fix_rpaths new file +8 -0
+def scan_and_apply(root, rpath_func, exclude_dirs=None,
+                   patchelf_bin=PATCHELF_BIN, verbose=False):
+    """ Scan and update RPATH on ELF files under the root location.
+
+    The new RPATH value is computed from the binaries's and the libraries
+    directories.
+
+    :param root: Root path to be scan

support/scripts/fix_rpaths line 226

Quoted 10 lines
support/scripts/fix_rpaths new file +10 -0
+    :param libdirs: List of library directory paths
+    :param patchelf_bin: patchelf program path
+    """
+    def file_filter(path):
+        """ Return True if the path points to a valid ELF file accepting RPATH.
+        """
+        # check for valid file (discard non-ELF files and broken symlinks)
+        if not is_elf_binary(path):
+            return False
+        return has_rpath(path)

This needs to pass on the value of patchelf_bin, otherwise it will break when patchelf is not available in PATH.

return has_rpath(path, patchelf_bin=patchelf_bin)

Quoted 71 lines
support/scripts/fix_rpaths new file +71 -0
+    exclude_dirs = exclude_dirs if exclude_dirs else list()
+    for filepath in find_files(root, file_filter_func=file_filter,
+                               exclude_dirs=exclude_dirs):
+        rpath_func(filepath, patchelf_bin=patchelf_bin, verbose=verbose)
+
+
+def main():
+    """ Main function
+    """
+    import argparse
+    parser = argparse.ArgumentParser(description="""\
+            Update the RPATH in all ELF files in ROOT.
+
+            It can perform 3 types of actions on the ELF files, preserving
+            their times:
+            1) 'set' the RPATH, with relative paths between ELF files and
+              the library directories;
+            2) or 'sanitize' the RPATH by removing any path matching some
+              given patterns;
+            3) or 'clear' the RPATH.
+
+            """)
+    parser.add_argument("action", choices=["set", "clear", "sanitize"],
+                        help="""\
+            Action processed on RPATH.
+            'set' updates the RPATH with relative path between each binary and
+                library directories passed via the required --libdirs option.
+            'sanitize' strips path matching patterns passed via the required
+                --patterns option.
+            'clear' empties the RPATH of the binaries.
+            """)
+    parser.add_argument("rootdir", metavar="ROOT",
+                        help="Root path to scan for RPATH fixup")
+    parser.add_argument("--libdirs", nargs="+", default=list(),
+                        help="""\
+            List of library directory paths (must be sub-location of ROOT)""")
+    parser.add_argument("--patterns", nargs="+", default=list(),
+                        help="""\
+            List of path patterns to be remove from RPATH""")
+    parser.add_argument("--exclude-dirs", nargs="+", default=list(),
+                        help="List of directories skipped for RPATH update")
+    parser.add_argument("--patchelf-program", dest="patchelf_bin",
+                        default=PATCHELF_BIN,
+                        help="Path to patchelf program to be used")
+    parser.add_argument("-v", "--verbose", action="store_true", default=False,
+                        help="Turn on verbose outputs")
+    args = parser.parse_args()
+    # sanitizing arguments
+    action = args.action
+    root = os.path.abspath(args.rootdir)
+    libdirs = [os.path.abspath(l) for l in args.libdirs if os.path.isdir(l)]
+    patterns = [x for x in args.patterns]
+    exclude_dirs = [x for x in args.exclude_dirs]
+    patchelf_bin = os.path.abspath(args.patchelf_bin)
+    # sanity checks
+    if action == "set" and not libdirs:
+        msg = "\nERROR: Setting RPATH requires non-empty --libdirs option\n\n"
+        msg += parser.format_help()
+        raise Exception(msg)
+    if action == "sanitize" and not patterns:
+        msg = "\nERROR: Sanitizing RPATH requires non-empty --patterns option\n\n"
+        msg += parser.format_help()
+        raise Exception(msg)
+    if not os.path.exists(root):
+        msg = "\nERROR: ROOT must be an existing path.\n"
+        msg += "\troot: %s\n\n" % root
+        msg += parser.format_help()
+        raise Exception(msg)
+    for libdir in libdirs:
+        if not libdir.startswith(root):
+            msg = "\nERROR: each libdirs must be under the root location.\n"

support/scripts/fix_rpaths line 307

Quoted 10 lines
support/scripts/fix_rpaths new file +10 -0
+            msg += "\troot: %s\n" % root
+            msg += "\tfaulty libdir: %s\n\n" % libdir
+            msg += parser.format_help()
+            raise Exception(msg)
+    if not os.path.exists(patchelf_bin):
+        patchelf_found = False
+        for path in os.environ.get("PATH", "").split(":"):
+            if not path:
+                continue
+            if PATCHELF_BIN in os.listdir(path):

This can abort with an exception if there is a directory in PATH that isn't a readable directory.

Better to write it as:

if not path or not os.path.isdir(path):
continue
if PATCHELF_BIN in os.listdir(path):

Signature · 6 lines
Sam Thursfield, Codethink Ltd.
Office telephone: +44 161 236 5575
_______________________________________________
buildroot mailing list
buildroot at busybox.net
http://lists.busybox.net/mailman/listinfo/buildroot
Signature · 6 lines
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 223 225 172 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'