Skip to contentSkip to conversation

binfmt_elf: FatELF support in the binary loader.

PATCH Anton D. Kachalov 2 participants 7 messages Oct 23, 2009 – Oct 30, 2009

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

I have made very similar patch but it's quite small and do not require deep hacks.
It should works with "setarch" too to force selection of binary. There is a tool to merge binaries. Glibc / binutils patch work in progress.

Rgds,
Anton

fs/binfmt_elf.c renamed from fs/binfmt_elf.c.orig +67 -4
@@ -571,6 +571,7 @@ static int load_elf_binary(struct linux_
 	unsigned long elf_bss, elf_brk;
 	int retval, i;
 	unsigned int size;
+	unsigned int offset;
 	unsigned long elf_entry;
 	unsigned long interp_load_addr = 0;
 	unsigned long start_code, end_code, start_data, end_data;
@@ -588,6 +589,8 @@ static int load_elf_binary(struct linux_
 		goto out_ret;
 	}
 	
+	offset = 0;
+retry_elf:
 	/* Get the exec-header */
 	loc->elf_ex = *((struct elfhdr *)bprm->buf);
 
@@ -598,8 +601,27 @@ static int load_elf_binary(struct linux_
 
 	if (loc->elf_ex.e_type != ET_EXEC && loc->elf_ex.e_type != ET_DYN)
 		goto out;
-	if (!elf_check_arch(&loc->elf_ex))
+#if (defined(__x86_64__) || defined(__powerpc64__) || defined(__s390x__)) \
+    && ELF_CLASS == ELFCLASS64
+	if (personality(current->personality) == PER_LINUX32)
 		goto out;
+#endif
+	if (!elf_check_arch(&loc->elf_ex)) {
+		offset = loc->elf_ex.e_ident[0xc]
+			| loc->elf_ex.e_ident[0xd] << 8
+			| loc->elf_ex.e_ident[0xe] << 16
+			| loc->elf_ex.e_ident[0xf] << 24;
+		if (offset != 0) {
+			memset(bprm->buf, 0, BINPRM_BUF_SIZE);
+			retval = kernel_read(bprm->file, offset, bprm->buf, BINPRM_BUF_SIZE);
+			if (retval != BINPRM_BUF_SIZE) {
+				retval = -ENOEXEC;
+				goto out;
+			}
+			goto retry_elf;
+		}
+		goto out;
+	}
 	if (!bprm->file->f_op||!bprm->file->f_op->mmap)
 		goto out;
 
@@ -723,13 +745,30 @@ static int load_elf_binary(struct linux_
 
 	/* Some simple consistency checks for the interpreter */
 	if (elf_interpreter) {
+		offset = 0;
+retry_interp:
 		retval = -ELIBBAD;
+		loc->interp_elf_ex = *((struct elfhdr *)bprm->buf);
 		/* Not an ELF interpreter */
 		if (memcmp(loc->interp_elf_ex.e_ident, ELFMAG, SELFMAG) != 0)
 			goto out_free_dentry;
 		/* Verify the interpreter has a valid arch */
-		if (!elf_check_arch(&loc->interp_elf_ex))
+		if (!elf_check_arch(&loc->interp_elf_ex)) {
+			offset = loc->interp_elf_ex.e_ident[0xc]
+				| loc->interp_elf_ex.e_ident[0xd] << 8
+				| loc->interp_elf_ex.e_ident[0xe] << 16
+				| loc->interp_elf_ex.e_ident[0xf] << 24;
+			if (offset != 0) {
+				memset(bprm->buf, 0, BINPRM_BUF_SIZE);
+				retval = kernel_read(bprm->file, offset, bprm->buf, BINPRM_BUF_SIZE);
+				if (retval != BINPRM_BUF_SIZE) {
+					retval = -ELIBBAD;
+					goto out;
+				}
+				goto retry_interp;
+			}
 			goto out_free_dentry;
+		}
 	} else {
 		/* Executables without an interpreter also need a personality  */
 		SET_PERSONALITY(loc->elf_ex);
@@ -1026,18 +1065,42 @@ static int load_elf_library(struct file 
 	unsigned long elf_bss, bss, len;
 	int retval, error, i, j;
 	struct elfhdr elf_ex;
+	unsigned int offset;
 
 	error = -ENOEXEC;
 	retval = kernel_read(file, 0, (char *)&elf_ex, sizeof(elf_ex));
 	if (retval != sizeof(elf_ex))
 		goto out;
 
+	offset = 0;
+retry_lib:
 	if (memcmp(elf_ex.e_ident, ELFMAG, SELFMAG) != 0)
 		goto out;
 
 	/* First of all, some simple consistency checks */
-	if (elf_ex.e_type != ET_EXEC || elf_ex.e_phnum > 2 ||
-	    !elf_check_arch(&elf_ex) || !file->f_op || !file->f_op->mmap)
+	if (elf_ex.e_type != ET_EXEC || elf_ex.e_phnum > 2)
+		goto out;
+#if (defined(__x86_64__) || defined(__powerpc64__) || defined(__s390x__)) \
+    && ELF_CLASS == ELFCLASS64
+	if (personality(current->personality) == PER_LINUX32)
+		goto out;
+#endif
+	if (!elf_check_arch(&elf_ex)) {
+		offset = elf_ex.e_ident[0xc]
+			| elf_ex.e_ident[0xd] << 8
+			| elf_ex.e_ident[0xe] << 16
+			| elf_ex.e_ident[0xf] << 24;
+		if (offset != 0) {
+			retval = kernel_read(file, offset, (char *)&elf_ex, sizeof(elf_ex));
+			if (retval != sizeof(elf_ex)) {
+				retval = -ENOEXEC;
+				goto out;
+			}
+			goto retry_lib;
+		}
+		goto out;
+	}
+	if (!file->f_op || !file->f_op->mmap)
 		goto out;
 
 	/* Now read in all of the header information */
#1 lore raw
Quoted 4 lines · Anton D. Kachalov

I have made very similar patch but it's quite small and do not require deep hacks.
It should works with "setarch" too to force selection of binary. There is a tool to merge binaries. Glibc / binutils patch work in progress.

$ uname -m
x86_64
$ ./truearch hello32 hello64 hellos
$ ./hellos
hello x86_64
$ setarch i386 ./hellos
Hello x86
$ setarch x86_64 ./hellos
hello x86_64
$ file hello32
hello: ELF 32-bit LSB executable, Intel 80386, version 1 (SYSV), statically linked, for GNU/Linux 2.6.15, not stripped
$ file hello64
hello64: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, for GNU/Linux 2.6.15, not stripped

While I didn't finish glibc hacks, I could use only statically linked binaries.

Rgds,
Anton

#2 lore raw
In reply to Anton D. Kachalov
Quoted 3 lines

I have made very similar patch but it's quite small and do not require deep hacks.

Wow, competing ideas! :)

Here are my notes on your idea. Ego compels me to prefer my approach, but I strove to be objective here, as there is a tradeoff of benefits in each of our approaches.

Quoted 1 line

It should works with "setarch" too to force selection of binary.

How does setarch work? Does it reorder the file before launching or copy out one of the ELF records?

If reordering:
What does this do to binaries you can't write to? Regular users couldn't rewrite /bin/ls before launching, for example.

If copying:
What does this do to programs that rely on the value of argv[0]? If setarch mangles up argv[0] in its exec*() call to match the original binary's patch, what does this do to programs that rely on /proc/self/exe?

The most compelling feature of this approach is that a "truearch" binary (is that the correct name?) could work with any existing Linux system, on the condition that the architecture you want is the first one in the file. If you put, say, x86 first in the file and you want to run it on an x86_64 system, you're either out of luck or going to be running the 32-bit version. In this same scenario, if you put x86_64 first, it just won't run at all on an unpatched x86 box. So, it's a cool trick, but it's not all that beneficial. We have to assume that either approach requires kernel patches to be truly useful. For unpatched boxes, FatELF provides a simple command line app, fatelf-extract, which can be used to get the original ELF binary you want out of the FatELF file, both for stripping unwanted bits and as a measure of last resort if the kernel and dynamic loader can't handle FatELF. I assume setarch works somewhat the same.

I'm concerned about using the padding bits in e_ident, too. A lot of manpower went into the ELF specification and I felt it was presumptuous for me to personally change the format. A container around them, like FatELF, was a safer, more future-proof choice. I'd rather those that control the ELF spec decide what those padding bits should be used for in the future.

The truearch method requires the kernel to seek throughout the whole file to decide if it can use it at all. FatELF uses the 128 bytes at the front of the file, which binfmt_elf reads anyhow, and then seeks to the right record from there, so disk bandwidth overhead is extremely small (one extra read of 128 bytes if we can use the file, zero extra reads if not). On the other hand, this approach allows for an unlimited amount of ELF binaries to reside in a single file below the four gigabyte mark (which is really, for all intents and purposes, a LOT of binaries). On the other hand, the FatELF limit of 255 records is probably way more than you could ever hope to reasonably cram into a file, and if it's not, we can raise it to 64k (we have reserved bits in the header still). FatELF can store ELF binaries above 4 gigabytes, unlike truearch, but I'm not sure that's really ever going to be valuable.

Both approaches have zero disk overhead if a normal ELF file is loaded, which is good.

In terms of this patch itself, I'd be concerned about using gotos for the retry_* blocks when a loop would be easy enough to incorporate. I saw you have a test for personality() that I didn't do; I might have to check into that, but the binfmt_elf_compat code is definitely catching x86 binaries on x86_64 here, so I'm not sure it's necessary.

Anyhow, I hope this was useful commentary, and not seen as a battle of egos. I'm glad to see other approaches, though, as it suggests there really is a genuine desire for this sort of functionality!

--ryan.

#3 lore raw

Hello, Ryan.

Quoted 4 lines · Ryan C. Gordon

Wow, competing ideas! :)

Here are my notes on your idea. Ego compels me to prefer my approach, but I strove to be objective here, as there is a tradeoff of benefits in each of our approaches.

Thanks :) It was born just out of Apple's concept utilizing unused space in ELF headers.

Quoted 6 lines
Quoted 2 lines

It should works with "setarch" too to force selection of binary.

How does setarch work? Does it reorder the file before launching or copy out one of the ELF records?

$ man setarch
NAME
      setarch  -  change reported architecture in new program 
environment and
      set personality flags
$ dpkg -S /usr/bin/setarch
util-linux: /usr/bin/setarch

"setarch" just set "personality" of running program.

[...]

Quoted 2 lines

The most compelling feature of this approach is that a "truearch" binary (is that the correct name?) could work with any existing Linux system, on the condition that the architecture you want is the first one in the file.

Nope, you may put binary files in any order. It's just a linked list of binaries.

Quoted 1 line

If you put, say, x86 first in the file and you want to run it on an x86_64 system, you're either out of luck or going to be running the 32-bit version.

As a previous state, you will able run x86_64. But you need to change order of binfmt and compat_binfmt in built-in.o by changing Makefile. Just swap two lines. I don't know why on x86_64 system first we try compat mode than native while simple run of native app will take more cpu cycles on x86_64 Vs. x86.

Quoted 2 lines

In this same scenario, if you put x86_64 first, it just won't run at all on an unpatched x86 box. So, it's a cool trick, but it's not all that beneficial. We have to assume that either approach requires kernel patches to be truly useful. For unpatched boxes, FatELF provides a simple command line app, fatelf-extract, which can be used to get the original ELF binary you want out of the FatELF file, both for stripping unwanted bits and as a measure of last resort if the kernel and dynamic loader can't handle FatELF. I assume setarch works somewhat the same.

Which arch will be "fatelf-extract"? Let's say, If I'm running Linux on PowerPC? x86? =) Only if it is a shell script, it will be beneficial for any arch. I can inject "offset" portion in script file too...

Quoted 4 lines

I'm concerned about using the padding bits in e_ident, too. A lot of manpower went into the ELF specification and I felt it was presumptuous for me to personally change the format. A container around them, like FatELF, was a safer, more future-proof choice. I'd rather those that control the ELF spec decide what those padding bits should be used for in the future.

The truearch method requires the kernel to seek throughout the whole file

Nope, it just read "offset" field and seek if needed. So, if file is just one-arch, it will read 128 bytes only.

Quoted 2 lines

to decide if it can use it at all. FatELF uses the 128 bytes at the front of the file, which binfmt_elf reads anyhow, and then seeks to the right record from there, so disk bandwidth overhead is extremely small (one extra read of 128 bytes if we can use the file, zero extra reads if not).

In my approach, it's just a few seeks more. Just a few additional reads are not so much compared to overall reads from that file.

[...]

Quoted 5 lines

Both approaches have zero disk overhead if a normal ELF file is loaded, which is good.

In terms of this patch itself, I'd be concerned about using gotos for the retry_* blocks when a loop would be easy enough to incorporate. I saw you have a test for personality() that I didn't do; I might have to check into that, but the binfmt_elf_compat code is definitely catching x86 binaries on x86_64 here, so I'm not sure it's necessary.

It's necessary if you would like to use setarch to choose binaries on biarch systems.

Quoted 2 lines

Anyhow, I hope this was useful commentary, and not seen as a battle of egos. I'm glad to see other approaches, though, as it suggests there really is a genuine desire for this sort of functionality!

:) Agreed

Rgds,
Anton

#4 lore raw
Quoted 5 lines
Quoted 4 lines

How does setarch work? Does it reorder the file before launching or copy out one of the ELF records?

$ man setarch

Huh, I guess that explains the personality() thing in your patch. :)

Quoted 5 lines
Quoted 3 lines

The most compelling feature of this approach is that a "truearch" binary (is that the correct name?) could work with any existing Linux system, on the condition that the architecture you want is the first one in the file.

Nope, you may put binary files in any order. It's just a linked list of binaries.

I was talking about an unpatched kernel in this case. Without a kernel change, it rejects the ELF file if the first architecture isn't the one it wants. So your approach is slightly better for backwards compatibility than mine, but not significantly so.

For a full compatibility plan, you'd probably still need to write something like fatelf-extract.

Quoted 4 lines

As a previous state, you will able run x86_64. But you need to change order of binfmt and compat_binfmt in built-in.o by changing Makefile. Just swap two lines. I don't know why on x86_64 system first we try compat mode than native while simple run of native app will take more cpu cycles on x86_64 Vs. x86.

I think it was a bug. It got corrected sometime after the 2.6.28 release that your patch was based on (I ran into that this week, too), register_binfmt() adds to the other side of the list now, so the Makefile doesn't need changing.

Quoted 3 lines

Which arch will be "fatelf-extract"? Let's say, If I'm running Linux on PowerPC? x86? =) Only if it is a shell script, it will be beneficial for any arch. I can inject "offset" portion in script file too...

Presumably, it'll be installed system-wide (as either an ELF or FatELF binary), like setarch, or built by the user if they don't otherwise have FatELF support. setarch and fatelf-extract are both simple C programs.

http://fedorahosted.org/setarch/browser/setarch.c
http://hg.icculus.org/icculus/fatelf/raw-file/tip/utils/fatelf-extract.c

Quoted 2 lines

Nope, it just read "offset" field and seek if needed. So, if file is just one-arch, it will read 128 bytes only.

But if it's 10 architectures?

Quoted 2 lines

In my approach, it's just a few seeks more. Just a few additional reads are not so much compared to overall reads from that file.

Yes, I'm not sure it's a serious concern, even if you put 200 binaries in there and have to iterate them all, the overhead is probably not serious. Still, it seems like a better practice to keep these things closer to O(1) when possible.

Quoted 2 lines

It's necessary if you would like to use setarch to choose binaries on biarch systems.

I didn't understand that when I first looked at the patch; sorry for any confusion I caused. I should probably add a similar test to FatELF.

--ryan.

#5 lore raw
Quoted 6 lines
Quoted 3 lines

It's necessary if you would like to use setarch to choose binaries on biarch systems.

I didn't understand that when I first looked at the patch; sorry for any confusion I caused. I should probably add a similar test to FatELF.

...and now I have.

This is the same as the previous revision of my patch, plus a few lines to check the personality, so setarch(8) can be used to force a 32-bit record to be selected. Thanks to Anton for the idea!

Quoted 1 line

From 083ebfa908c1bfbdabf81e921f3ce6b5d430bcb9 Mon Sep 17 00:00:00 2001

From: Ryan C. Gordon <[email protected]>
Date: Mon, 26 Oct 2009 01:50:41 -0400
Subject: [PATCH] binfmt_elf: FatELF support in the binary loader.

This allows the kernel to load a single file that contains multiple ELF binaries (a "FatELF" file), selecting the correct one for the system.

Details, rationale, tools, and patches for handling FatELF binaries can be found at http://icculus.org/fatelf/

---
 arch/ia64/ia32/binfmt_elf32.c |    4 +-
 fs/binfmt_elf.c               |  159 ++++++++++++++++++++++++++++++++++------
 include/linux/elf.h           |   23 ++++++
 3 files changed, 160 insertions(+), 26 deletions(-)
arch/ia64/ia32/binfmt_elf32.c +2 -2
index c69552b..02d5dd1 100644
@@ -223,12 +223,12 @@ elf32_set_personality (void)
 
 static unsigned long
 elf32_map(struct file *filep, unsigned long addr, struct elf_phdr *eppnt,
-		int prot, int type, unsigned long unused)
+		int prot, int type, unsigned long unused, unsigned long base)
 {
 	unsigned long pgoff = (eppnt->p_vaddr) & ~IA32_PAGE_MASK;
 
 	return ia32_do_mmap(filep, (addr & IA32_PAGE_MASK), eppnt->p_filesz + pgoff, prot, type,
-			    eppnt->p_offset - pgoff);
+			    (eppnt->p_offset + base) - pgoff);
 }
 
 #define cpu_uses_ia32el()	(local_cpu_data->family > 0x1f)
fs/binfmt_elf.c +135 -24

268 lines not on this page · raw message

include/linux/elf.h +23 -0
index 90a4ed0..ded1fb6 100644
@@ -33,6 +33,29 @@ typedef __u32	Elf64_Word;
 typedef __u64	Elf64_Xword;
 typedef __s64	Elf64_Sxword;
 
+/* FatELF (multiple ELF binaries in one file) support */
+#define FATELF_MAGIC (0x1F0E70FA)
+
+typedef struct Fatelf_record {
+  __le16 machine;      /* maps to e_machine */
+  __u8 osabi;          /* maps to e_ident[EI_OSABI] */
+  __u8 osabi_version;  /* maps to e_ident[EI_ABIVERSION] */
+  __u8 word_size;      /* maps to e_ident[EI_CLASS] */
+  __u8 byte_order;     /* maps to e_ident[EI_DATA] */
+  __u8 reserved0;
+  __u8 reserved1;
+  __le64 offset;
+  __le64 size;
+} Fatelf_record;
+
+typedef struct Fatelf_hdr {
+  __le32 magic;
+  __le16 version;
+  __u8 num_records;
+  __u8 reserved0;
+  Fatelf_record records[];
+} Fatelf_hdr;
+
 /* These constants are for the segment types stored in the image headers */
 #define PT_NULL    0
 #define PT_LOAD    1
Signature · 1 line
1.6.0.4
#6 lore raw

(Hopefully final version, now with checkpatch.pl approval!)

Quoted 1 line

From c6fd97215e9cd1de050bed500195df617f01a291 Mon Sep 17 00:00:00 2001

From: Ryan C. Gordon <[email protected]>
Date: Tue, 27 Oct 2009 13:53:39 -0400
Subject: [PATCH] binfmt_elf: FatELF support in the binary loader.

This allows the kernel to load a single file that contains multiple ELF binaries (a "FatELF" file), selecting the correct one for the system.

Details, rationale, tools, and patches for handling FatELF binaries can be found at http://icculus.org/fatelf/

---
 arch/ia64/ia32/binfmt_elf32.c |    4 +-
 fs/binfmt_elf.c               |  159 ++++++++++++++++++++++++++++++++++------
 include/linux/elf.h           |   23 ++++++
 3 files changed, 160 insertions(+), 26 deletions(-)
arch/ia64/ia32/binfmt_elf32.c +2 -2
index c69552b..02d5dd1 100644
@@ -223,12 +223,12 @@ elf32_set_personality (void)
 
 static unsigned long
 elf32_map(struct file *filep, unsigned long addr, struct elf_phdr *eppnt,
-		int prot, int type, unsigned long unused)
+		int prot, int type, unsigned long unused, unsigned long base)
 {
 	unsigned long pgoff = (eppnt->p_vaddr) & ~IA32_PAGE_MASK;
 
 	return ia32_do_mmap(filep, (addr & IA32_PAGE_MASK), eppnt->p_filesz + pgoff, prot, type,
-			    eppnt->p_offset - pgoff);
+			    (eppnt->p_offset + base) - pgoff);
 }
 
 #define cpu_uses_ia32el()	(local_cpu_data->family > 0x1f)
fs/binfmt_elf.c +135 -24

268 lines not on this page · raw message

include/linux/elf.h +23 -0
index 90a4ed0..742bab9 100644
@@ -33,6 +33,29 @@ typedef __u32	Elf64_Word;
 typedef __u64	Elf64_Xword;
 typedef __s64	Elf64_Sxword;
 
+/* FatELF (multiple ELF binaries in one file) support */
+#define FATELF_MAGIC (0x1F0E70FA)
+
+struct Fatelf_record {
+  __le16 machine;      /* maps to e_machine */
+  __u8 osabi;          /* maps to e_ident[EI_OSABI] */
+  __u8 osabi_version;  /* maps to e_ident[EI_ABIVERSION] */
+  __u8 word_size;      /* maps to e_ident[EI_CLASS] */
+  __u8 byte_order;     /* maps to e_ident[EI_DATA] */
+  __u8 reserved0;
+  __u8 reserved1;
+  __le64 offset;
+  __le64 size;
+};
+
+struct Fatelf_hdr {
+  __le32 magic;
+  __le16 version;
+  __u8 num_records;
+  __u8 reserved0;
+  struct Fatelf_record records[];
+};
+
 /* These constants are for the segment types stored in the image headers */
 #define PT_NULL    0
 #define PT_LOAD    1
Signature · 1 line
1.6.0.4