binfmt_elf: FatELF support in the binary loader.
Subsystems (2)
I have made very similar patch but it's quite small and do not require deep hacks.
It should works with "setarch" too to force selection of binary. There is a tool to merge binaries. Glibc / binutils patch work in progress.
Rgds,
Anton
fs/binfmt_elf.c renamed from fs/binfmt_elf.c.orig +67 -4
@@ -571,6 +571,7 @@ static int load_elf_binary(struct linux_unsigned long elf_bss, elf_brk;int retval, i;unsigned int size; +unsigned int offset;unsigned long elf_entry;unsigned long interp_load_addr =0 ;unsigned long start_code, end_code, start_data, end_data;
@@ -588,6 +589,8 @@ static int load_elf_binary(struct linux_ goto out_ret; } + offset =0 ; +retry_elf:/* Get the exec-header */ loc->elf_ex = *((struct elfhdr *)bprm->buf);
@@ -598,8 +601,27 @@ static int load_elf_binary(struct linux_if (loc->elf_ex.e_type !=ET_EXEC && loc->elf_ex.e_type !=ET_DYN ) goto out; -if (!elf_check_arch (&loc->elf_ex)) +#if (defined(__x86_64__) || defined(__powerpc64__) || defined(__s390x__)) \ + &&ELF_CLASS ==ELFCLASS64 +if (personality (current->personality) ==PER_LINUX32 ) goto out; +#endif +if (!elf_check_arch (&loc->elf_ex)) { + offset = loc->elf_ex.e_ident[0xc ] + | loc->elf_ex.e_ident[0xd ] <<8 + | loc->elf_ex.e_ident[0xe ] <<16 + | loc->elf_ex.e_ident[0xf ] <<24 ; +if (offset !=0 ) { +memset (bprm->buf,0 ,BINPRM_BUF_SIZE ); + retval =kernel_read (bprm->file, offset, bprm->buf,BINPRM_BUF_SIZE ); +if (retval !=BINPRM_BUF_SIZE ) { + retval = -ENOEXEC ; + goto out; + } + goto retry_elf; + } + goto out; + }if (!bprm->file->f_op||!bprm->file->f_op->mmap) goto out;
@@ -723,13 +745,30 @@ static int load_elf_binary(struct linux_/* Some simple consistency checks for the interpreter */ if (elf_interpreter) { + offset =0 ; +retry_interp: retval = -ELIBBAD ; + loc->interp_elf_ex = *((struct elfhdr *)bprm->buf);/* Not an ELF interpreter */ if (memcmp (loc->interp_elf_ex.e_ident,ELFMAG ,SELFMAG ) !=0 ) goto out_free_dentry;/* Verify the interpreter has a valid arch */ -if (!elf_check_arch (&loc->interp_elf_ex)) +if (!elf_check_arch (&loc->interp_elf_ex)) { + offset = loc->interp_elf_ex.e_ident[0xc ] + | loc->interp_elf_ex.e_ident[0xd ] <<8 + | loc->interp_elf_ex.e_ident[0xe ] <<16 + | loc->interp_elf_ex.e_ident[0xf ] <<24 ; +if (offset !=0 ) { +memset (bprm->buf,0 ,BINPRM_BUF_SIZE ); + retval =kernel_read (bprm->file, offset, bprm->buf,BINPRM_BUF_SIZE ); +if (retval !=BINPRM_BUF_SIZE ) { + retval = -ELIBBAD ; + goto out; + } + goto retry_interp; + } goto out_free_dentry; + } }else {/* Executables without an interpreter also need a personality */ SET_PERSONALITY (loc->elf_ex);
@@ -1026,18 +1065,42 @@ static int load_elf_library(struct fileunsigned long elf_bss, bss, len;int retval, error, i, j;struct elfhdr elf_ex; +unsigned int offset; error = -ENOEXEC ; retval =kernel_read (file,0 , (char *)&elf_ex,sizeof (elf_ex));if (retval !=sizeof (elf_ex)) goto out; + offset =0 ; +retry_lib:if (memcmp (elf_ex.e_ident,ELFMAG ,SELFMAG ) !=0 ) goto out;/* First of all, some simple consistency checks */ -if (elf_ex.e_type !=ET_EXEC || elf_ex.e_phnum >2 || - !elf_check_arch (&elf_ex) || !file->f_op || !file->f_op->mmap) +if (elf_ex.e_type !=ET_EXEC || elf_ex.e_phnum >2 ) + goto out; +#if (defined(__x86_64__) || defined(__powerpc64__) || defined(__s390x__)) \ + &&ELF_CLASS ==ELFCLASS64 +if (personality (current->personality) ==PER_LINUX32 ) + goto out; +#endif +if (!elf_check_arch (&elf_ex)) { + offset = elf_ex.e_ident[0xc ] + | elf_ex.e_ident[0xd ] <<8 + | elf_ex.e_ident[0xe ] <<16 + | elf_ex.e_ident[0xf ] <<24 ; +if (offset !=0 ) { + retval =kernel_read (file, offset, (char *)&elf_ex,sizeof (elf_ex)); +if (retval !=sizeof (elf_ex)) { + retval = -ENOEXEC ; + goto out; + } + goto retry_lib; + } + goto out; + } +if (!file->f_op || !file->f_op->mmap) goto out;/* Now read in all of the header information */
Quoted 4 lines · Anton D. Kachalov
I have made very similar patch but it's quite small and do not require deep hacks.
It should works with "setarch" too to force selection of binary. There is a tool to merge binaries. Glibc / binutils patch work in progress.
$ uname -m x86_64 $ ./truearch hello32 hello64 hellos $ ./hellos hello x86_64 $ setarch i386 ./hellos Hello x86 $ setarch x86_64 ./hellos hello x86_64 $ file hello32 hello: ELF 32-bit LSB executable, Intel 80386, version 1 (SYSV), statically linked, for GNU/Linux 2.6.15, not stripped $ file hello64 hello64: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, for GNU/Linux 2.6.15, not stripped
While I didn't finish glibc hacks, I could use only statically linked binaries.
Rgds,
Anton
Quoted 3 lines
I have made very similar patch but it's quite small and do not require deep hacks.
Wow, competing ideas! :)
Here are my notes on your idea. Ego compels me to prefer my approach, but I strove to be objective here, as there is a tradeoff of benefits in each of our approaches.
Quoted 1 line
It should works with "setarch" too to force selection of binary.
How does setarch work? Does it reorder the file before launching or copy out one of the ELF records?
If reordering:
What does this do to binaries you can't write to? Regular users couldn't rewrite /bin/ls before launching, for example.
If copying:
What does this do to programs that rely on the value of argv[0]? If setarch mangles up argv[0] in its exec*() call to match the original binary's patch, what does this do to programs that rely on /proc/self/exe?
The most compelling feature of this approach is that a "truearch" binary (is that the correct name?) could work with any existing Linux system, on the condition that the architecture you want is the first one in the file. If you put, say, x86 first in the file and you want to run it on an x86_64 system, you're either out of luck or going to be running the 32-bit version. In this same scenario, if you put x86_64 first, it just won't run at all on an unpatched x86 box. So, it's a cool trick, but it's not all that beneficial. We have to assume that either approach requires kernel patches to be truly useful. For unpatched boxes, FatELF provides a simple command line app, fatelf-extract, which can be used to get the original ELF binary you want out of the FatELF file, both for stripping unwanted bits and as a measure of last resort if the kernel and dynamic loader can't handle FatELF. I assume setarch works somewhat the same.
I'm concerned about using the padding bits in e_ident, too. A lot of manpower went into the ELF specification and I felt it was presumptuous for me to personally change the format. A container around them, like FatELF, was a safer, more future-proof choice. I'd rather those that control the ELF spec decide what those padding bits should be used for in the future.
The truearch method requires the kernel to seek throughout the whole file to decide if it can use it at all. FatELF uses the 128 bytes at the front of the file, which binfmt_elf reads anyhow, and then seeks to the right record from there, so disk bandwidth overhead is extremely small (one extra read of 128 bytes if we can use the file, zero extra reads if not). On the other hand, this approach allows for an unlimited amount of ELF binaries to reside in a single file below the four gigabyte mark (which is really, for all intents and purposes, a LOT of binaries). On the other hand, the FatELF limit of 255 records is probably way more than you could ever hope to reasonably cram into a file, and if it's not, we can raise it to 64k (we have reserved bits in the header still). FatELF can store ELF binaries above 4 gigabytes, unlike truearch, but I'm not sure that's really ever going to be valuable.
Both approaches have zero disk overhead if a normal ELF file is loaded, which is good.
In terms of this patch itself, I'd be concerned about using gotos for the retry_* blocks when a loop would be easy enough to incorporate. I saw you have a test for personality() that I didn't do; I might have to check into that, but the binfmt_elf_compat code is definitely catching x86 binaries on x86_64 here, so I'm not sure it's necessary.
Anyhow, I hope this was useful commentary, and not seen as a battle of egos. I'm glad to see other approaches, though, as it suggests there really is a genuine desire for this sort of functionality!
--ryan.
Hello, Ryan.
Quoted 4 lines · Ryan C. Gordon
Wow, competing ideas! :)
Here are my notes on your idea. Ego compels me to prefer my approach, but I strove to be objective here, as there is a tradeoff of benefits in each of our approaches.
Thanks :) It was born just out of Apple's concept utilizing unused space in ELF headers.
Quoted 6 lines
Quoted 2 lines
It should works with "setarch" too to force selection of binary.
How does setarch work? Does it reorder the file before launching or copy out one of the ELF records?
$ man setarch
NAME
setarch - change reported architecture in new program
environment and
set personality flags$ dpkg -S /usr/bin/setarch util-linux: /usr/bin/setarch
"setarch" just set "personality" of running program.
[...]
Quoted 2 lines
The most compelling feature of this approach is that a "truearch" binary (is that the correct name?) could work with any existing Linux system, on the condition that the architecture you want is the first one in the file.
Nope, you may put binary files in any order. It's just a linked list of binaries.
Quoted 1 line
If you put, say, x86 first in the file and you want to run it on an
x86_64system, you're either out of luck or going to be running the 32-bit version.
As a previous state, you will able run x86_64. But you need to change order of binfmt and compat_binfmt in built-in.o by changing Makefile. Just swap two lines. I don't know why on x86_64 system first we try compat mode than native while simple run of native app will take more cpu cycles on x86_64 Vs. x86.
Quoted 2 lines
In this same scenario, if you put
x86_64first, it just won't run at all on an unpatched x86 box. So, it's a cool trick, but it's not all that beneficial. We have to assume that either approach requires kernel patches to be truly useful. For unpatched boxes, FatELF provides a simple command line app, fatelf-extract, which can be used to get the original ELF binary you want out of the FatELF file, both for stripping unwanted bits and as a measure of last resort if the kernel and dynamic loader can't handle FatELF. I assume setarch works somewhat the same.
Which arch will be "fatelf-extract"? Let's say, If I'm running Linux on PowerPC? x86? =) Only if it is a shell script, it will be beneficial for any arch. I can inject "offset" portion in script file too...
Quoted 4 lines
I'm concerned about using the padding bits in
e_ident, too. A lot of manpower went into the ELF specification and I felt it was presumptuous for me to personally change the format. A container around them, like FatELF, was a safer, more future-proof choice. I'd rather those that control the ELF spec decide what those padding bits should be used for in the future.The truearch method requires the kernel to seek throughout the whole file
Nope, it just read "offset" field and seek if needed. So, if file is just one-arch, it will read 128 bytes only.
Quoted 2 lines
to decide if it can use it at all. FatELF uses the 128 bytes at the front of the file, which
binfmt_elfreads anyhow, and then seeks to the right record from there, so disk bandwidth overhead is extremely small (one extra read of 128 bytes if we can use the file, zero extra reads if not).
In my approach, it's just a few seeks more. Just a few additional reads are not so much compared to overall reads from that file.
[...]
Quoted 5 lines
Both approaches have zero disk overhead if a normal ELF file is loaded, which is good.
In terms of this patch itself, I'd be concerned about using gotos for the retry_* blocks when a loop would be easy enough to incorporate. I saw you have a test for
personality()that I didn't do; I might have to check into that, but thebinfmt_elf_compatcode is definitely catching x86 binaries onx86_64here, so I'm not sure it's necessary.
It's necessary if you would like to use setarch to choose binaries on biarch systems.
Quoted 2 lines
Anyhow, I hope this was useful commentary, and not seen as a battle of egos. I'm glad to see other approaches, though, as it suggests there really is a genuine desire for this sort of functionality!
:) Agreed
Rgds,
Anton
Quoted 5 lines
Quoted 4 lines
How does setarch work? Does it reorder the file before launching or copy out one of the ELF records?
$ man setarch
Huh, I guess that explains the personality() thing in your patch. :)
Quoted 5 lines
Quoted 3 lines
The most compelling feature of this approach is that a "truearch" binary (is that the correct name?) could work with any existing Linux system, on the condition that the architecture you want is the first one in the file.
Nope, you may put binary files in any order. It's just a linked list of binaries.
I was talking about an unpatched kernel in this case. Without a kernel change, it rejects the ELF file if the first architecture isn't the one it wants. So your approach is slightly better for backwards compatibility than mine, but not significantly so.
For a full compatibility plan, you'd probably still need to write something like fatelf-extract.
Quoted 4 lines
As a previous state, you will able run
x86_64. But you need to change order of binfmt andcompat_binfmtin built-in.o by changingMakefile. Just swap two lines. I don't know why onx86_64system first we try compat mode than native while simple run of native app will take more cpu cycles onx86_64Vs. x86.
I think it was a bug. It got corrected sometime after the 2.6.28 release that your patch was based on (I ran into that this week, too), register_binfmt() adds to the other side of the list now, so the Makefile doesn't need changing.
Quoted 3 lines
Which arch will be "fatelf-extract"? Let's say, If I'm running Linux on PowerPC? x86? =) Only if it is a shell script, it will be beneficial for any arch. I can inject "offset" portion in script file too...
Presumably, it'll be installed system-wide (as either an ELF or FatELF binary), like setarch, or built by the user if they don't otherwise have FatELF support. setarch and fatelf-extract are both simple C programs.
http://fedorahosted.org/setarch/browser/setarch.c
http://hg.icculus.org/icculus/fatelf/raw-file/tip/utils/fatelf-extract.c
Quoted 2 lines
Nope, it just read "offset" field and seek if needed. So, if file is just one-arch, it will read 128 bytes only.
But if it's 10 architectures?
Quoted 2 lines
In my approach, it's just a few seeks more. Just a few additional reads are not so much compared to overall reads from that file.
Yes, I'm not sure it's a serious concern, even if you put 200 binaries in there and have to iterate them all, the overhead is probably not serious. Still, it seems like a better practice to keep these things closer to O(1) when possible.
Quoted 2 lines
It's necessary if you would like to use setarch to choose binaries on biarch systems.
I didn't understand that when I first looked at the patch; sorry for any confusion I caused. I should probably add a similar test to FatELF.
--ryan.
Quoted 6 lines
Quoted 3 lines
It's necessary if you would like to use setarch to choose binaries on biarch systems.
I didn't understand that when I first looked at the patch; sorry for any confusion I caused. I should probably add a similar test to FatELF.
...and now I have.
This is the same as the previous revision of my patch, plus a few lines to check the personality, so setarch(8) can be used to force a 32-bit record to be selected. Thanks to Anton for the idea!
Quoted 1 line
From
083ebfa908c1bfbdabf81e921f3ce6b5d430bcb9Mon Sep 17 00:00:00 2001
From: Ryan C. Gordon <[email protected]>
Date: Mon, 26 Oct 2009 01:50:41 -0400
Subject: [PATCH] binfmt_elf: FatELF support in the binary loader.
This allows the kernel to load a single file that contains multiple ELF binaries (a "FatELF" file), selecting the correct one for the system.
Details, rationale, tools, and patches for handling FatELF binaries can be found at http://icculus.org/fatelf/
--- arch/ia64/ia32/binfmt_elf32.c | 4 +- fs/binfmt_elf.c | 159 ++++++++++++++++++++++++++++++++++------ include/linux/elf.h | 23 ++++++ 3 files changed, 160 insertions(+), 26 deletions(-)
arch/ia64/ia32/binfmt_elf32.c +2 -2
@@ -223,12 +223,12 @@ elf32_set_personality (void)static unsigned long elf32_map (struct file *filep,unsigned long addr,struct elf_phdr *eppnt, -int prot,int type,unsigned long unused) +int prot,int type,unsigned long unused,unsigned long base) {unsigned long pgoff = (eppnt->p_vaddr) & ~IA32_PAGE_MASK ;return ia32_do_mmap (filep, (addr &IA32_PAGE_MASK ), eppnt->p_filesz + pgoff, prot, type, - eppnt->p_offset - pgoff); + (eppnt->p_offset + base) - pgoff); }#define cpu_uses_ia32el () (local_cpu_data->family > 0x1f)
fs/binfmt_elf.c +135 -24
268 lines not on this page · raw message
include/linux/elf.h +23 -0
@@ -33,6 +33,29 @@ typedef __u32 Elf64_Word;typedef __u64 Elf64_Xword ;typedef __s64 Elf64_Sxword ; +/* FatELF (multiple ELF binaries in one file) support */ +#define FATELF_MAGIC (0x1F0E70FA) + +typedef struct Fatelf_record { +__le16 machine;/* maps to e_machine */ +__u8 osabi;/* maps to e_ident[EI_OSABI] */ +__u8 osabi_version;/* maps to e_ident[EI_ABIVERSION] */ +__u8 word_size;/* maps to e_ident[EI_CLASS] */ +__u8 byte_order;/* maps to e_ident[EI_DATA] */ +__u8 reserved0; +__u8 reserved1; +__le64 offset; +__le64 size; +}Fatelf_record ; + +typedef struct Fatelf_hdr { +__le32 magic; +__le16 version; +__u8 num_records; +__u8 reserved0; +Fatelf_record records[]; +}Fatelf_hdr ; +/* These constants are for the segment types stored in the image headers */ #define PT_NULL 0#define PT_LOAD 1
Signature · 1 line
1.6.0.4
(Hopefully final version, now with checkpatch.pl approval!)
Quoted 1 line
From
c6fd97215e9cd1de050bed500195df617f01a291Mon Sep 17 00:00:00 2001
From: Ryan C. Gordon <[email protected]>
Date: Tue, 27 Oct 2009 13:53:39 -0400
Subject: [PATCH] binfmt_elf: FatELF support in the binary loader.
This allows the kernel to load a single file that contains multiple ELF binaries (a "FatELF" file), selecting the correct one for the system.
Details, rationale, tools, and patches for handling FatELF binaries can be found at http://icculus.org/fatelf/
--- arch/ia64/ia32/binfmt_elf32.c | 4 +- fs/binfmt_elf.c | 159 ++++++++++++++++++++++++++++++++++------ include/linux/elf.h | 23 ++++++ 3 files changed, 160 insertions(+), 26 deletions(-)
arch/ia64/ia32/binfmt_elf32.c +2 -2
@@ -223,12 +223,12 @@ elf32_set_personality (void)static unsigned long elf32_map (struct file *filep,unsigned long addr,struct elf_phdr *eppnt, -int prot,int type,unsigned long unused) +int prot,int type,unsigned long unused,unsigned long base) {unsigned long pgoff = (eppnt->p_vaddr) & ~IA32_PAGE_MASK ;return ia32_do_mmap (filep, (addr &IA32_PAGE_MASK ), eppnt->p_filesz + pgoff, prot, type, - eppnt->p_offset - pgoff); + (eppnt->p_offset + base) - pgoff); }#define cpu_uses_ia32el () (local_cpu_data->family > 0x1f)
fs/binfmt_elf.c +135 -24
268 lines not on this page · raw message
include/linux/elf.h +23 -0
@@ -33,6 +33,29 @@ typedef __u32 Elf64_Word;typedef __u64 Elf64_Xword ;typedef __s64 Elf64_Sxword ; +/* FatELF (multiple ELF binaries in one file) support */ +#define FATELF_MAGIC (0x1F0E70FA) + +struct Fatelf_record { +__le16 machine;/* maps to e_machine */ +__u8 osabi;/* maps to e_ident[EI_OSABI] */ +__u8 osabi_version;/* maps to e_ident[EI_ABIVERSION] */ +__u8 word_size;/* maps to e_ident[EI_CLASS] */ +__u8 byte_order;/* maps to e_ident[EI_DATA] */ +__u8 reserved0; +__u8 reserved1; +__le64 offset; +__le64 size; +}; + +struct Fatelf_hdr { +__le32 magic; +__le16 version; +__u8 num_records; +__u8 reserved0; +struct Fatelf_record records[]; +}; +/* These constants are for the segment types stored in the image headers */ #define PT_NULL 0#define PT_LOAD 1
Signature · 1 line
1.6.0.4