Skip to contentSkip to conversation

linux-user: add option to chroot before emulation

PATCH In review Matteo Croce 3 participants 5 messages Dec 8, 2020

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 linux-user: add option to chroot before emulation

From: Matteo Croce <[email protected]>

Add a '-c' option which does a chroot() just before starting the emulation. This is useful when the static QEMU user binary can't be copied into the target root filesystem, e.g. if it's readonly.

Move some code which accesses /proc/sys/vm/mmap_min_addr before the chroot, otherwise it would fail.

---
 linux-user/main.c | 128 +++++++++++++++++++++++++++-------------------
 1 file changed, 75 insertions(+), 53 deletions(-)
linux-user/main.c +75 -53
index 24d1eb73ad..4788e4b5bc 100644
@@ -60,6 +60,7 @@ static const char *seed_optarg;
 unsigned long mmap_min_addr;
 unsigned long guest_base;
 bool have_guest_base;
+static const char *qemu_chroot;
 
 /*
  * Used to implement backwards-compatibility for the `-strace`, and
@@ -304,6 +305,11 @@ static void handle_arg_pagesize(const char *arg)
     }
 }
 
+static void handle_arg_chroot(const char *arg)
+{
+    qemu_chroot = arg;
+}
+
 static void handle_arg_seed(const char *arg)
 {
     seed_optarg = arg;
@@ -450,6 +456,8 @@ static const struct qemu_argument arg_table[] = {
      "logfile",     "write logs to 'logfile' (default stderr)"},
     {"p",          "QEMU_PAGESIZE",    true,  handle_arg_pagesize,
      "pagesize",   "set the host page size to 'pagesize'"},
+    {"c",          "QEMU_CHROOT",      true,  handle_arg_chroot,
+     "chroot",     "chroot to 'chroot' before starting emulation"},
     {"singlestep", "QEMU_SINGLESTEP",  false, handle_arg_singlestep,
      "",           "run in singlestep mode"},
     {"strace",     "QEMU_STRACE",      false, handle_arg_strace,
@@ -688,6 +696,73 @@ int main(int argc, char **argv, char **envp)
 
     init_qemu_uname_release();
 
+    /*
+     * Read in mmap_min_addr kernel parameter.  This value is used
+     * When loading the ELF image to determine whether guest_base
+     * is needed.  It is also used in mmap_find_vma.
+     */
+    {
+        FILE *fp;
+
+        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {
+            unsigned long tmp;
+            if (fscanf(fp, "%lu", &tmp) == 1 && tmp != 0) {
+                mmap_min_addr = tmp;
+                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n",
+                              mmap_min_addr);
+            }
+            fclose(fp);
+        }
+    }
+
+    /*
+     * We prefer to not make NULL pointers accessible to QEMU.
+     * If we're in a chroot with no /proc, fall back to 1 page.
+     */
+    if (mmap_min_addr == 0) {
+        mmap_min_addr = qemu_host_page_size;
+        qemu_log_mask(CPU_LOG_PAGE,
+                      "host mmap_min_addr=0x%lx (fallback)\n",
+                      mmap_min_addr);
+    }
+
+    /*
+     * Prepare copy of argv vector for target.
+     */
+    target_argc = argc - optind;
+    target_argv = calloc(target_argc + 1, sizeof (char *));
+    if (target_argv == NULL) {
+        (void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
+        exit(EXIT_FAILURE);
+    }
+
+    /*
+     * If argv0 is specified (using '-0' switch) we replace
+     * argv[0] pointer with the given one.
+     */
+    i = 0;
+    if (argv0 != NULL) {
+        target_argv[i++] = strdup(argv0);
+    }
+    for (; i < target_argc; i++) {
+        target_argv[i] = strdup(argv[optind + i]);
+    }
+    target_argv[target_argc] = NULL;
+
+    /*
+     * Change root if requested wuth '-c'
+     */
+    if (qemu_chroot) {
+        if (chroot(qemu_chroot) < 0) {
+            error_report("chroot failed");
+            exit(1);
+        }
+        if (chdir("/")) {
+            error_report("not able to chdir to /: %s", strerror(errno));
+            exit(1);
+        }
+    }
+
     execfd = qemu_getauxval(AT_EXECFD);
     if (execfd == 0) {
         execfd = open(exec_path, O_RDONLY);
@@ -746,59 +821,6 @@ int main(int argc, char **argv, char **envp)
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-    /*
-     * Read in mmap_min_addr kernel parameter.  This value is used
-     * When loading the ELF image to determine whether guest_base
-     * is needed.  It is also used in mmap_find_vma.
-     */
-    {
-        FILE *fp;
-
-        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {
-            unsigned long tmp;
-            if (fscanf(fp, "%lu", &tmp) == 1 && tmp != 0) {
-                mmap_min_addr = tmp;
-                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n",
-                              mmap_min_addr);
-            }
-            fclose(fp);
-        }
-    }
-
-    /*
-     * We prefer to not make NULL pointers accessible to QEMU.
-     * If we're in a chroot with no /proc, fall back to 1 page.
-     */
-    if (mmap_min_addr == 0) {
-        mmap_min_addr = qemu_host_page_size;
-        qemu_log_mask(CPU_LOG_PAGE,
-                      "host mmap_min_addr=0x%lx (fallback)\n",
-                      mmap_min_addr);
-    }
-
-    /*
-     * Prepare copy of argv vector for target.
-     */
-    target_argc = argc - optind;
-    target_argv = calloc(target_argc + 1, sizeof (char *));
-    if (target_argv == NULL) {
-        (void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
-        exit(EXIT_FAILURE);
-    }
-
-    /*
-     * If argv0 is specified (using '-0' switch) we replace
-     * argv[0] pointer with the given one.
-     */
-    i = 0;
-    if (argv0 != NULL) {
-        target_argv[i++] = strdup(argv0);
-    }
-    for (; i < target_argc; i++) {
-        target_argv[i] = strdup(argv[optind + i]);
-    }
-    target_argv[target_argc] = NULL;
-
     ts = g_new0(TaskState, 1);
     init_task_state(ts);
     /* build Task State */
Signature · 1 line
2.28.0
#1 lore raw
Patchew URL: https://patchew.org/QEMU/[email protected]/

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Type: series
Message-id: [email protected]
Subject: [PATCH] linux-user: add option to chroot before emulation

=== TEST SCRIPT BEGIN ===
#!/bin/bash
git rev-parse base > /dev/null || exit 0
git config --local diff.renamelimit 0
git config --local diff.renames True
git config --local diff.algorithm histogram
./scripts/checkpatch.pl --mailback base..
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384 From https://github.com/patchew-project/qemu
* [new tag] patchew/[email protected] -> patchew/[email protected]
Switched to a new branch 'test'
939b01d linux-user: add option to chroot before emulation

=== OUTPUT BEGIN ===
ERROR: do not use assignment in if condition
#62: FILE: linux-user/main.c:707:
+ if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {

ERROR: space prohibited between function name and open parenthesis '(' #88: FILE: linux-user/main.c:733:
+ target_argv = calloc(target_argc + 1, sizeof (char *));

total: 2 errors, 0 warnings, 158 lines checked

Commit 939b01d16330 (linux-user: add option to chroot before emulation) has style problems, please review. If any of these errors are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.
=== OUTPUT END ===

Test command exited with code: 1

The full log is available at
http://patchew.org/logs/[email protected]/testing.checkpatch/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]

#2 lore raw
In reply to Matteo Croce

Le 08/12/2020 à 01:17, Matteo Croce a écrit :

Quoted 5 lines

From: Matteo Croce <[email protected]>

Add a '-c' option which does a chroot() just before starting the emulation. This is useful when the static QEMU user binary can't be copied into the target root filesystem, e.g. if it's readonly.

Did you try to use the binfmt_misc 'F' flag (fix binary)?

https://www.kernel.org/doc/Documentation/admin-guide/binfmt-misc.rst

``F`` - fix binary

The usual behaviour of binfmt_misc is to spawn the binary lazily when the misc format file is invoked. However, this doesn``t work very well in the face of mount namespaces and changeroots, so the ``F`` mode opens the binary as soon as the emulation is installed and uses the opened image to spawn the emulator, meaning it is always available once installed, regardless of how the environment changes.

This can be configured with scripts/qemu-binfmt-conf.sh and "--persistent yes"" option

Thanks,
Laurent

Quoted 177 lines

Move some code which accesses /proc/sys/vm/mmap_min_addr before the chroot, otherwise it would fail.

---
 linux-user/main.c | 128 +++++++++++++++++++++++++++-------------------
 1 file changed, 75 insertions(+), 53 deletions(-)
linux-user/main.c +75 -53
index 24d1eb73ad..4788e4b5bc 100644
@@ -60,6 +60,7 @@ static const char *seed_optarg;
 unsigned long mmap_min_addr;
 unsigned long guest_base;
 bool have_guest_base;
+static const char *qemu_chroot;
 
 /*
  * Used to implement backwards-compatibility for the `-strace`, and
@@ -304,6 +305,11 @@ static void handle_arg_pagesize(const char *arg)
     }
 }
 
+static void handle_arg_chroot(const char *arg)
+{
+    qemu_chroot = arg;
+}
+
 static void handle_arg_seed(const char *arg)
 {
     seed_optarg = arg;
@@ -450,6 +456,8 @@ static const struct qemu_argument arg_table[] = {
      "logfile",     "write logs to 'logfile' (default stderr)"},
     {"p",          "QEMU_PAGESIZE",    true,  handle_arg_pagesize,
      "pagesize",   "set the host page size to 'pagesize'"},
+    {"c",          "QEMU_CHROOT",      true,  handle_arg_chroot,
+     "chroot",     "chroot to 'chroot' before starting emulation"},
     {"singlestep", "QEMU_SINGLESTEP",  false, handle_arg_singlestep,
      "",           "run in singlestep mode"},
     {"strace",     "QEMU_STRACE",      false, handle_arg_strace,
@@ -688,6 +696,73 @@ int main(int argc, char **argv, char **envp)
 
     init_qemu_uname_release();
 
+    /*
+     * Read in mmap_min_addr kernel parameter.  This value is used
+     * When loading the ELF image to determine whether guest_base
+     * is needed.  It is also used in mmap_find_vma.
+     */
+    {
+        FILE *fp;
+
+        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {
+            unsigned long tmp;
+            if (fscanf(fp, "%lu", &tmp) == 1 && tmp != 0) {
+                mmap_min_addr = tmp;
+                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n",
+                              mmap_min_addr);
+            }
+            fclose(fp);
+        }
+    }
+
+    /*
+     * We prefer to not make NULL pointers accessible to QEMU.
+     * If we're in a chroot with no /proc, fall back to 1 page.
+     */
+    if (mmap_min_addr == 0) {
+        mmap_min_addr = qemu_host_page_size;
+        qemu_log_mask(CPU_LOG_PAGE,
+                      "host mmap_min_addr=0x%lx (fallback)\n",
+                      mmap_min_addr);
+    }
+
+    /*
+     * Prepare copy of argv vector for target.
+     */
+    target_argc = argc - optind;
+    target_argv = calloc(target_argc + 1, sizeof (char *));
+    if (target_argv == NULL) {
+        (void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
+        exit(EXIT_FAILURE);
+    }
+
+    /*
+     * If argv0 is specified (using '-0' switch) we replace
+     * argv[0] pointer with the given one.
+     */
+    i = 0;
+    if (argv0 != NULL) {
+        target_argv[i++] = strdup(argv0);
+    }
+    for (; i < target_argc; i++) {
+        target_argv[i] = strdup(argv[optind + i]);
+    }
+    target_argv[target_argc] = NULL;
+
+    /*
+     * Change root if requested wuth '-c'
+     */
+    if (qemu_chroot) {
+        if (chroot(qemu_chroot) < 0) {
+            error_report("chroot failed");
+            exit(1);
+        }
+        if (chdir("/")) {
+            error_report("not able to chdir to /: %s", strerror(errno));
+            exit(1);
+        }
+    }
+
     execfd = qemu_getauxval(AT_EXECFD);
     if (execfd == 0) {
         execfd = open(exec_path, O_RDONLY);
@@ -746,59 +821,6 @@ int main(int argc, char **argv, char **envp)
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-    /*
-     * Read in mmap_min_addr kernel parameter.  This value is used
-     * When loading the ELF image to determine whether guest_base
-     * is needed.  It is also used in mmap_find_vma.
-     */
-    {
-        FILE *fp;
-
-        if ((fp = fopen("/proc/sys/vm/mmap_min_addr", "r")) != NULL) {
-            unsigned long tmp;
-            if (fscanf(fp, "%lu", &tmp) == 1 && tmp != 0) {
-                mmap_min_addr = tmp;
-                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n",
-                              mmap_min_addr);
-            }
-            fclose(fp);
-        }
-    }
-
-    /*
-     * We prefer to not make NULL pointers accessible to QEMU.
-     * If we're in a chroot with no /proc, fall back to 1 page.
-     */
-    if (mmap_min_addr == 0) {
-        mmap_min_addr = qemu_host_page_size;
-        qemu_log_mask(CPU_LOG_PAGE,
-                      "host mmap_min_addr=0x%lx (fallback)\n",
-                      mmap_min_addr);
-    }
-
-    /*
-     * Prepare copy of argv vector for target.
-     */
-    target_argc = argc - optind;
-    target_argv = calloc(target_argc + 1, sizeof (char *));
-    if (target_argv == NULL) {
-        (void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
-        exit(EXIT_FAILURE);
-    }
-
-    /*
-     * If argv0 is specified (using '-0' switch) we replace
-     * argv[0] pointer with the given one.
-     */
-    i = 0;
-    if (argv0 != NULL) {
-        target_argv[i++] = strdup(argv0);
-    }
-    for (; i < target_argc; i++) {
-        target_argv[i] = strdup(argv[optind + i]);
-    }
-    target_argv[target_argc] = NULL;
-
     ts = g_new0(TaskState, 1);
     init_task_state(ts);
     /* build Task State */
#3 lore raw
Quoted 22 lines · Laurent Vivier
Quoted 5 lines · Matteo Croce

From: Matteo Croce <[email protected]>

Add a '-c' option which does a chroot() just before starting the emulation. This is useful when the static QEMU user binary can't be copied into the target root filesystem, e.g. if it's readonly.

Did you try to use the binfmt_misc 'F' flag (fix binary)?

https://www.kernel.org/doc/Documentation/admin-guide/binfmt-misc.rst

``F`` - fix binary

The usual behaviour of binfmt_misc is to spawn the binary lazily when the misc format file is invoked. However, this doesn``t work very well in the face of mount namespaces and changeroots, so the ``F`` mode opens the binary as soon as the emulation is installed and uses the opened image to spawn the emulator, meaning it is always available once installed, regardless of how the environment changes.

Quoted 3 lines

This can be configured with scripts/qemu-binfmt-conf.sh and "--persistent yes"" option

Yes, this works too.
Basically it's the same trick, open the binary early and then emulate. The only difference is that with binfmt emulation the procfs open still fails:

# strace -feopenat chroot debian /bin/true 2>&1 |grep /proc/sys
[pid  9359] openat(AT_FDCWD, "/proc/sys/vm/mmap_min_addr", O_RDONLY) =
-1 ENOENT (No such file or directory)

vs

# strace -feopenat qemu-aarch64 -c debian /bin/true 2>&1 |grep /proc/sys
[pid  9348] openat(AT_FDCWD, "/proc/sys/vm/mmap_min_addr", O_RDONLY) = 3
Signature · 1 line
per aspera ad upstream
#4 lore raw

Le 08/12/2020 à 17:04, Matteo Croce a écrit :

Quoted 32 lines · Laurent Vivier
Quoted 22 lines · Matteo Croce
Quoted 5 lines

From: Matteo Croce <[email protected]>

Add a '-c' option which does a chroot() just before starting the emulation. This is useful when the static QEMU user binary can't be copied into the target root filesystem, e.g. if it's readonly.

Did you try to use the binfmt_misc 'F' flag (fix binary)?

https://www.kernel.org/doc/Documentation/admin-guide/binfmt-misc.rst

``F`` - fix binary

The usual behaviour of binfmt_misc is to spawn the binary lazily when the misc format file is invoked. However, this doesn``t work very well in the face of mount namespaces and changeroots, so the ``F`` mode opens the binary as soon as the emulation is installed and uses the opened image to spawn the emulator, meaning it is always available once installed, regardless of how the environment changes.

Quoted 3 lines

This can be configured with scripts/qemu-binfmt-conf.sh and "--persistent yes"" option

Yes, this works too.
Basically it's the same trick, open the binary early and then emulate. The only difference is that with binfmt emulation the procfs open still fails:

Quoted 9 lines
# strace -feopenat chroot debian /bin/true 2>&1 |grep /proc/sys
[pid  9359] openat(AT_FDCWD, "/proc/sys/vm/mmap_min_addr", O_RDONLY) =
-1 ENOENT (No such file or directory)

vs

# strace -feopenat qemu-aarch64 -c debian /bin/true 2>&1 |grep /proc/sys
[pid  9348] openat(AT_FDCWD, "/proc/sys/vm/mmap_min_addr", O_RDONLY) = 3

If you want to have the /proc mounted earlier you can use "unshare".

something like:

unshare --mount-proc -R debian /bin/true

There is also the "-L" option, something like:

qemu-aarch64 -L debian debian/bin/true

Thanks,
Laurent