qmp: extend QMP to provide read/write access to physical memory

This patch adds a new QMP command that sets up a domain socket. This socket can then be used for fast read/write access to the guest's physical memory. The key benefit to this system over existing solutions is speed. Using this patch, guest memory can be copied out at a rate of ~200MB/sec, depending on the hardware. Existing solutions only achieve a small fraction of this speed.

Signed-off-by Bryan D. Payne <[email protected]>

---
 Makefile.target |   2 +-
 memory-access.c | 200 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 memory-access.h |  11 ++++
 monitor.c       |  10 +++
 qmp-commands.hx |  27 ++++++++

+++ b/qmp-commands.hx

qmp-commands.hx +12 -0

@@ -609,6 +609,33 @@ Example:
 EQMP
 
     {
+        .name       = "pmemaccess",
+        .args_type  = "path:s",
+        .params     = "path",
+        .help       = "mount guest physical memory image at 'path'",
+        .user_print = monitor_user_noop,
+        .mhandler.cmd_new = do_physical_memory_access,
+    },
+
+SQMP
+pmemaccess
+----------
+

This patch adds a new QMP command that sets up a domain socket. This socket can then be used for fast read/write access to the guest's physical memory. The key benefit to this system over existing solutions is speed. Using this patch, guest memory can be copied out at a rate of ~200MB/sec, depending on the hardware. Existing solutions only achieve a small fraction of this speed.

Signed-off-by Bryan D. Payne <[email protected]>

---
 Makefile.target  |   2 +-
 memory-access.c  | 200 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 memory-access.h  |  11 +++
 monitor.c        |   6 ++
 qapi-schema.json |  12 ++++
 qmp-commands.hx  |  28 ++++++++
 6 files changed, 258 insertions(+), 1 deletion(-)
 create mode 100644 memory-access.c
 create mode 100644 memory-access.h

Makefile.target +1 -1

index e9ff1ee..4b3cd99 100644
@@ -127,7 +127,7 @@ endif #CONFIG_BSD_USER
 # System emulator target
 ifdef CONFIG_SOFTMMU
 obj-y += arch_init.o cpus.o monitor.o gdbstub.o balloon.o ioport.o numa.o
-obj-y += qtest.o bootdevice.o
+obj-y += qtest.o bootdevice.o memory-access.o
 obj-y += hw/
 obj-$(CONFIG_FDT) += device_tree.o
 obj-$(CONFIG_KVM) += kvm-all.o

memory-access.c new file +18 -0

new file mode 100644
index 0000000..f696d7b
@@ -0,0 +1,200 @@
+/*
+ * Access guest physical memory via a domain socket.
+ *
+ * Copyright (c) 2014 Bryan D. Payne ([email protected])
+ */
+
+#include "memory-access.h"
+#include "qemu-common.h"
+#include "exec/cpu-common.h"
+#include "config.h"
+
+#include <glib.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <pthread.h>
+#include <sys/types.h>
+#include <sys/socket.h>

memory-access.c new file +33 -0

+    if (!guestmem) {
+        return 0;
+    }
+    memcpy(buf, guestmem, len);
+    cpu_physical_memory_unmap(guestmem, len, 0, len);
+
+    return len;
+}
+
+static uint64_t
+connection_write_memory(uint64_t user_paddr,
+                        const void *buf,
+                        uint64_t user_len)
+{
+    hwaddr paddr = (hwaddr) user_paddr;
+    hwaddr len = (hwaddr) user_len;
+    void *guestmem = cpu_physical_memory_map(paddr, &len, 1);
+    if (!guestmem) {
+        return 0;
+    }
+    memcpy(guestmem, buf, len);
+    cpu_physical_memory_unmap(guestmem, len, 0, len);
+
+    return len;
+}
+
+static void
+send_success_ack(int connection_fd)
+{
+    uint8_t success = 1;
+    int nbytes = write(connection_fd, &success, 1);
+    if (nbytes != 1) {
+        printf("QemuMemoryAccess: failed to send success ack\n");

memory-access.c new file +7 -0

+    }
+}
+
+static void
+connection_handler(int connection_fd)
+{
+    int nbytes;

memory-access.c new file +6 -0

+            nbytes = connection_read_memory(req.address, buf, req.length);
+            if (nbytes != req.length) {
+                /* read failure, return failure message */
+                buf[req.length] = 0; /* set last byte to 0 for failure */
+                nbytes = write(connection_fd, buf, 1);
+            } else {

memory-access.c new file +34 -0

+                /* do the write */
+                nbytes = connection_write_memory(req.address,
+                                                 write_buf,
+                                                 req.length);
+                if (nbytes == req.length) {
+                    send_success_ack(connection_fd);
+                } else {
+                    send_fail_ack(connection_fd);
+                }
+            }
+            g_free(write_buf);
+        } else {
+            /* unknown command */
+            printf("QemuMemoryAccess: ignoring unknown command (%d)\n",
+                   req.type);
+            send_fail_ack(connection_fd);
+        }
+    }
+
+    close(connection_fd);
+}
+
+static void *
+memory_access_thread(void *path)
+{
+    struct sockaddr_un address;
+    int socket_fd, connection_fd, path_len;
+    socklen_t address_length;
+
+    socket_fd = socket(PF_UNIX, SOCK_STREAM, 0);
+    if (socket_fd < 0) {
+        printf("QemuMemoryAccess: socket failed\n");
+        goto error_exit;
+    }

memory-access.c new file +18 -0

+    }
+    if (listen(socket_fd, 0) != 0) {
+        printf("QemuMemoryAccess: listen failed\n");
+        goto error_exit;
+    }
+
+    connection_fd = accept(socket_fd,
+                           (struct sockaddr *) &address,
+                           &address_length);
+    connection_handler(connection_fd);
+
+    close(socket_fd);
+    unlink(path);
+error_exit:
+    g_free(path);
+    return NULL;
+}
+

memory-access.c new file +9 -0

+
+    /* start the thread */
+    sigfillset(&set);
+    pthread_sigmask(SIG_SETMASK, &set, &oldset);
+    ret = pthread_create(&thread, NULL, memory_access_thread, pathcopy);
+    pthread_sigmask(SIG_SETMASK, &oldset, NULL);
+
+    return ret;
+}

memory-access.h new file +11 -0

new file mode 100644
index 0000000..cb5fe33
@@ -0,0 +1,11 @@
+/*
+ * Access guest physical memory via a domain socket.
+ *
+ * Copyright (c) 2014 Bryan D. Payne ([email protected])
+ */
+#ifndef MEMORY_ACCESS_H
+#define MEMORY_ACCESS_H
+
+int memory_access_start(const char *path);
+
+#endif /* MEMORY_ACCESS_H */

monitor.c +6 -0

index fa00594..19724b8 100644
@@ -72,6 +72,7 @@
 #include "block/qapi.h"
 #include "qapi/qmp-event.h"
 #include "qapi-event.h"
+#include "memory-access.h"
 
 /* for pic/irq_info */
 #if defined(TARGET_SPARC)

@@ -1371,6 +1372,11 @@ static void do_print(Monitor *mon, const QDict *qdict)
     monitor_printf(mon, "\n");
 }
 
+void qmp_pmemaccess(const char *path, Error **err)
+{
+    memory_access_start(path);
+}
+
 static void do_sum(Monitor *mon, const QDict *qdict)
 {
     uint32_t addr;

qapi-schema.json +12 -0

index 9ffdcf8..37a6657 100644
@@ -3515,3 +3515,15 @@
 # Since: 2.1
 ##
 { 'command': 'rtc-reset-reinjection' }
+
+##
+# @pmemaccess
+#
+# This command enables access to guest physical memory using
+# a simple protocol over a UNIX domain socket.
+#
+# @path Location to use for the UNIX domain socket
+#
+# Since: 2.3
+##
+{ 'command': 'pmemaccess', 'data': { 'path': 'str' } }

qmp-commands.hx +4 -0

index 718dd92..fab1322 100644
@@ -609,6 +609,34 @@ Example:
 EQMP
 
     {
+        .name       = "pmemaccess",
+        .args_type  = "path:s",
+        .params     = "path",
+        .help       = "access to guest memory via domain socket at 'path'",

qmp-commands.hx +14 -0

+
+Arguments:
+
+- "path": path for domain socket (json-string)
+
+Example:
+
+-> { "execute": "pmemaccess",
+             "arguments": { "path": "/tmp/guestname" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "migrate",
         .args_type  = "detach:-d,blk:-b,inc:-i,uri:s",
         .mhandler.cmd_new = qmp_marshal_input_migrate,

Signature · 1 line

1.9.1

Out of curiosity, what are existing solutions?

+struct request {

memory-access.c line 28

Quoted 4 lines

memory-access.c new file +4 -0

+    uint8_t type;      /* 0 quit, 1 read, 2 write, ... rest reserved */
+    uint64_t address;  /* address to read from OR write to */
+    uint64_t length;   /* number of bytes to read OR write */
+};

Please add QEMU_PACKED to this structure, and probably name it QEMUMARequest,
for name collision avoidance and CamelCase convension.

Quoted 3 lines

Out of curiosity, what are existing solutions?

Basically just attaching gdb and pulling memory out manually (or writing a program to do the same).

Quoted 6 lines

+struct request {

Quoted 4 lines

memory-access.c new file +4 -0

+    uint8_t type;      /* 0 quit, 1 read, 2 write, ... rest reserved */
+    uint64_t address;  /* address to read from OR write to */
+    uint64_t length;   /* number of bytes to read OR write */
+};

Quoted 4 lines

Please add QEMU_PACKED to this structure, and probably name it QEMUMARequest,
for name collision avoidance and CamelCase convension.

How critical is QEMU_PACKED? This changes the layout of the struct which means that existing clients (LibVMI) would need to change their code as well.

It is critical as a transport data structure. You have to define a byte-by-byte
layout (concerning endianness and padding) and use padding fields together with
QEMU_PACKED, so the representation is not dependent on alignment

Quoted 6 lines

It is critical as a transport data structure. You have to define a byte-by-byte
layout (concerning endianness and padding) and use padding fields together with
QEMU_PACKED, so the representation is not dependent on alignment

This makes sense for network protocols. But, in this case, the protocol is always taking placing over a unix socket on the localhost. This is why I wasn't sure that it was necessary.

Just to be clear... the QMP connection could go over the network, but its only purpose is to set up the unix socket on the local machine (meaning that doing it over the network is kind of pointless, even if it is possible). Once the unix socket is setup, the memory access protocol happens over that socket.

This doesn't stop the client from using a different alignment than we expect.
It's necessary to be explicit as a binary protocol.

Quoted 5 lines

This doesn't stop the client from using a different alignment than we expect.
It's necessary to be explicit as a binary protocol.

Ok, I'll move ahead with packing the data and sort out the backwards compat issues on the client side.

Quoted 3 lines

Out of curiosity, what are existing solutions?

Basically just attaching gdb and pulling memory out manually (or writing a program to do the same).

Can you explain again why the existing commands to read guest memory (from the top of my head: dump-guest-memory, memsave, pmemsave) are insufficient? How does your solution improve on them? What exactly can it do what these commands can't? What exactly can't it do what these commands can?

I feel we need to understand the answers to these questions to sensibly evolve the API in this area.

Quoted 9 lines

Can you explain again why the existing commands to read guest memory (from the top of my head: dump-guest-memory, memsave, pmemsave) are insufficient? How does your solution improve on them? What exactly can it do what these commands can't? What exactly can't it do what these commands can?

I feel we need to understand the answers to these questions to sensibly evolve the API in this area.

Certainly. The main issue with this series of commands is that they dump the memory to a file on disk. What I'm trying to facilitate here is an application that monitors the guest memory in real time while the guest is running (likely with brief pauses during memory analysis periods). Writing all of this data to disk, and then reading it back again for the analysis application is several orders of magnitude too slow for these types of applications.

This new method that I'm proposing here keeps everything in memory, which makes it much faster.

Tldr; existing solutions are suitable for forensic analysis, whereas I'm looking to solve the runtime analysis problem.

Actually, it sends it through a UNIX domain socket. Still I/O, just different I/O. "Faster" is plausible, but by how much I can't say without measurements. "Several orders of magnitude" needs evidence.

sets up a special-purpose monitor-like thing on a newly created a UNIX domain socket. This monitor-like thing talks its own language, which is so far undocmented.

Why restrict this to a UNIX domain socket? In other places involving byte streams we use a QEMU character device.

Can you explain why you need a separate stream instead of embedding the conversation in QMP? We already have such embedded I/O: ringbuf-read, ringbuf-write. Yes, it got multiple documented bugs, but that's because it was badly done, not because such a thing couldn't be done nicely.

document the language. A file in docs/ should do. The QMP command documentation should point to it.

Quoted 5 lines

Actually, it sends it through a UNIX domain socket. Still I/O, just different I/O. "Faster" is plausible, but by how much I can't say without measurements. "Several orders of magnitude" needs evidence.

I've been testing introspection performance on both Xen and KVM for over 5 years now, and I'm pretty comfortable with this statement. Bottom line is that sending lots of data through a disk is much slower than sending lots of data through memory (for example, see Figure 3 in
http://queue.acm.org/detail.cfm?id=1563874, which shows about 5 orders of magnitude difference for random access data@). A UNIX domain socket is just going through memory.

I have other Qemu modifications that can reduce the use of the memory bus by using shared memory instead. This provides another big bump in performance. However, this work is still in earlier stages.

Based on your qapi-schema.json change only, I figure your new command

Quoted 4 lines

sets up a special-purpose monitor-like thing on a newly created a UNIX domain socket. This monitor-like thing talks its own language, which is so far undocmented.

Correct. Per request from one of the reviewers in this thread, I have an updated patch that has full documentation on this protocol. I've been waiting to push that as I try to sort out some of the remaining discussion in this thread, though.

Quoted 8 lines

Why restrict this to a UNIX domain socket? In other places involving byte streams we use a QEMU character device.

Can you explain why you need a separate stream instead of embedding the conversation in QMP? We already have such embedded I/O: ringbuf-read, ringbuf-write. Yes, it got multiple documented bugs, but that's because it was badly done, not because such a thing couldn't be done nicely.

I think that it could be done in QMP. I would want to validate the performance of that approach though as there will be some extra overhead.

I would also be concerned about access to this data over a network.

For security reasons, it feels better to me to restrict this feature to someone that is on the local machine. Using a UNIX domain socket forces this constraint. I don't believe that is the case with QMP, but please correct me if I'm wrong about that.

If you add a separate stream talking its own language, you get to

Quoted 3 lines

document the language. A file in docs/ should do. The QMP command documentation should point to it.

My updated patch documents it as a comment directly in the C source, as that is what I felt a previous reviewer was asking for. Is docs more appropriate or ??

By "evidence", I mean actual numbers for actual QEMU code. Nothing sophisticated, just use your new interface in a way you consider relevant for your own use case, then approximate this use with existing interfaces. The approximation can be very rough. For instance, showing that doing the whole job with your approach is a much faster than doing a necessary part of the job with existing commands would do.

The QMP overhead could be too high. QMP is control plane, not data plane. How much data do you envisage flowing here?

In theory, even the character device overhead could be too high. Measuring it shouldn't be too hard.

I feel code comment is find for internal interfaces, but for external interfaces, a separate interface document is more appropriate.

By "evidence", I mean actual numbers for actual QEMU code. Nothing

Quoted 6 lines

sophisticated, just use your new interface in a way you consider relevant for your own use case, then approximate this use with existing interfaces. The approximation can be very rough. For instance, showing that doing the whole job with your approach is a much faster than doing a necessary part of the job with existing commands would do.

Sure, I can put together some numbers to help with this discussion.

Quoted 10 lines

By "evidence", I mean actual numbers for actual QEMU code. Nothing

Quoted 6 lines

sophisticated, just use your new interface in a way you consider relevant for your own use case, then approximate this use with existing interfaces. The approximation can be very rough. For instance, showing that doing the whole job with your approach is a much faster than doing a necessary part of the job with existing commands would do.

Sure, I can put together some numbers to help with this discussion.

I've done some performance testing on my laptop. I have setup LibVMI to access KVM guest memory using one of three different options:

QMP/pmemsave (dump bytes to a file and then read that back into memory) HMC/xp command (dump bytes via human monitor command and repack the output to binary form)
QMP/pmemaccess (access bytes through a Unix socket, this is the patch we are discussing in this thread)

For each of these I measured the time required to translate a guest kernel virtual address into a guest physical address. This is done with a 64-bit Ubuntu Trusty guest and the addresses are translated by walking the page tables in the guest using the selected memory access technique.

You can see the graph here:
http://cl.ly/image/322a3s0h1V05

For those that prefer text, here's the numbers (in microseconds):
QMP/pmemsave: 77706
HMC/xp command: 92552
QMP/pmemaccess: 95

As you can see, the technique proposed in this patch is about 3 orders of magnitude faster than the options available in Qemu today. I will also note that this is for a _single_ virtual address translation. A typical introspection application will be doing much more work than this, which will only compound the problem.

Hopefully this helps to explain why I believe that this is an improvement to the existing mechanisms. Please let me know if you have any questions about these results.

I'll continue some exploration of doing the pmemaccess bulk data transfer over QMP, instead of the UNIX socket. And I'll report back here once I've come up with some thoughts on the best way to proceed.

Look good. I believe QMP will be in between, and if it doesn't work as well,
could you also try to use QEMU's char dev instead of limit this to unix socket?

Quoted 5 lines

For those that prefer text, here's the numbers (in microseconds):
QMP/pmemsave: 77706
HMC/xp command: 92552
QMP/pmemaccess: 95

could you also try to use QEMU's char dev instead of limit this to unix socket?

Quoted 6 lines

Quoted 5 lines

For those that prefer text, here's the numbers (in microseconds):
QMP/pmemsave: 77706
HMC/xp command: 92552
QMP/pmemaccess: 95

I completed a proof of concept implementation for doing this memory access completely over QMP. Basically, it works much like pmemsave except instead of sending the data to a file it does a base64 encode and sends it back over the QMP connection. Interestingly, my testing shows that this is slightly slower than the pmemsave option. Running the same test as before (virtual address translation), I get 88063 microsecs on average. So I don't believe this option is viable.

Look good. I believe QMP will be in between, and if it doesn't work as well,

Quoted 2 lines

could you also try to use QEMU's char dev instead of limit this to unix socket?

I'm moving forward to try the Qemu chardev approach now. I haven't working much with this construct before, so any pointers are appreciated. From what I'm seeing, it looks like the user would create a chardev using one of the QMP @chardev* commands? The schema doesn't indicate that these commands return the chardev id, which seems odd as I was then assuming that one could obtain the id and pass this into the pmemaccess QMP command. Thoughts?

This patch adds a new QMP command that sets up a domain socket. This socket can then be used for fast read/write access to the guest's physical memory. The key benefit to this system over existing solutions is speed. Using this patch, guest memory can be copied out at a rate of ~200MB/sec, depending on the hardware. Existing solutions only achieve a small fraction of this speed.

Signed-off-by Bryan D. Payne <[email protected]>

qapi-schema.json +2 -0

+
+##

In addition to Fam's review, I have a question - does this code properly use qemu_open() so that I can use 'add-fd' to pass in a pre-opened socket fd into fdset 1, then call pmemaccess with '/dev/fdset/1'? If not, can you please fix it to allow this usage?

Quoted 6 lines

In addition to Fam's review, I have a question - does this code properly use qemu_open() so that I can use 'add-fd' to pass in a pre-opened socket fd into fdset 1, then call pmemaccess with '/dev/fdset/1'? If not, can you please fix it to allow this usage?

I'm not familiar with this setup. Could you point me to an example of where this is done properly somewhere else in the Qemu code base? Once I figure this out, I'll be able to post the v3 patch.

Off the top of my head, I know the -tpm command line options (related to the 'query-tpm' QMP command) do this; look at hw/tpm/tpm_passthrough.c for that implementation.

Off the top of my head, I know the -tpm command line options (related to the 'query-tpm' QMP command) do this; look at hw/tpm/tpm_passthrough.c for that implementation.

Quoted 4 lines

Off the top of my head, I know the -tpm command line options (related to the 'query-tpm' QMP command) do this; look at hw/tpm/tpm_passthrough.c for that implementation.

So now I do see what you are talking about. But I don't think it applies to this patch. I'm not using qemu_open in this patch.

Thanks for the feedback Eric, I've updated the patch.

v2 changes:
- added QMP command contract to qapi-schema.json
- corrected some comments
- rewired QMP command to use schema code

Quoted 6 lines · Bryan D. Payne

Thanks for the feedback Eric, I've updated the patch.

v2 changes:
- added QMP command contract to qapi-schema.json
- corrected some comments
- rewired QMP command to use schema code

When sending a v2, it's best to send it as a new top-level thread, instead of buried in-reply-to an existing thread. Also, for a single patch, a cover letter is not strictly necessary (the information you gave here can instead be given after the --- separator of the one-and-only patch email). Cover letters are mandatory only for multi-patch series.

--
Eric Blake eblake redhat com +1-919-301-3266
Libvirt virtualization library http://libvirt.org

Ok thanks for the advice, I'll adjust for v3. This is (clearly!) my first contribution to Qemu so I'm still learning how you guys operate.

Cheers,
-bryan

Quoted 20 lines · Eric Blake

Quoted 6 lines · Bryan D. Payne

Thanks for the feedback Eric, I've updated the patch.

v2 changes:
- added QMP command contract to qapi-schema.json
- corrected some comments
- rewired QMP command to use schema code

When sending a v2, it's best to send it as a new top-level thread, instead of buried in-reply-to an existing thread. Also, for a single patch, a cover letter is not strictly necessary (the information you gave here can instead be given after the --- separator of the one-and-only patch email). Cover letters are mandatory only for multi-patch series.

--
Eric Blake eblake redhat com +1-919-301-3266
Libvirt virtualization library http://libvirt.org