Skip to contentSkip to conversation

madvise() not triggering page discard with MMU-notifiers

PATCH Anthony Liguori 3 participants 5 messages Aug 18, 2008 – Aug 28, 2008 KVM

lore
Share & tools
lore
#0 lore raw

Attached is an updated migration patch. It does not appear, however, that madvise() is triggering discarding of pages even with an MMU-notifier enabled kernel. I take it that this is because we're still holding a reference count to the page when it is in the shadow page table? Any ETA on when this will change?

N.B. the virtio_balloon driver is broken in Linus' tree. Sending patch for that following this one.

Regards,

Anthony Liguori

libkvm/libkvm.c +9 -0
index 5edfad7..ebee5b8 100644
@@ -1053,6 +1053,15 @@ int kvm_pit_in_kernel(kvm_context_t kvm)
 	return kvm->pit_in_kernel;
 }
 
+int kvm_has_mmu_notifiers(kvm_context_t kvm)
+{
+        int r = 0;
+#ifdef KVM_CAP_SYNC_MMU
+        r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SYNC_MMU);
+#endif
+        return r;
+}
+
 int kvm_init_coalesced_mmio(kvm_context_t kvm)
 {
 	int r = 0;
libkvm/libkvm.h +2 -0
index 9f06fcc..fb51ee1 100644
@@ -528,6 +528,8 @@ int kvm_dirty_pages_log_reset(kvm_context_t kvm);
  */
 int kvm_irqchip_in_kernel(kvm_context_t kvm);
 
+int kvm_has_mmu_notifiers(kvm_context_t kvm);
+
 #ifdef KVM_CAP_IRQCHIP
 /*!
  * \brief Dump in kernel IRQCHIP contents
qemu/Makefile.target +1 -1
index a86464f..cd985f6 100644
@@ -598,7 +598,7 @@ OBJS += rtl8139.o
 OBJS += e1000.o
 
 # virtio devices
-OBJS += virtio.o virtio-net.o virtio-blk.o
+OBJS += virtio.o virtio-net.o virtio-blk.o virtio-balloon.o
 
 OBJS += device-hotplug.o
 
qemu/balloon.h new file +27 -0
new file mode 100644
index 0000000..60b4a5d
@@ -0,0 +1,27 @@
+/*
+ * Balloon
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef _QEMU_BALLOON_H
+#define _QEMU_BALLOON_H
+
+#include "cpu-defs.h"
+
+typedef ram_addr_t (QEMUBalloonEvent)(void *opaque, ram_addr_t target);
+
+void qemu_add_balloon_handler(QEMUBalloonEvent *func, void *opaque);
+
+void qemu_balloon(ram_addr_t target);
+
+ram_addr_t qemu_balloon_status(void);
+
+#endif
qemu/hw/pc.c +3 -0
index 3a8269b..b19e558 100644
@@ -1161,6 +1161,9 @@ static void pc_init1(ram_addr_t ram_size, int vga_ram_size,
 
 	extboot_init(info->bdrv, 1);
     }
+
+    if (pci_enabled)
+        virtio_balloon_init(pci_bus);
 }
 
 static void pc_init_pci(ram_addr_t ram_size, int vga_ram_size,
qemu/hw/pc.h +3 -0
index c284bf1..5b68d69 100644
@@ -164,4 +164,7 @@ void *virtio_blk_init(PCIBus *bus, uint16_t vendor, uint16_t device,
 
 void extboot_init(BlockDriverState *bs, int cmd);
 
+/* virtio-balloon.h */
+void *virtio_balloon_init(PCIBus *bus);
+
 #endif
qemu/hw/virtio-balloon.c new file +194 -0

194 lines not on this page · raw message

qemu/hw/virtio-balloon.h new file +34 -0
new file mode 100644
index 0000000..27d6985
@@ -0,0 +1,34 @@
+/*
+ * Virtio Support
+ *
+ * Copyright IBM, Corp. 2007-2008
+ *
+ * Authors:
+ *  Anthony Liguori   <[email protected]>
+ *  Rusty Russell     <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef _QEMU_VIRTIO_BALLOON_H
+#define _QEMU_VIRTIO_BALLOON_H
+
+/* from Linux's linux/virtio_balloon.h */
+
+/* The ID for virtio_balloon */
+#define VIRTIO_ID_BALLOON	5
+
+/* The feature bitmap for virtio balloon */
+#define VIRTIO_BALLOON_F_MUST_TELL_HOST	0 /* Tell before reclaiming pages */
+
+struct virtio_balloon_config
+{
+    /* Number of pages host wants Guest to give up. */
+    uint32_t num_pages;
+    /* Number of pages we've actually got in balloon. */
+    uint32_t actual;
+};
+
+#endif
qemu/monitor.c +24 -0
index 4acf346..b8c4fde 100644
@@ -35,6 +35,7 @@
 #include "audio/audio.h"
 #include "disas.h"
 #include "migration.h"
+#include "balloon.h"
 #include <dirent.h>
 #include "qemu-timer.h"
 
@@ -1399,6 +1400,25 @@ static void do_inject_nmi(int cpu_index)
 }
 #endif
 
+static void do_balloon(int value)
+{
+    ram_addr_t target = value;
+    qemu_balloon(target << 20);
+}
+
+static void do_info_balloon(void)
+{
+    ram_addr_t actual;
+
+    actual = qemu_balloon_status();
+    if (kvm_enabled() && !qemu_kvm_has_mmu_notifiers())
+        term_printf("Using KVM without MMU-notifiers, ballooning disabled\n");
+    else if (actual == 0)
+	term_printf("Ballooning not activated in VM\n");
+    else
+	term_printf("balloon: actual=%d\n", (int)(actual >> 20));
+}
+
 static term_cmd_t term_cmds[] = {
     { "help|?", "s?", do_help,
       "[cmd]", "show the help" },
@@ -1494,6 +1514,8 @@ static term_cmd_t term_cmds[] = {
     { "pci_add", "iss", device_hot_add, "bus nic|storage [[vlan=n][,macaddr=addr][,model=type]] [file=file][,if=type][,bus=nr]...", "hot-add PCI device" },
     { "pci_del", "ii", device_hot_remove, "bus slot-number", "hot remove PCI device" },
 #endif
+    { "balloon", "i", do_balloon,
+      "target", "request VM to change it's memory allocation (in MB)" },
     { NULL, NULL, },
 };
 
@@ -1558,6 +1580,8 @@ static term_cmd_t info_cmds[] = {
 #endif
     { "migration", "", do_info_migration,
       "", "show migration information" },
+    { "balloon", "", do_info_balloon,
+      "", "show balloon information" },
     { NULL, NULL, },
 };
 
qemu/qemu-kvm.h +2 -0
index 7e28428..088bae3 100644
@@ -110,10 +110,12 @@ extern kvm_context_t kvm_context;
 #define kvm_enabled() (kvm_allowed)
 #define qemu_kvm_irqchip_in_kernel() kvm_irqchip_in_kernel(kvm_context)
 #define qemu_kvm_pit_in_kernel() kvm_pit_in_kernel(kvm_context)
+#define qemu_kvm_has_mmu_notifiers() kvm_has_mmu_notifiers(kvm_context)
 #else
 #define kvm_enabled() (0)
 #define qemu_kvm_irqchip_in_kernel() (0)
 #define qemu_kvm_pit_in_kernel() (0)
+#define qemu_kvm_has_mmu_notifiers() (0)
 #endif
 
 void kvm_mutex_unlock(void);
qemu/vl.c +26 -0
index 2dc1311..ec89921 100644
@@ -38,6 +38,7 @@
 #include "block.h"
 #include "audio/audio.h"
 #include "migration.h"
+#include "balloon.h"
 #include "qemu-kvm.h"
 
 #include <unistd.h>
@@ -530,6 +531,31 @@ void hw_error(const char *fmt, ...)
     va_end(ap);
     abort();
 }
+ 
+/***************/
+/* ballooning */
+
+static QEMUBalloonEvent *qemu_balloon_event;
+void *qemu_balloon_event_opaque;
+
+void qemu_add_balloon_handler(QEMUBalloonEvent *func, void *opaque)
+{
+    qemu_balloon_event = func;
+    qemu_balloon_event_opaque = opaque;
+}
+
+void qemu_balloon(ram_addr_t target)
+{
+    if (qemu_balloon_event)
+	qemu_balloon_event(qemu_balloon_event_opaque, target);
+}
+
+ram_addr_t qemu_balloon_status(void)
+{
+    if (qemu_balloon_event)
+	return qemu_balloon_event(qemu_balloon_event_opaque, 0);
+    return 0;
+}
 
 /***********************************************************/
 /* keyboard/mouse */
#1 lore raw

Hi Anthony,

Quoted 5 lines · Anthony Liguori

Attached is an updated migration patch. It does not appear, however, that madvise() is triggering discarding of pages even with an MMU-notifier enabled kernel. I take it that this is because we're still holding a reference count to the page when it is in the shadow page table? Any ETA on when this will change?

Shadow references should be dropped via kvm_unmap_rmapp -> rmap_remove.

So the symptom is that RSS for the task does not decrease accordingly to the ballooned down size?

#2 lore raw
Quoted 4 lines · Marcelo Tosatti

Hi Anthony,

On Mon, Aug 18, 2008 at 05:04:25PM -0500, Anthony Liguori wrote:

Quoted 8 lines
Quoted 2 lines

Attached is an updated migration patch. It does not appear, however, that madvise() is triggering discarding of pages even with an MMU-notifier enabled kernel. I take it that this is because we're still holding a reference count to the page when it is in the shadow page table? Any ETA on when this will change?

Shadow references should be dropped via kvm_unmap_rmapp -> rmap_remove.

So the symptom is that RSS for the task does not decrease accordingly to the ballooned down size?

Correct. I run a workload in the guest to touch all of memory, watch the RSS size go up to 1G, then balloon down to 512mb. I don't see the RSS size reduce at all though.

This is not the case when just using QEMU. When I get a chance to debug, the next step will be to annotate the code to see whether rmap_remove is actually being invoked on madvise().

Regards,

Anthony Liguori

#3 lore raw
In reply to Anthony Liguori

qemu/hw/virtio-balloon.c line 45

Quoted 3 lines · Anthony Liguori
qemu/hw/virtio-balloon.c new file +3 -0
+    if (!kvm_enabled() || !qemu_kvm_has_mmu_notifiers())
+        madvise(addr, TARGET_PAGE_SIZE,
+                deflate ? MADV_WILLNEED : MADV_DONTNEED);

The check was reversed so madvise wasn't invoked in the first place if mmu notifier was enabled in the host kernel. This should fix it (untested):

       if (!kvm_enabled() || kvm_has_sync_mmu())
#4 lore raw
Quoted 7 lines · Andrea Arcangeli

qemu/hw/virtio-balloon.c line 45

Quoted 4 lines · Anthony Liguori
qemu/hw/virtio-balloon.c new file +3 -0
+    if (!kvm_enabled() || !qemu_kvm_has_mmu_notifiers())
+        madvise(addr, TARGET_PAGE_SIZE,
+                deflate ? MADV_WILLNEED : MADV_DONTNEED);
Quoted 6 lines

The check was reversed so madvise wasn't invoked in the first place if mmu notifier was enabled in the host kernel. This should fix it (untested):

       if (!kvm_enabled() || kvm_has_sync_mmu())

Ugh, sorry about that Andrea.

I've sent out an updated patch set. With MMU notifiers, ballooning is working exactly as expected.

Regards,

Anthony Liguori