madvise() not triggering page discard with MMU-notifiers
Attached is an updated migration patch. It does not appear, however, that madvise() is triggering discarding of pages even with an MMU-notifier enabled kernel. I take it that this is because we're still holding a reference count to the page when it is in the shadow page table? Any ETA on when this will change?
N.B. the virtio_balloon driver is broken in Linus' tree. Sending patch for that following this one.
Regards,
Anthony Liguori
libkvm/libkvm.c +9 -0
@@ -1053,6 +1053,15 @@ int kvm_pit_in_kernel(kvm_context_t kvm)return kvm->pit_in_kernel; } +int kvm_has_mmu_notifiers (kvm_context_t kvm) +{ +int r =0 ; +#ifdef KVM_CAP_SYNC_MMU + r =ioctl (kvm->fd,KVM_CHECK_EXTENSION ,KVM_CAP_SYNC_MMU ); +#endif +return r; +} +int kvm_init_coalesced_mmio (kvm_context_t kvm) {int r =0 ;
libkvm/libkvm.h +2 -0
@@ -528,6 +528,8 @@ int kvm_dirty_pages_log_reset(kvm_context_t kvm);*/ int kvm_irqchip_in_kernel (kvm_context_t kvm); +int kvm_has_mmu_notifiers (kvm_context_t kvm); +#ifdef KVM_CAP_IRQCHIP /*! * \brief Dump in kernel IRQCHIP contents
qemu/Makefile.target +1 -1
@@ -598,7 +598,7 @@ OBJS += rtl8139.o OBJS += e1000.o # virtio devices -OBJS += virtio.o virtio-net.o virtio-blk.o +OBJS += virtio.o virtio-net.o virtio-blk.o virtio-balloon.o OBJS += device-hotplug.o
qemu/balloon.h new file +27 -0
@@ -0,0 +1,27 @@ +/* +* Balloon +* +* Copyright IBM, Corp. 2008 +* +* Authors: +* Anthony Liguori <[email protected]> +* +* This work is licensed under the terms of the GNU GPL, version 2. See +* the COPYING file in the top-level directory. +* +*/ + +#ifndef _QEMU_BALLOON_H +#define _QEMU_BALLOON_H + +#include "cpu-defs.h" + +typedef ram_addr_t (QEMUBalloonEvent )(void *opaque,ram_addr_t target); + +void qemu_add_balloon_handler (QEMUBalloonEvent *func,void *opaque); + +void qemu_balloon (ram_addr_t target); + +ram_addr_t qemu_balloon_status (void ); + +#endif
qemu/hw/pc.c +3 -0
@@ -1161,6 +1161,9 @@ static void pc_init1(ram_addr_t ram_size, int vga_ram_size,extboot_init (info->bdrv,1 ); } + +if (pci_enabled) +virtio_balloon_init (pci_bus); } static void pc_init_pci(ram_addr_t ram_size, int vga_ram_size,
qemu/hw/pc.h +3 -0
@@ -164,4 +164,7 @@ void *virtio_blk_init(PCIBus *bus, uint16_t vendor, uint16_t device,void extboot_init (BlockDriverState *bs,int cmd); +/* virtio-balloon.h */ +void *virtio_balloon_init (PCIBus *bus); +#endif
qemu/hw/virtio-balloon.c new file +194 -0
194 lines not on this page · raw message
qemu/hw/virtio-balloon.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +* Virtio Support +* +* Copyright IBM, Corp. 2007-2008 +* +* Authors: +* Anthony Liguori <[email protected]> +* Rusty Russell <[email protected]> +* +* This work is licensed under the terms of the GNU GPL, version 2. See +* the COPYING file in the top-level directory. +* +*/ + +#ifndef _QEMU_VIRTIO_BALLOON_H +#define _QEMU_VIRTIO_BALLOON_H + +/* from Linux's linux/virtio_balloon.h */ + +/* The ID for virtio_balloon */ +#define VIRTIO_ID_BALLOON 5 + +/* The feature bitmap for virtio balloon */ +#define VIRTIO_BALLOON_F_MUST_TELL_HOST 0/* Tell before reclaiming pages */ + +struct virtio_balloon_config +{ +/* Number of pages host wants Guest to give up. */ +uint32_t num_pages; +/* Number of pages we've actually got in balloon. */ +uint32_t actual; +}; + +#endif
qemu/monitor.c +24 -0
@@ -35,6 +35,7 @@#include "audio/audio.h" #include "disas.h" #include "migration.h" +#include "balloon.h" #include <dirent.h> #include "qemu-timer.h"
@@ -1399,6 +1400,25 @@ static void do_inject_nmi(int cpu_index) }#endif +static void do_balloon (int value) +{ +ram_addr_t target = value; +qemu_balloon (target <<20 ); +} + +static void do_info_balloon (void ) +{ +ram_addr_t actual; + + actual =qemu_balloon_status (); +if (kvm_enabled () && !qemu_kvm_has_mmu_notifiers ()) +term_printf ("Using KVM without MMU-notifiers, ballooning disabled\n" ); +else if (actual ==0 ) +term_printf ("Ballooning not activated in VM\n" ); +else +term_printf ("balloon: actual=%d\n" , (int )(actual >>20 )); +} +static term_cmd_t term_cmds[] = { {"help|?" ,"s?" , do_help,"[cmd]" ,"show the help" },
@@ -1494,6 +1514,8 @@ static term_cmd_t term_cmds[] = { { "pci_add", "iss", device_hot_add, "bus nic|storage [[vlan=n][,macaddr=addr][,model=type]] [file=file][,if=type][,bus=nr]...", "hot-add PCI device" }, { "pci_del", "ii", device_hot_remove, "bus slot-number", "hot remove PCI device" }, #endif + { "balloon", "i", do_balloon, + "target", "request VM to change it's memory allocation (in MB)" }, { NULL, NULL, }, };
@@ -1558,6 +1580,8 @@ static term_cmd_t info_cmds[] = {#endif {"migration" ,"" , do_info_migration, "", "show migration information" }, + { "balloon", "", do_info_balloon, + "", "show balloon information" }, { NULL, NULL, }, };
qemu/qemu-kvm.h +2 -0
@@ -110,10 +110,12 @@ extern kvm_context_t kvm_context;#define kvm_enabled () (kvm_allowed)#define qemu_kvm_irqchip_in_kernel () kvm_irqchip_in_kernel(kvm_context)#define qemu_kvm_pit_in_kernel () kvm_pit_in_kernel(kvm_context) +#define qemu_kvm_has_mmu_notifiers () kvm_has_mmu_notifiers(kvm_context)#else #define kvm_enabled () (0)#define qemu_kvm_irqchip_in_kernel () (0)#define qemu_kvm_pit_in_kernel () (0) +#define qemu_kvm_has_mmu_notifiers () (0)#endif void kvm_mutex_unlock (void );
qemu/vl.c +26 -0
@@ -38,6 +38,7 @@#include "block.h" #include "audio/audio.h" #include "migration.h" +#include "balloon.h" #include "qemu-kvm.h" #include <unistd.h>
@@ -530,6 +531,31 @@ void hw_error(const char *fmt, ...)va_end (ap);abort (); } + +/***************/ +/* ballooning */ + +static QEMUBalloonEvent *qemu_balloon_event; +void *qemu_balloon_event_opaque; + +void qemu_add_balloon_handler (QEMUBalloonEvent *func,void *opaque) +{ + qemu_balloon_event = func; + qemu_balloon_event_opaque = opaque; +} + +void qemu_balloon (ram_addr_t target) +{ +if (qemu_balloon_event) +qemu_balloon_event (qemu_balloon_event_opaque, target); +} + +ram_addr_t qemu_balloon_status (void ) +{ +if (qemu_balloon_event) +return qemu_balloon_event (qemu_balloon_event_opaque,0 ); +return 0 ; +}/***********************************************************/ /* keyboard/mouse */
Hi Anthony,
Quoted 5 lines · Anthony Liguori
Attached is an updated migration patch. It does not appear, however, that
madvise()is triggering discarding of pages even with an MMU-notifier enabled kernel. I take it that this is because we're still holding a reference count to the page when it is in the shadow page table? Any ETA on when this will change?
Shadow references should be dropped via kvm_unmap_rmapp -> rmap_remove.
So the symptom is that RSS for the task does not decrease accordingly to the ballooned down size?
Quoted 4 lines · Marcelo Tosatti
Hi Anthony,
On Mon, Aug 18, 2008 at 05:04:25PM -0500, Anthony Liguori wrote:
Quoted 8 lines
Quoted 2 lines
Attached is an updated migration patch. It does not appear, however, that
madvise()is triggering discarding of pages even with an MMU-notifier enabled kernel. I take it that this is because we're still holding a reference count to the page when it is in the shadow page table? Any ETA on when this will change?Shadow references should be dropped via
kvm_unmap_rmapp->rmap_remove.So the symptom is that RSS for the task does not decrease accordingly to the ballooned down size?
Correct. I run a workload in the guest to touch all of memory, watch the RSS size go up to 1G, then balloon down to 512mb. I don't see the RSS size reduce at all though.
This is not the case when just using QEMU. When I get a chance to debug, the next step will be to annotate the code to see whether rmap_remove is actually being invoked on madvise().
Regards,
Anthony Liguori
qemu/hw/virtio-balloon.c line 45
Quoted 3 lines · Anthony Liguori
qemu/hw/virtio-balloon.c new file +3
-0+if (!kvm_enabled () || !qemu_kvm_has_mmu_notifiers ()) +madvise (addr,TARGET_PAGE_SIZE , + deflate ?MADV_WILLNEED :MADV_DONTNEED );
The check was reversed so madvise wasn't invoked in the first place if mmu notifier was enabled in the host kernel. This should fix it (untested):
if (!kvm_enabled() || kvm_has_sync_mmu())
Quoted 7 lines · Andrea Arcangeli
qemu/hw/virtio-balloon.c line 45
Quoted 4 lines · Anthony Liguori
qemu/hw/virtio-balloon.c new file +3
-0+if (!kvm_enabled () || !qemu_kvm_has_mmu_notifiers ()) +madvise (addr,TARGET_PAGE_SIZE , + deflate ?MADV_WILLNEED :MADV_DONTNEED );
Quoted 6 lines
The check was reversed so madvise wasn't invoked in the first place if mmu notifier was enabled in the host kernel. This should fix it (untested):
if (!kvm_enabled() || kvm_has_sync_mmu())
Ugh, sorry about that Andrea.
I've sent out an updated patch set. With MMU notifiers, ballooning is working exactly as expected.
Regards,
Anthony Liguori