Virq for low memory condition, V3

From: Andres Lagar-Cavilla [mailto:[email protected]]
Sent: Tuesday, February 28, 2012 2:56 PM
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [Xen-devel] [PATCH 1 of 2] Global virq for low memory situations

 xen/common/page_alloc.c  |  92 ++++++++++++++++++++++++++++++++++++++++++++++++
 xen/include/public/xen.h |   1 +
 2 files changed, 93 insertions(+), 0 deletions(-)

When a low memory threshold on the Xen heap is reached, we fire a global dom0 virq. If someone's listening, they can free up some more memory. The low threshold is configurable via the command line token 'low_mem_virq_limit", and defaults to 64MiB.

We define a new virq VIRQ_ENOMEM. Potential listeners include squeezed, xenballoond, or anything else that can be fired through xencommons.

We error-check the low mem virq against initial available heap (after dom0 allocation), to avoid firing immediately.

Virq issuing is controlled by a hysteresis algorithm: when memory dips below a threshold, the virq is issued and the next virq will fire when memory shrinks another order of magnitude. The virq will not fire again in the current "band" until memory grows over the next higher order of magnitude.

Signed-off-by Andres Lagar-Cavilla <[email protected]>

Quoted 37 lines

From: Andres Lagar-Cavilla [mailto:[email protected]]
Sent: Tuesday, February 28, 2012 2:56 PM
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected];
[email protected]; [email protected]
Subject: [Xen-devel] [PATCH 1 of 2] Global virq for low memory situations

 xen/common/page_alloc.c  |  92
++++++++++++++++++++++++++++++++++++++++++++++++
 xen/include/public/xen.h |   1 +
 2 files changed, 93 insertions(+), 0 deletions(-)

When a low memory threshold on the Xen heap is reached, we fire a global dom0
virq. If someone's listening, they can free up some more memory. The low threshold is configurable via the command line token 'low_mem_virq_limit", and
defaults to 64MiB.

We define a new virq VIRQ_ENOMEM. Potential listeners include squeezed, xenballoond, or anything else that can be fired through xencommons.

We error-check the low mem virq against initial available heap (after dom0
allocation), to avoid firing immediately.

Virq issuing is controlled by a hysteresis algorithm: when memory dips below a
threshold, the virq is issued and the next virq will fire when memory shrinks
another order of magnitude. The virq will not fire again in the current "band"
until memory grows over the next higher order of magnitude.

Quoted 1 line

Signed-off-by Andres Lagar-Cavilla <[email protected]>

Sorry to be late to the party here, but does this take into account any fragmentation, i.e. does it reflect that allocations of order==0 will succeed but order>0 might fail? Should it?

Or maybe Jan's recent work has eliminated all but the corner cases of order>0 allocations? (/me crosses fingers)

Thanks,
Dan

Quoted 1 line

Quoted 1 line · Andres Lagar-Cavilla

xen/common/page_alloc.c +25 -0

@@ -300,6 +301,91 @@ static unsigned long init_node_heap(int 
     return needed;
 }
 
+/* Default to 64 MiB */
+#define DEFAULT_LOW_MEM_VIRQ    (((paddr_t) 64)   << 20)
+#define MAX_LOW_MEM_VIRQ        (((paddr_t) 1024) << 20)
+
+static paddr_t __read_mostly opt_low_mem_virq = DEFAULT_LOW_MEM_VIRQ;
+size_param("low_mem_virq_limit", opt_low_mem_virq);
+
+/* Thresholds to control hysteresis. In pages */
+/* When memory grows above this threshold, reset hysteresis.
+ * -1 initially to not reset until at least one virq issued. */
+static unsigned long low_mem_virq_high      = -1UL;
+/* Threshold at which we issue virq */
+static unsigned long low_mem_virq_th        = 0;
+/* Original threshold after all checks completed */
+static unsigned long low_mem_virq_orig      = 0;
+/* Order for current threshold */
+static unsigned int  low_mem_virq_th_order  = 0;
+
+/* Perform bootstrapping checks and set bounds */
+static void __init setup_low_mem_virq(void)
+{
+    unsigned int order;
+    paddr_t threshold;
+
+    /* Dom0 has already been allocated by now. So check we won't 

xen/common/page_alloc.c +35 -0

+    if ( (1UL << order) > low_mem_virq_th ) 
+        order--;
+
+    /* Set bounds, ready to go */
+    low_mem_virq_th = low_mem_virq_orig = 1UL << order;
+    low_mem_virq_th_order = order;
+
+    printk("Initial low memory virq threshold set at 0x%lx pages.\n",
+            low_mem_virq_th);
+}
+
+static void check_low_mem_virq(void)
+{
+    if ( total_avail_pages <= low_mem_virq_th )
+    {
+        send_global_virq(VIRQ_ENOMEM);
+
+        /* Update thresholds. Next warning will be when we drop below
+         * next order. However, we wait until we grow beyond one
+         * order above us to complain again at the current order */
+        low_mem_virq_high   = 1UL << (low_mem_virq_th_order + 1);
+        if ( low_mem_virq_th_order > 0 )
+            low_mem_virq_th_order--;
+        low_mem_virq_th     = 1UL << low_mem_virq_th_order;
+        return;
+    }
+
+    if ( total_avail_pages >= low_mem_virq_high )
+    {
+        /* Reset hysteresis. Bring threshold up one order.
+         * If we are back where originally set, set high
+         * threshold to -1 to avoid further growth of 
+         * virq threshold. */
+        low_mem_virq_th_order++;
+        low_mem_virq_th = 1UL << low_mem_virq_th_order;

Quoted 1 line

Quoted 1 line · Dan Magenheimer

Or maybe Jan's recent work has eliminated all but the corner cases of order>0 allocations? (/me crosses fingers)

Quoted 1 line

Quoted 1 line · Andres Lagar-Cavilla

TARGETS-$(CONFIG_MIGRATE) += xen-hptool
TARGETS := $(TARGETS-y)

tools/misc/Makefile +0 -1

@@ -21,7 +21,7 @@ INSTALL_BIN-y := xencons
 INSTALL_BIN-$(CONFIG_X86) += xen-detect
 INSTALL_BIN := $(INSTALL_BIN-y)
 
-INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm 

xen-tmem-list-parse gtraceview gtracestat xenlockprof xenwatchdogd xen-ringwatch
+INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof xenwatchdogd xen-ringwatch lowmemd
INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx xen-hvmcrash INSTALL_SBIN-$(CONFIG_MIGRATE) += xen-hptool
INSTALL_SBIN := $(INSTALL_SBIN-y)

tools/misc/Makefile +3 -0

@@ -70,6 +70,9 @@ xen-hptool: xen-hptool.o
 xenwatchdogd: xenwatchdogd.o
 	$(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenctrl) $(APPEND_LDFLAGS)
 
+lowmemd: lowmemd.o
+	$(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenctrl) $(LDLIBS_libxenstore) 
 $(APPEND_LDFLAGS)
+
 gtraceview: gtraceview.o
 	$(CC) $(LDFLAGS) -o $@ $< $(CURSES_LIBS) $(APPEND_LDFLAGS)

diff -r c44e9fe0ecca -r 4300b0d76575 tools/misc/lowmemd.c

tools/misc/lowmemd.c new file +79 -0

@@ -0,0 +1,148 @@
+/*
+ * lowmemd: demo VIRQ_ENOMEM
+ * Andres Lagar-Cavilla (GridCentric Inc.)
+ */
+
+#include <stdio.h>
+#include <xenctrl.h>
+#include <xs.h>
+#include <stdlib.h>
+#include <string.h>
+
+static evtchn_port_t virq_port      = -1;
+static xc_evtchn *xce_handle        = NULL;
+static xc_interface *xch            = NULL;
+static struct xs_handle *xs_handle  = NULL;
+
+void cleanup(void)
+{
+    if (virq_port > -1)
+        xc_evtchn_unbind(xce_handle, virq_port);
+    if (xce_handle)
+        xc_evtchn_close(xce_handle);
+    if (xch)
+        xc_interface_close(xch);
+    if (xs_handle)
+        xs_daemon_close(xs_handle);
+}
+
+/* Never shrink dom0 below 1 GiB */
+#define DOM0_FLOOR  (1 << 30)
+#define DOM0_FLOOR_PG   ((DOM0_FLOOR) >> 12)
+
+/* Act if free memory is less than 92 MiB */
+#define THRESHOLD   (92 << 20)
+#define THRESHOLD_PG    ((THRESHOLD) >> 12)
+
+#define BUFSZ 512
+void handle_low_mem(void)
+{
+    xc_dominfo_t  dom0_info;
+    xc_physinfo_t info;
+    unsigned long long free_pages, dom0_pages, diff, dom0_target;
+    char data[BUFSZ], error[BUFSZ];
+
+    if (xc_physinfo(xch, &info) < 0)
+    {
+        perror("Getting physinfo failed");
+        return;
+    }
+
+    free_pages = (unsigned long long) info.free_pages;
+    printf("Available free pages: 0x%llx:%llux\n",
+            free_pages, free_pages);
+
+    /* Don't do anything if we have more than the threshold free */
+    if ( free_pages >= THRESHOLD_PG )
+        return;
+    diff = THRESHOLD_PG - free_pages; 
+
+    if (xc_domain_getinfo(xch, 0, 1, &dom0_info) < 1)
+    {
+        perror("Failed to get dom0 info");
+        return;
+    }
+
+    dom0_pages = (unsigned long long) dom0_info.nr_pages;
+    printf("Dom0 pages: 0x%llx:%llu\n", dom0_pages, dom0_pages);
+    dom0_target = dom0_pages - diff;
+    if (dom0_target <= DOM0_FLOOR_PG)
+        return;
+
+    printf("Shooting for dom0 target 0x%llx:%llu\n", 
+            dom0_target, dom0_target);
+
+    snprintf(data, BUFSZ, "%llu", dom0_target);
+    if (!xs_write(xs_handle, XBT_NULL, 
+            "/local/domain/0/memory/target", data, strlen(data)))
+    {
+        snprintf(error, BUFSZ,"Failed to write target %s to xenstore", 

data);

tools/misc/lowmemd.c new file +53 -0

+        perror(error);
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    int rc;
+
+    atexit(cleanup);
+
+	xch = xc_interface_open(NULL, NULL, 0);
+	if (xch == NULL)
+    {
+        perror("Failed to open xc interface");
+        return 1;
+    }
+
+	xce_handle = xc_evtchn_open(NULL, 0);
+	if (xce_handle == NULL)
+    {
+        perror("Failed to open evtchn device");
+        return 2;
+    }
+
+    xs_handle = xs_daemon_open();
+    if (xs_handle == NULL)
+    {
+        perror("Failed to open xenstore connection");
+        return 3;
+    }
+
+	if ((rc = xc_evtchn_bind_virq(xce_handle, VIRQ_ENOMEM)) == -1)
+    {
+        perror("Failed to bind to domain exception virq port");
+        return 4;
+    }
+
+    virq_port = rc;
+    
+    while(1)
+    {
+        evtchn_port_t port;
+
+        if ((port = xc_evtchn_pending(xce_handle)) == -1)
+        {
+            perror("Failed to listen for pending event channel");
+            return 5;
+        }
+
+        if (port != virq_port)
+        {
+            char data[BUFSZ];
+            snprintf(data, BUFSZ, "Wrong port, got %d expected %d", port, 

virq_port);

tools/misc/lowmemd.c new file +16 -0

+            perror(data);
+            return 6;
+        }
+
+        if (xc_evtchn_unmask(xce_handle, port) == -1)
+        {
+            perror("Failed to unmask port");
+            return 7;
+        }
+
+        printf("Got a virq kick, time to get work\n");
+        handle_low_mem();
+    }
+
+    return 0;
+}