xen/balloon: don't online new memory initially

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

---
 drivers/xen/balloon.c     |  3 +++
 drivers/xen/xen-balloon.c | 22 ++++++++++++----------
 include/xen/balloon.h     |  8 ++++++++
 3 files changed, 23 insertions(+), 10 deletions(-)

drivers/xen/balloon.c +3 -0

index 50dcb68d8070..ab609255a0f3 100644
@@ -780,6 +780,9 @@ static int __init balloon_init(void)
 	}
 #endif
 
+	/* Init the xen-balloon driver. */
+	xen_balloon_init();
+
 	return 0;
 }
 subsys_initcall(balloon_init);

drivers/xen/xen-balloon.c +12 -10

index e7715cb62eef..e89136ab851e 100644
@@ -59,6 +59,8 @@ static void watch_target(struct xenbus_watch *watch,
 {
 	unsigned long long new_target;
 	int err;
+	static bool watch_fired;
+	static long target_diff;
 
 	err = xenbus_scanf(XBT_NIL, "memory", "target", "%llu", &new_target);
 	if (err != 1) {

@@ -69,7 +71,14 @@ static void watch_target(struct xenbus_watch *watch,
 	/* The given memory/target value is in KiB, so it needs converting to
 	 * pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
 	 */
-	balloon_set_new_target(new_target >> (PAGE_SHIFT - 10));
+	new_target >>= PAGE_SHIFT - 10;
+	if (watch_fired) {
+		balloon_set_new_target(new_target - target_diff);
+		return;
+	}
+
+	watch_fired = true;
+	target_diff = new_target - balloon_stats.target_pages;
 }
 static struct xenbus_watch target_watch = {
 	.node = "memory/target",

@@ -94,22 +103,15 @@ static struct notifier_block xenstore_notifier = {
 	.notifier_call = balloon_init_watcher,
 };
 
-static int __init balloon_init(void)
+void xen_balloon_init(void)
 {
-	if (!xen_domain())
-		return -ENODEV;
-
-	pr_info("Initialising balloon driver\n");
-
 	register_balloon(&balloon_dev);
 
 	register_xen_selfballooning(&balloon_dev);
 
 	register_xenstore_notifier(&xenstore_notifier);
-
-	return 0;
 }
-subsys_initcall(balloon_init);
+EXPORT_SYMBOL_GPL(xen_balloon_init);
 
 #define BALLOON_SHOW(name, format, args...)				\
 	static ssize_t show_##name(struct device *dev,			\

include/xen/balloon.h +0 -0

index d1767dfb0d95..8906361bb50c 100644
@@ -35,3 +35,11 @@ static inline int register_xen_selfballooning(struct device *dev)
 	return -ENOSYS;

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

---
 drivers/xen/balloon.c     |  3 +++
 drivers/xen/xen-balloon.c | 22 ++++++++++++----------
 include/xen/balloon.h     |  8 ++++++++
 3 files changed, 23 insertions(+), 10 deletions(-)

drivers/xen/balloon.c +3 -0

index 50dcb68d8070..ab609255a0f3 100644
@@ -780,6 +780,9 @@ static int __init balloon_init(void)
 	}
 #endif
 
+	/* Init the xen-balloon driver. */
+	xen_balloon_init();
+
 	return 0;
 }
 subsys_initcall(balloon_init);

drivers/xen/xen-balloon.c +12 -10

index e7715cb62eef..e89136ab851e 100644
@@ -59,6 +59,8 @@ static void watch_target(struct xenbus_watch *watch,
 {
 	unsigned long long new_target;
 	int err;
+	static bool watch_fired;
+	static long target_diff;
 
 	err = xenbus_scanf(XBT_NIL, "memory", "target", "%llu", &new_target);
 	if (err != 1) {

@@ -69,7 +71,14 @@ static void watch_target(struct xenbus_watch *watch,
 	/* The given memory/target value is in KiB, so it needs converting to
 	 * pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
 	 */
-	balloon_set_new_target(new_target >> (PAGE_SHIFT - 10));
+	new_target >>= PAGE_SHIFT - 10;
+	if (watch_fired) {
+		balloon_set_new_target(new_target - target_diff);
+		return;
+	}
+
+	watch_fired = true;
+	target_diff = new_target - balloon_stats.target_pages;
 }
 static struct xenbus_watch target_watch = {
 	.node = "memory/target",

@@ -94,22 +103,15 @@ static struct notifier_block xenstore_notifier = {
 	.notifier_call = balloon_init_watcher,
 };
 
-static int __init balloon_init(void)
+void xen_balloon_init(void)
 {
-	if (!xen_domain())
-		return -ENODEV;
-
-	pr_info("Initialising balloon driver\n");
-
 	register_balloon(&balloon_dev);
 
 	register_xen_selfballooning(&balloon_dev);
 
 	register_xenstore_notifier(&xenstore_notifier);
-
-	return 0;
 }
-subsys_initcall(balloon_init);
+EXPORT_SYMBOL_GPL(xen_balloon_init);
 
 #define BALLOON_SHOW(name, format, args...)				\
 	static ssize_t show_##name(struct device *dev,			\

include/xen/balloon.h +8 -0

index d1767dfb0d95..8906361bb50c 100644
@@ -35,3 +35,11 @@ static inline int register_xen_selfballooning(struct device *dev)
 	return -ENOSYS;
 }
 #endif
+
+#ifdef CONFIG_XEN_BALLOON
+void xen_balloon_init(void);
+#else
+static inline void xen_balloon_init(void)
+{
+}
+#endif

Quoted 8 lines · Juergen Gross

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Quoted 3 lines

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

I don't think I understand how deferring setting the target until next watch firing will help with the original problem. Could you explain?

Thanks.
-boris

Quoted 87 lines

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

---
 drivers/xen/balloon.c     |  3 +++
 drivers/xen/xen-balloon.c | 22 ++++++++++++----------
 include/xen/balloon.h     |  8 ++++++++
 3 files changed, 23 insertions(+), 10 deletions(-)

drivers/xen/balloon.c +3 -0

index 50dcb68d8070..ab609255a0f3 100644
@@ -780,6 +780,9 @@ static int __init balloon_init(void)
 	}
 #endif
 
+	/* Init the xen-balloon driver. */
+	xen_balloon_init();
+
 	return 0;
 }
 subsys_initcall(balloon_init);

drivers/xen/xen-balloon.c +12 -10

index e7715cb62eef..e89136ab851e 100644
@@ -59,6 +59,8 @@ static void watch_target(struct xenbus_watch *watch,
 {
 	unsigned long long new_target;
 	int err;
+	static bool watch_fired;
+	static long target_diff;
 
 	err = xenbus_scanf(XBT_NIL, "memory", "target", "%llu", &new_target);
 	if (err != 1) {

@@ -69,7 +71,14 @@ static void watch_target(struct xenbus_watch *watch,
 	/* The given memory/target value is in KiB, so it needs converting to
 	 * pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
 	 */
-	balloon_set_new_target(new_target >> (PAGE_SHIFT - 10));
+	new_target >>= PAGE_SHIFT - 10;
+	if (watch_fired) {
+		balloon_set_new_target(new_target - target_diff);
+		return;
+	}
+
+	watch_fired = true;
+	target_diff = new_target - balloon_stats.target_pages;
 }
 static struct xenbus_watch target_watch = {
 	.node = "memory/target",

@@ -94,22 +103,15 @@ static struct notifier_block xenstore_notifier = {
 	.notifier_call = balloon_init_watcher,
 };
 
-static int __init balloon_init(void)
+void xen_balloon_init(void)
 {
-	if (!xen_domain())
-		return -ENODEV;
-
-	pr_info("Initialising balloon driver\n");
-
 	register_balloon(&balloon_dev);
 
 	register_xen_selfballooning(&balloon_dev);
 
 	register_xenstore_notifier(&xenstore_notifier);
-
-	return 0;
 }
-subsys_initcall(balloon_init);
+EXPORT_SYMBOL_GPL(xen_balloon_init);
 
 #define BALLOON_SHOW(name, format, args...)				\
 	static ssize_t show_##name(struct device *dev,			\

include/xen/balloon.h +0 -0

index d1767dfb0d95..8906361bb50c 100644
@@ -35,3 +35,11 @@ static inline int register_xen_selfballooning(struct device *dev)
 	return -ENOSYS;
 }
 #endif

include/xen/balloon.h static inline int register_xen_selfballooning(struct device *dev) · line 45

Quoted 8 lines

include/xen/balloon.h +8 -0

+
+#ifdef CONFIG_XEN_BALLOON
+void xen_balloon_init(void);
+#else
+static inline void xen_balloon_init(void)
+{
+}
+#endif

Quoted 8 lines · Juergen Gross

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Quoted 3 lines

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

I don't think I understand how deferring setting the target until next watch firing will help with the original problem. Could you explain?

Thanks.
-boris

Quoted 95 lines

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

---
 drivers/xen/balloon.c     |  3 +++
 drivers/xen/xen-balloon.c | 22 ++++++++++++----------
 include/xen/balloon.h     |  8 ++++++++
 3 files changed, 23 insertions(+), 10 deletions(-)

drivers/xen/balloon.c +3 -0

index 50dcb68d8070..ab609255a0f3 100644
@@ -780,6 +780,9 @@ static int __init balloon_init(void)
 	}
 #endif
 
+	/* Init the xen-balloon driver. */
+	xen_balloon_init();
+
 	return 0;
 }
 subsys_initcall(balloon_init);

drivers/xen/xen-balloon.c +12 -10

index e7715cb62eef..e89136ab851e 100644
@@ -59,6 +59,8 @@ static void watch_target(struct xenbus_watch *watch,
 {
 	unsigned long long new_target;
 	int err;
+	static bool watch_fired;
+	static long target_diff;
 
 	err = xenbus_scanf(XBT_NIL, "memory", "target", "%llu", &new_target);
 	if (err != 1) {

@@ -69,7 +71,14 @@ static void watch_target(struct xenbus_watch *watch,
 	/* The given memory/target value is in KiB, so it needs converting to
 	 * pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
 	 */
-	balloon_set_new_target(new_target >> (PAGE_SHIFT - 10));
+	new_target >>= PAGE_SHIFT - 10;
+	if (watch_fired) {
+		balloon_set_new_target(new_target - target_diff);
+		return;
+	}
+
+	watch_fired = true;
+	target_diff = new_target - balloon_stats.target_pages;
 }
 static struct xenbus_watch target_watch = {
 	.node = "memory/target",

@@ -94,22 +103,15 @@ static struct notifier_block xenstore_notifier = {
 	.notifier_call = balloon_init_watcher,
 };
 
-static int __init balloon_init(void)
+void xen_balloon_init(void)
 {
-	if (!xen_domain())
-		return -ENODEV;
-
-	pr_info("Initialising balloon driver\n");
-
 	register_balloon(&balloon_dev);
 
 	register_xen_selfballooning(&balloon_dev);
 
 	register_xenstore_notifier(&xenstore_notifier);
-
-	return 0;
 }
-subsys_initcall(balloon_init);
+EXPORT_SYMBOL_GPL(xen_balloon_init);
 
 #define BALLOON_SHOW(name, format, args...)				\
 	static ssize_t show_##name(struct device *dev,			\

include/xen/balloon.h +8 -0

index d1767dfb0d95..8906361bb50c 100644
@@ -35,3 +35,11 @@ static inline int register_xen_selfballooning(struct device *dev)
 	return -ENOSYS;
 }
 #endif
+
+#ifdef CONFIG_XEN_BALLOON
+void xen_balloon_init(void);
+#else
+static inline void xen_balloon_init(void)
+{
+}
+#endif

Quoted 9 lines · Boris Ostrovsky

Quoted 8 lines · Juergen Gross

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Quoted 5 lines

Quoted 3 lines

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

I don't think I understand how deferring setting the target until next watch firing will help with the original problem. Could you explain?

The difference of target and current size will be evaluated at the time the watch fires for the first time. This difference is subtracted from the target on later watch events.

Juergen

Quoted 99 lines

Thanks.
-boris

Quoted 94 lines

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

---
 drivers/xen/balloon.c     |  3 +++
 drivers/xen/xen-balloon.c | 22 ++++++++++++----------
 include/xen/balloon.h     |  8 ++++++++
 3 files changed, 23 insertions(+), 10 deletions(-)

drivers/xen/balloon.c +3 -0

index 50dcb68d8070..ab609255a0f3 100644
@@ -780,6 +780,9 @@ static int __init balloon_init(void)
 	}
 #endif
 
+	/* Init the xen-balloon driver. */
+	xen_balloon_init();
+
 	return 0;
 }
 subsys_initcall(balloon_init);

drivers/xen/xen-balloon.c +12 -10

index e7715cb62eef..e89136ab851e 100644
@@ -59,6 +59,8 @@ static void watch_target(struct xenbus_watch *watch,
 {
 	unsigned long long new_target;
 	int err;
+	static bool watch_fired;
+	static long target_diff;
 
 	err = xenbus_scanf(XBT_NIL, "memory", "target", "%llu", &new_target);
 	if (err != 1) {

@@ -69,7 +71,14 @@ static void watch_target(struct xenbus_watch *watch,
 	/* The given memory/target value is in KiB, so it needs converting to
 	 * pages. PAGE_SHIFT converts bytes to pages, hence PAGE_SHIFT - 10.
 	 */
-	balloon_set_new_target(new_target >> (PAGE_SHIFT - 10));
+	new_target >>= PAGE_SHIFT - 10;
+	if (watch_fired) {
+		balloon_set_new_target(new_target - target_diff);
+		return;
+	}
+
+	watch_fired = true;
+	target_diff = new_target - balloon_stats.target_pages;
 }
 static struct xenbus_watch target_watch = {
 	.node = "memory/target",

@@ -94,22 +103,15 @@ static struct notifier_block xenstore_notifier = {
 	.notifier_call = balloon_init_watcher,
 };
 
-static int __init balloon_init(void)
+void xen_balloon_init(void)
 {
-	if (!xen_domain())
-		return -ENODEV;
-
-	pr_info("Initialising balloon driver\n");
-
 	register_balloon(&balloon_dev);
 
 	register_xen_selfballooning(&balloon_dev);
 
 	register_xenstore_notifier(&xenstore_notifier);
-
-	return 0;
 }
-subsys_initcall(balloon_init);
+EXPORT_SYMBOL_GPL(xen_balloon_init);
 
 #define BALLOON_SHOW(name, format, args...)				\
 	static ssize_t show_##name(struct device *dev,			\

include/xen/balloon.h +8 -0

index d1767dfb0d95..8906361bb50c 100644
@@ -35,3 +35,11 @@ static inline int register_xen_selfballooning(struct device *dev)
 	return -ENOSYS;
 }
 #endif
+
+#ifdef CONFIG_XEN_BALLOON
+void xen_balloon_init(void);
+#else
+static inline void xen_balloon_init(void)
+{
+}
+#endif

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

Juergen Gross:

Quoted 8 lines

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Quoted 8 lines

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

This patch seems to introduce a regression. If I boot a HVM or PVH domain with memory != maxmem then the kernel inside the domain reports that it has maxmem available even though Xen reports only what is set as memory. Sooner or later Xen logs "out of PoD memory!" and kills the domain. If I revert the corresponding commit (96edd61d) then everything works as expected.

Tested this with Xen 4.9.0 and Linux 4.13.4.

Juergen Gross:

Quoted 8 lines

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Quoted 8 lines

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

This patch seems to introduce a regression. If I boot a HVM or PVH domain with memory != maxmem then the kernel inside the domain reports that it has maxmem available even though Xen reports only what is set as memory. Sooner or later Xen logs "out of PoD memory!" and kills the domain. If I revert the corresponding commit (96edd61d) then everything works as expected.

Tested this with Xen 4.9.0 and Linux 4.13.4.

Quoted 24 lines · On 10/02/2017 05:37 PM, HW42 wrote:

Juergen Gross:

Quoted 8 lines

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Quoted 8 lines

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

This patch seems to introduce a regression. If I boot a HVM or PVH domain with memory != maxmem then the kernel inside the domain reports that it has maxmem available even though Xen reports only what is set as memory. Sooner or later Xen logs "out of PoD memory!" and kills the domain. If I revert the corresponding commit (96edd61d) then everything works as expected.

Quoted 2 lines

Tested this with Xen 4.9.0 and Linux 4.13.4.

Yes, this indeed doesn't look like it's doing the right thing (although I haven't seen the "out of memory" error).

I wonder whether target_diff should be computed against xenstore's "static-max" and not "target" and the memory should be ballooned down immediately and not on a subsequent watch firing.

Quoted 24 lines · On 10/02/2017 05:37 PM, HW42 wrote:

Juergen Gross:

Quoted 8 lines

When setting up the Xenstore watch for the memory target size the new watch will fire at once. Don't try to reach the configured target size by onlining new memory in this case, as the current memory size will be smaller in almost all cases due to e.g. BIOS reserved pages.

Onlining new memory will lead to more problems e.g. undesired conflicts with NVMe devices meant to be operated as block devices.

Quoted 8 lines

Instead remember the difference between target size and current size when the watch fires for the first time and apply it to any further size changes, too.

In order to avoid races between balloon.c and xen-balloon.c init calls do the xen-balloon.c initialization from balloon.c.

Signed-off-by Juergen Gross <[email protected]>

This patch seems to introduce a regression. If I boot a HVM or PVH domain with memory != maxmem then the kernel inside the domain reports that it has maxmem available even though Xen reports only what is set as memory. Sooner or later Xen logs "out of PoD memory!" and kills the domain. If I revert the corresponding commit (96edd61d) then everything works as expected.

Quoted 2 lines

Tested this with Xen 4.9.0 and Linux 4.13.4.

Yes, this indeed doesn't look like it's doing the right thing (although I haven't seen the "out of memory" error).

I wonder whether target_diff should be computed against xenstore's "static-max" and not "target" and the memory should be ballooned down immediately and not on a subsequent watch firing.