Skip to contentSkip to conversation

xen: fix regressions regarding memory hotplug in pv domains

PATCH Superseded Juergen Gross 5 participants 17 messages Mar 19, 2015

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Juergen Gross

author maintainer
#0 lore raw

cover xen: fix regressions regarding memory hotplug in pv domains

Fix some regressions introduced in 3.16 and 3.19.

Juergen Gross (2):
xen: prepare p2m list for memory hotplug
xen: before ballooning hotplugged memory, set frames to invalid

 arch/x86/xen/p2m.c    | 13 ++++++++++++-
 drivers/xen/Kconfig   | 13 +++++++++++++
 drivers/xen/balloon.c | 13 +++++++++++--
 3 files changed, 36 insertions(+), 3 deletions(-)
Signature · 1 line
2.1.4
author maintainer
#1 lore raw

PATCH 1/2 xen: prepare p2m list for memory hotplug

Commit 054954eb051f35e74b75a566a96fe756015352c8 ("xen: switch to linear virtual mapped sparse p2m list") introduced a regression regarding to memory hotplug for a pv-domain: as the virtual space for the p2m list is allocated for the to be expected memory size of the domain only, hotplugged memory above that size will not be usable by the domain.

Correct this by using a configurable size for the p2m list in case of memory hotplug enabled (default supported memory size is 512 GB for 64 bit domains and 4 GB for 32 bit domains).

---
 arch/x86/xen/p2m.c  | 13 ++++++++++++-
 drivers/xen/Kconfig | 13 +++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
arch/x86/xen/p2m.c +12 -1
index 9f93af5..30e84ae 100644
@@ -91,6 +91,17 @@ EXPORT_SYMBOL_GPL(xen_p2m_size);
 unsigned long xen_max_p2m_pfn __read_mostly;
 EXPORT_SYMBOL_GPL(xen_max_p2m_pfn);
 
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)
+#endif
+#define P2M_LIMIT max(xen_max_p2m_pfn,					\
+	((unsigned long)((u64)CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT *	\
+	1024 * 1024 * 1024 / PAGE_SIZE)))
+#else
+#define P2M_LIMIT xen_max_p2m_pfn
+#endif
+
 static DEFINE_SPINLOCK(p2m_update_lock);
 
 static unsigned long *p2m_mid_missing_mfn;
@@ -387,7 +398,7 @@ void __init xen_vmalloc_p2m_tree(void)
 	static struct vm_struct vm;
 
 	vm.flags = VM_ALLOC;
-	vm.size = ALIGN(sizeof(unsigned long) * xen_max_p2m_pfn,
+	vm.size = ALIGN(sizeof(unsigned long) * P2M_LIMIT,
 			PMD_SIZE * PMDS_PER_MID_PAGE);
 	vm_area_register_early(&vm, PMD_SIZE * PMDS_PER_MID_PAGE);
 	pr_notice("p2m virtual area at %p, size is %lx\n", vm.addr, vm.size);
drivers/xen/Kconfig +13 -0
index b812462..0a61ddf 100644
@@ -55,6 +55,19 @@ config XEN_BALLOON_MEMORY_HOTPLUG
 
 	  In that case step 3 should be omitted.
 
+config XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+	int
+	default 512 if X86_64
+	default 4 if X86_32
+	depends on XEN_HAVE_PVMMU
+	depends on XEN_BALLOON_MEMORY_HOTPLUG
+	help
+	  Upper limit in GBs a pv domain can be expanded to using memory
+	  hotplug.
+
+	  This value is used to allocate enough space in internal tables needed
+	  for physical memory administration.
+
 config XEN_SCRUB_PAGES
 	bool "Scrub pages before returning them to system"
 	depends on XEN_BALLOON
Signature · 1 line
2.1.4
#2 lore raw

Re: [PATCH 1/2] xen: prepare p2m list for memory hotplug

Quoted 21 lines · Juergen Gross

Commit 054954eb051f35e74b75a566a96fe756015352c8 ("xen: switch to linear virtual mapped sparse p2m list") introduced a regression regarding to memory hotplug for a pv-domain: as the virtual space for the p2m list is allocated for the to be expected memory size of the domain only, hotplugged memory above that size will not be usable by the domain.

Correct this by using a configurable size for the p2m list in case of memory hotplug enabled (default supported memory size is 512 GB for 64 bit domains and 4 GB for 32 bit domains).

---
  arch/x86/xen/p2m.c  | 13 ++++++++++++-
  drivers/xen/Kconfig | 13 +++++++++++++
  2 files changed, 25 insertions(+), 1 deletion(-)
arch/x86/xen/p2m.c +0 -0
index 9f93af5..30e84ae 100644

arch/x86/xen/p2m.c EXPORT_SYMBOL_GPL(xen_p2m_size); · line 96

Quoted 6 lines
arch/x86/xen/p2m.c +3 -0
@@ -91,6 +91,17 @@ EXPORT_SYMBOL_GPL(xen_p2m_size);
  unsigned long xen_max_p2m_pfn __read_mostly;
  EXPORT_SYMBOL_GPL(xen_max_p2m_pfn);
 
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)

Will this work in Kconfig instead?

    range 0 64 if CONFIG_X86_32

-boris

Quoted 44 lines
arch/x86/xen/p2m.c +9 -1
+#endif
+#define P2M_LIMIT max(xen_max_p2m_pfn,					\
+	((unsigned long)((u64)CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT *	\
+	1024 * 1024 * 1024 / PAGE_SIZE)))
+#else
+#define P2M_LIMIT xen_max_p2m_pfn
+#endif
+
  static DEFINE_SPINLOCK(p2m_update_lock);
 
  static unsigned long *p2m_mid_missing_mfn;
@@ -387,7 +398,7 @@ void __init xen_vmalloc_p2m_tree(void)
  	static struct vm_struct vm;
 
  	vm.flags = VM_ALLOC;
-	vm.size = ALIGN(sizeof(unsigned long) * xen_max_p2m_pfn,
+	vm.size = ALIGN(sizeof(unsigned long) * P2M_LIMIT,
  			PMD_SIZE * PMDS_PER_MID_PAGE);
  	vm_area_register_early(&vm, PMD_SIZE * PMDS_PER_MID_PAGE);
  	pr_notice("p2m virtual area at %p, size is %lx\n", vm.addr, vm.size);
drivers/xen/Kconfig +13 -0
index b812462..0a61ddf 100644
@@ -55,6 +55,19 @@ config XEN_BALLOON_MEMORY_HOTPLUG
 
  	  In that case step 3 should be omitted.
 
+config XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+	int
+	default 512 if X86_64
+	default 4 if X86_32
+	depends on XEN_HAVE_PVMMU
+	depends on XEN_BALLOON_MEMORY_HOTPLUG
+	help
+	  Upper limit in GBs a pv domain can be expanded to using memory
+	  hotplug.
+
+	  This value is used to allocate enough space in internal tables needed
+	  for physical memory administration.
+
  config XEN_SCRUB_PAGES
  	bool "Scrub pages before returning them to system"
  	depends on XEN_BALLOON
author maintainer
#3 lore raw

Re: [PATCH 1/2] xen: prepare p2m list for memory hotplug

Quoted 30 lines · Boris Ostrovsky
Quoted 21 lines · Juergen Gross

Commit 054954eb051f35e74b75a566a96fe756015352c8 ("xen: switch to linear virtual mapped sparse p2m list") introduced a regression regarding to memory hotplug for a pv-domain: as the virtual space for the p2m list is allocated for the to be expected memory size of the domain only, hotplugged memory above that size will not be usable by the domain.

Correct this by using a configurable size for the p2m list in case of memory hotplug enabled (default supported memory size is 512 GB for 64 bit domains and 4 GB for 32 bit domains).

---
  arch/x86/xen/p2m.c  | 13 ++++++++++++-
  drivers/xen/Kconfig | 13 +++++++++++++
  2 files changed, 25 insertions(+), 1 deletion(-)
arch/x86/xen/p2m.c +0 -0
index 9f93af5..30e84ae 100644

arch/x86/xen/p2m.c EXPORT_SYMBOL_GPL(xen_p2m_size); · line 95

Quoted 5 lines
arch/x86/xen/p2m.c +3 -0
@@ -91,6 +91,17 @@ EXPORT_SYMBOL_GPL(xen_p2m_size);
  unsigned long xen_max_p2m_pfn __read_mostly;
  EXPORT_SYMBOL_GPL(xen_max_p2m_pfn);
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)

Will this work in Kconfig instead?

Quoted 1 line
     range 0 64 if CONFIG_X86_32

I'll give it a try.

Juergen

Quoted 53 lines

-boris

Quoted 37 lines
arch/x86/xen/p2m.c +9 -1
+#endif
+#define P2M_LIMIT max(xen_max_p2m_pfn,                    \
+    ((unsigned long)((u64)CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT *    \
+    1024 * 1024 * 1024 / PAGE_SIZE)))
+#else
+#define P2M_LIMIT xen_max_p2m_pfn
+#endif
+
  static DEFINE_SPINLOCK(p2m_update_lock);
  static unsigned long *p2m_mid_missing_mfn;
@@ -387,7 +398,7 @@ void __init xen_vmalloc_p2m_tree(void)
      static struct vm_struct vm;
      vm.flags = VM_ALLOC;
-    vm.size = ALIGN(sizeof(unsigned long) * xen_max_p2m_pfn,
+    vm.size = ALIGN(sizeof(unsigned long) * P2M_LIMIT,
              PMD_SIZE * PMDS_PER_MID_PAGE);
      vm_area_register_early(&vm, PMD_SIZE * PMDS_PER_MID_PAGE);
      pr_notice("p2m virtual area at %p, size is %lx\n", vm.addr,
 vm.size);
drivers/xen/Kconfig +11 -0
index b812462..0a61ddf 100644
@@ -55,6 +55,19 @@ config XEN_BALLOON_MEMORY_HOTPLUG
        In that case step 3 should be omitted.
+config XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+    int
+    default 512 if X86_64
+    default 4 if X86_32
+    depends on XEN_HAVE_PVMMU
+    depends on XEN_BALLOON_MEMORY_HOTPLUG
+    help
+      Upper limit in GBs a pv domain can be expanded to using memory
+      hotplug.
+
+      This value is used to allocate enough space in internal tables

needed

drivers/xen/Kconfig config XEN_BALLOON_MEMORY_HOTPLUG · line 73

Quoted 5 lines
drivers/xen/Kconfig +2 -0
+      for physical memory administration.
+
  config XEN_SCRUB_PAGES
      bool "Scrub pages before returning them to system"
      depends on XEN_BALLOON

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

#4 lore raw

Re: [PATCH 1/2] xen: prepare p2m list for memory hotplug

In reply to Juergen Gross
Quoted 39 lines · Juergen Gross

Commit 054954eb051f35e74b75a566a96fe756015352c8 ("xen: switch to linear virtual mapped sparse p2m list") introduced a regression regarding to memory hotplug for a pv-domain: as the virtual space for the p2m list is allocated for the to be expected memory size of the domain only, hotplugged memory above that size will not be usable by the domain.

Correct this by using a configurable size for the p2m list in case of memory hotplug enabled (default supported memory size is 512 GB for 64 bit domains and 4 GB for 32 bit domains).

---
 arch/x86/xen/p2m.c  | 13 ++++++++++++-
 drivers/xen/Kconfig | 13 +++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
arch/x86/xen/p2m.c +11 -0
index 9f93af5..30e84ae 100644
@@ -91,6 +91,17 @@ EXPORT_SYMBOL_GPL(xen_p2m_size);
 unsigned long xen_max_p2m_pfn __read_mostly;
 EXPORT_SYMBOL_GPL(xen_max_p2m_pfn);
 
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)
+#endif
+#define P2M_LIMIT max(xen_max_p2m_pfn,					\
+	((unsigned long)((u64)CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT *	\
+	1024 * 1024 * 1024 / PAGE_SIZE)))
+#else
+#define P2M_LIMIT xen_max_p2m_pfn
+#endif
+
 static DEFINE_SPINLOCK(p2m_update_lock);
 
 static unsigned long *p2m_mid_missing_mfn;

arch/x86/xen/p2m.c void __init xen_vmalloc_p2m_tree(void) · line 402

Quoted 6 lines
arch/x86/xen/p2m.c +1 -1
@@ -387,7 +398,7 @@ void __init xen_vmalloc_p2m_tree(void)
 	static struct vm_struct vm;
 
 	vm.flags = VM_ALLOC;
-	vm.size = ALIGN(sizeof(unsigned long) * xen_max_p2m_pfn,
+	vm.size = ALIGN(sizeof(unsigned long) * P2M_LIMIT,
 			PMD_SIZE * PMDS_PER_MID_PAGE);

What happens when somebody will allocate more memory for guest than XEN_BALLOON_MEMORY_HOTPLUG_LIMIT? Additionally, I think that we do not need allocate extra virtual address space if memory hotplug is disabled.

Quoted 7 lines
arch/x86/xen/p2m.c +0 -0
 	vm_area_register_early(&vm, PMD_SIZE * PMDS_PER_MID_PAGE);
 	pr_notice("p2m virtual area at %p, size is %lx\n", vm.addr, vm.size);
drivers/xen/Kconfig +0 -0
index b812462..0a61ddf 100644

drivers/xen/Kconfig config XEN_BALLOON_MEMORY_HOTPLUG · line 62

Quoted 8 lines
drivers/xen/Kconfig +5 -0
@@ -55,6 +55,19 @@ config XEN_BALLOON_MEMORY_HOTPLUG
 
 	  In that case step 3 should be omitted.
 
+config XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+	int
+	default 512 if X86_64
+	default 4 if X86_32
+	depends on XEN_HAVE_PVMMU

Do we need dependency on XEN_HAVE_PVMMU?
If yes than maybe "depends on XEN_HAVE_PVMMU && XEN_BALLOON_MEMORY_HOTPLUG".

drivers/xen/Kconfig config XEN_BALLOON_MEMORY_HOTPLUG · line 63

Quoted 1 line
drivers/xen/Kconfig +1 -0
+	depends on XEN_BALLOON_MEMORY_HOTPLUG

Should not this option be available even when memory hotplug is disabled?

Daniel

author maintainer
#5 lore raw

Re: [PATCH 1/2] xen: prepare p2m list for memory hotplug

Quoted 42 lines · Daniel Kiper
Quoted 41 lines · Juergen Gross

Commit 054954eb051f35e74b75a566a96fe756015352c8 ("xen: switch to linear virtual mapped sparse p2m list") introduced a regression regarding to memory hotplug for a pv-domain: as the virtual space for the p2m list is allocated for the to be expected memory size of the domain only, hotplugged memory above that size will not be usable by the domain.

Correct this by using a configurable size for the p2m list in case of memory hotplug enabled (default supported memory size is 512 GB for 64 bit domains and 4 GB for 32 bit domains).

---
  arch/x86/xen/p2m.c  | 13 ++++++++++++-
  drivers/xen/Kconfig | 13 +++++++++++++
  2 files changed, 25 insertions(+), 1 deletion(-)
arch/x86/xen/p2m.c +11 -0
index 9f93af5..30e84ae 100644
@@ -91,6 +91,17 @@ EXPORT_SYMBOL_GPL(xen_p2m_size);
  unsigned long xen_max_p2m_pfn __read_mostly;
  EXPORT_SYMBOL_GPL(xen_max_p2m_pfn);
 
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)
+#endif
+#define P2M_LIMIT max(xen_max_p2m_pfn,					\
+	((unsigned long)((u64)CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT *	\
+	1024 * 1024 * 1024 / PAGE_SIZE)))
+#else
+#define P2M_LIMIT xen_max_p2m_pfn
+#endif
+
  static DEFINE_SPINLOCK(p2m_update_lock);
 
  static unsigned long *p2m_mid_missing_mfn;
@@ -387,7 +398,7 @@ void __init xen_vmalloc_p2m_tree(void)
  	static struct vm_struct vm;
 
Quoted 7 lines

arch/x86/xen/p2m.c void __init xen_vmalloc_p2m_tree(void) · line 402

Quoted 4 lines
arch/x86/xen/p2m.c +1 -1
  	vm.flags = VM_ALLOC;
-	vm.size = ALIGN(sizeof(unsigned long) * xen_max_p2m_pfn,
+	vm.size = ALIGN(sizeof(unsigned long) * P2M_LIMIT,
  			PMD_SIZE * PMDS_PER_MID_PAGE);

What happens when somebody will allocate more memory for guest than XEN_BALLOON_MEMORY_HOTPLUG_LIMIT? Additionally, I think that we do

It will fail to add like it does without this patch.

Quoted 2 lines

not need allocate extra virtual address space if memory hotplug is disabled.

As XEN_BALLOON_MEMORY_HOTPLUG_LIMIT depends on XEN_BALLOON_MEMORY_HOTPLUG which in turn depends on MEMORY_HOTPLUG, this is already the case.

Quoted 11 lines
Quoted 11 lines
arch/x86/xen/p2m.c +0 -0
  	vm_area_register_early(&vm, PMD_SIZE * PMDS_PER_MID_PAGE);
  	pr_notice("p2m virtual area at %p, size is %lx\n", vm.addr, vm.size);
drivers/xen/Kconfig +0 -0
index b812462..0a61ddf 100644
@@ -55,6 +55,19 @@ config XEN_BALLOON_MEMORY_HOTPLUG
 
  	  In that case step 3 should be omitted.
 
Quoted 7 lines

drivers/xen/Kconfig config XEN_BALLOON_MEMORY_HOTPLUG · line 62

Quoted 5 lines
drivers/xen/Kconfig +5 -0
+config XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+	int
+	default 512 if X86_64
+	default 4 if X86_32
+	depends on XEN_HAVE_PVMMU

Do we need dependency on XEN_HAVE_PVMMU?

Yes. Needed only for pv domains.

Quoted 1 line

If yes than maybe "depends on XEN_HAVE_PVMMU && XEN_BALLOON_MEMORY_HOTPLUG".

Matter of taste, I guess.

Quoted 4 lines

drivers/xen/Kconfig config XEN_BALLOON_MEMORY_HOTPLUG · line 63

Quoted 2 lines
drivers/xen/Kconfig +1 -0
+	depends on XEN_BALLOON_MEMORY_HOTPLUG

Should not this option be available even when memory hotplug is disabled?

Why?

Juergen

#6 lore raw

Re: [Xen-devel] [PATCH 1/2] xen: prepare p2m list for memory hotplug

In reply to Juergen Gross
Quoted 9 lines · Juergen Gross

Commit 054954eb051f35e74b75a566a96fe756015352c8 ("xen: switch to linear virtual mapped sparse p2m list") introduced a regression regarding to memory hotplug for a pv-domain: as the virtual space for the p2m list is allocated for the to be expected memory size of the domain only, hotplugged memory above that size will not be usable by the domain.

Correct this by using a configurable size for the p2m list in case of memory hotplug enabled (default supported memory size is 512 GB for 64 bit domains and 4 GB for 32 bit domains).

[...]

Quoted 6 lines
arch/x86/xen/p2m.c +0 -0
@@ -91,6 +91,17 @@ EXPORT_SYMBOL_GPL(xen_p2m_size);
 unsigned long xen_max_p2m_pfn __read_mostly;
 EXPORT_SYMBOL_GPL(xen_max_p2m_pfn);
 

arch/x86/xen/p2m.c EXPORT_SYMBOL_GPL(xen_p2m_size); · line 103

Quoted 10 lines
arch/x86/xen/p2m.c +10 -0
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)
+#endif
+#define P2M_LIMIT max(xen_max_p2m_pfn,					\
+	((unsigned long)((u64)CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT *	\
+	1024 * 1024 * 1024 / PAGE_SIZE)))
+#else
+#define P2M_LIMIT xen_max_p2m_pfn
+#endif

Can you arrange the #ifdef's to set xen_max_p2m_pfn to the right value instead of introducing P2M_LIMIT?

David

author maintainer
#7 lore raw

Re: [Xen-devel] [PATCH 1/2] xen: prepare p2m list for memory hotplug

Quoted 17 lines · David Vrabel
Quoted 9 lines · Juergen Gross

Commit 054954eb051f35e74b75a566a96fe756015352c8 ("xen: switch to linear virtual mapped sparse p2m list") introduced a regression regarding to memory hotplug for a pv-domain: as the virtual space for the p2m list is allocated for the to be expected memory size of the domain only, hotplugged memory above that size will not be usable by the domain.

Correct this by using a configurable size for the p2m list in case of memory hotplug enabled (default supported memory size is 512 GB for 64 bit domains and 4 GB for 32 bit domains).

[...]

Quoted 6 lines
arch/x86/xen/p2m.c +0 -0
@@ -91,6 +91,17 @@ EXPORT_SYMBOL_GPL(xen_p2m_size);
  unsigned long xen_max_p2m_pfn __read_mostly;
  EXPORT_SYMBOL_GPL(xen_max_p2m_pfn);
 
Quoted 13 lines

arch/x86/xen/p2m.c EXPORT_SYMBOL_GPL(xen_p2m_size); · line 103

Quoted 10 lines
arch/x86/xen/p2m.c +10 -0
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)
+#endif
+#define P2M_LIMIT max(xen_max_p2m_pfn,					\
+	((unsigned long)((u64)CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT *	\
+	1024 * 1024 * 1024 / PAGE_SIZE)))
+#else
+#define P2M_LIMIT xen_max_p2m_pfn
+#endif

Can you arrange the #ifdef's to set xen_max_p2m_pfn to the right value instead of introducing P2M_LIMIT?

Hmm, this would require additional checks in setup.c. What about:

#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT #define P2M_LIMIT CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT #else
#define P2M_LIMIT 0
#endif

and do the max(...) calculation in xen_vmalloc_p2m_tree()?

Juergen

#8 lore raw

Re: [Xen-devel] [PATCH 1/2] xen: prepare p2m list for memory hotplug

Quoted 35 lines · Juergen Gross
Quoted 17 lines · David Vrabel
Quoted 9 lines · Juergen Gross

Commit 054954eb051f35e74b75a566a96fe756015352c8 ("xen: switch to linear virtual mapped sparse p2m list") introduced a regression regarding to memory hotplug for a pv-domain: as the virtual space for the p2m list is allocated for the to be expected memory size of the domain only, hotplugged memory above that size will not be usable by the domain.

Correct this by using a configurable size for the p2m list in case of memory hotplug enabled (default supported memory size is 512 GB for 64 bit domains and 4 GB for 32 bit domains).

[...]

Quoted 6 lines
arch/x86/xen/p2m.c +0 -0
@@ -91,6 +91,17 @@ EXPORT_SYMBOL_GPL(xen_p2m_size);
  unsigned long xen_max_p2m_pfn __read_mostly;
  EXPORT_SYMBOL_GPL(xen_max_p2m_pfn);
 
Quoted 14 lines

arch/x86/xen/p2m.c EXPORT_SYMBOL_GPL(xen_p2m_size); · line 103

Quoted 11 lines
arch/x86/xen/p2m.c +10 -0
+#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)
+#endif
+#define P2M_LIMIT max(xen_max_p2m_pfn,                    \
+    ((unsigned long)((u64)CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
 *    \
+    1024 * 1024 * 1024 / PAGE_SIZE)))
+#else
+#define P2M_LIMIT xen_max_p2m_pfn
+#endif

Can you arrange the #ifdef's to set xen_max_p2m_pfn to the right value instead of introducing P2M_LIMIT?

Hmm, this would require additional checks in setup.c. What about:

Quoted 7 lines

#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT #define P2M_LIMIT CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT #else
#define P2M_LIMIT 0
#endif

and do the max(...) calculation in xen_vmalloc_p2m_tree()?

Yes, this is fine.

David

#9 lore raw

Re: [PATCH 1/2] xen: prepare p2m list for memory hotplug

In reply to Juergen Gross

arch/x86/xen/p2m.c EXPORT_SYMBOL_GPL(xen_p2m_size); · line 97

Quoted 6 lines · Juergen Gross
arch/x86/xen/p2m.c +3 -0
 
+#ifdef CONFIG_X86_32
+BUILD_BUG_ON_MSG(CONFIG_XEN_BALLOON_MEMORY_HOTPLUG_LIMIT > 64)
+#endif

I assume BUILD_BUG_ON_MSG() aborts the build.

Quoted 6 lines
drivers/xen/Kconfig +3 -0
 
+config XEN_BALLOON_MEMORY_HOTPLUG_LIMIT
+	int
+	default 512 if X86_64

drivers/xen/Kconfig config XEN_BALLOON_MEMORY_HOTPLUG · line 70

Quoted 10 lines
drivers/xen/Kconfig +10 -0
+	default 4 if X86_32
+	depends on XEN_HAVE_PVMMU
+	depends on XEN_BALLOON_MEMORY_HOTPLUG
+	help
+	  Upper limit in GBs a pv domain can be expanded to using memory
+	  hotplug.
+
+	  This value is used to allocate enough space in internal tables needed
+	  for physical memory administration.
+

I think adding a

	range 1 64 if X86_32

would allow to drop the BUILD_BUG_ON_MSG(). (I haven't tested this so you're allowed to bark at me if this ends up wasting your time.)

Paul Bolle

author maintainer
#10 lore raw

PATCH 2/2 xen: before ballooning hotplugged memory, set frames to invalid

In reply to Juergen Gross

Commit 25b884a83d487fd62c3de7ac1ab5549979188482 ("x86/xen: set regions above the end of RAM as 1:1") introduced a regression.

To be able to add memory pages which were added via memory hotplug to a pv domain, the pages must be "invalid" instead of "identity" in the p2m list before they can be added.

---
 drivers/xen/balloon.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
drivers/xen/balloon.c +11 -2
index 0b52d92..52e331f 100644
@@ -221,15 +221,24 @@ static bool balloon_is_inflated(void)
 
 static enum bp_state reserve_additional_memory(long credit)
 {
-	int nid, rc;
+	int nid, rc = 0;
 	u64 hotplug_start_paddr;
 	unsigned long balloon_hotplug = credit;
+	unsigned long pfn;
 
 	hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
 	balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
 	nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
 
-	rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT);
+	for (pfn = PFN_DOWN(hotplug_start_paddr);
+	     !rc && pfn < PFN_DOWN(hotplug_start_paddr) + balloon_hotplug;
+	     pfn++)
+		if (!set_phys_to_machine(pfn, INVALID_P2M_ENTRY))
+			rc = 1;
+
+	if (!rc)
+		rc = add_memory(nid, hotplug_start_paddr,
+				balloon_hotplug << PAGE_SHIFT);
 
 	if (rc) {
 		pr_warn("Cannot add additional memory (%i)\n", rc);
Signature · 1 line
2.1.4
#11 lore raw

Re: [Xen-devel] [PATCH 2/2] xen: before ballooning hotplugged memory, set frames to invalid

Quoted 6 lines · Juergen Gross

Commit 25b884a83d487fd62c3de7ac1ab5549979188482 ("x86/xen: set regions above the end of RAM as 1:1") introduced a regression.

To be able to add memory pages which were added via memory hotplug to a pv domain, the pages must be "invalid" instead of "identity" in the p2m list before they can be added.

[...]

Quoted 12 lines
drivers/xen/balloon.c +2 -1
@@ -221,15 +221,24 @@ static bool balloon_is_inflated(void)
 
 static enum bp_state reserve_additional_memory(long credit)
 {
-	int nid, rc;
+	int nid, rc = 0;
 	u64 hotplug_start_paddr;
 	unsigned long balloon_hotplug = credit;
+	unsigned long pfn;
 

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 237

Quoted 10 lines
drivers/xen/balloon.c +5 -1
 	hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
 	balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
 	nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
 
-	rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT);
+	for (pfn = PFN_DOWN(hotplug_start_paddr);
+	     !rc && pfn < PFN_DOWN(hotplug_start_paddr) + balloon_hotplug;
+	     pfn++)
+		if (!set_phys_to_machine(pfn, INVALID_P2M_ENTRY))
+			rc = 1;
    rc = -ENOMEM;
    break;

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 241

Quoted 4 lines
drivers/xen/balloon.c +4 -0
+
+	if (!rc)
+		rc = add_memory(nid, hotplug_start_paddr,
+				balloon_hotplug << PAGE_SHIFT);

Use else here.

Quoted 3 lines
drivers/xen/balloon.c +0 -0
 	if (rc) {
 		pr_warn("Cannot add additional memory (%i)\n", rc);
 
author maintainer
#12 lore raw

Re: [Xen-devel] [PATCH 2/2] xen: before ballooning hotplugged memory, set frames to invalid

Quoted 24 lines · David Vrabel
Quoted 6 lines · Juergen Gross

Commit 25b884a83d487fd62c3de7ac1ab5549979188482 ("x86/xen: set regions above the end of RAM as 1:1") introduced a regression.

To be able to add memory pages which were added via memory hotplug to a pv domain, the pages must be "invalid" instead of "identity" in the p2m list before they can be added.

[...]

Quoted 16 lines
drivers/xen/balloon.c +2 -1
@@ -221,15 +221,24 @@ static bool balloon_is_inflated(void)
 
  static enum bp_state reserve_additional_memory(long credit)
  {
-	int nid, rc;
+	int nid, rc = 0;
  	u64 hotplug_start_paddr;
  	unsigned long balloon_hotplug = credit;
+	unsigned long pfn;
 
  	hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
  	balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
  	nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
 
Quoted 8 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 237

Quoted 6 lines
drivers/xen/balloon.c +5 -1
-	rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT);
+	for (pfn = PFN_DOWN(hotplug_start_paddr);
+	     !rc && pfn < PFN_DOWN(hotplug_start_paddr) + balloon_hotplug;
+	     pfn++)
+		if (!set_phys_to_machine(pfn, INVALID_P2M_ENTRY))
+			rc = 1;
     rc = -ENOMEM;

I used the value 1 on purpose to be able to identify the case by the value printed in the warning below.

Quoted 1 line
     break;

Why? !rc is already tested in the for() clause.

Quoted 7 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 241

Quoted 5 lines
drivers/xen/balloon.c +4 -0
+
+	if (!rc)
+		rc = add_memory(nid, hotplug_start_paddr,
+				balloon_hotplug << PAGE_SHIFT);

Use else here.

Huh? I want the message to be printed if either set_phys_to_machine() or add_memory() failed.

Quoted 4 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void)

Quoted 4 lines
drivers/xen/balloon.c +0 -0
  	if (rc) {
  		pr_warn("Cannot add additional memory (%i)\n", rc);
 

Juergen

#13 lore raw

Re: [Xen-devel] [PATCH 2/2] xen: before ballooning hotplugged memory, set frames to invalid

Quoted 35 lines · Juergen Gross
Quoted 24 lines · David Vrabel
Quoted 6 lines · Juergen Gross

Commit 25b884a83d487fd62c3de7ac1ab5549979188482 ("x86/xen: set regions above the end of RAM as 1:1") introduced a regression.

To be able to add memory pages which were added via memory hotplug to a pv domain, the pages must be "invalid" instead of "identity" in the p2m list before they can be added.

[...]

Quoted 16 lines
drivers/xen/balloon.c +2 -1
@@ -221,15 +221,24 @@ static bool balloon_is_inflated(void)
 
  static enum bp_state reserve_additional_memory(long credit)
  {
-    int nid, rc;
+    int nid, rc = 0;
      u64 hotplug_start_paddr;
      unsigned long balloon_hotplug = credit;
+    unsigned long pfn;
 
      hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
      balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
      nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
 
Quoted 9 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 237

Quoted 7 lines
drivers/xen/balloon.c +5 -1
-    rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug <<
 PAGE_SHIFT);
+    for (pfn = PFN_DOWN(hotplug_start_paddr);
+         !rc && pfn < PFN_DOWN(hotplug_start_paddr) + balloon_hotplug;
+         pfn++)
+        if (!set_phys_to_machine(pfn, INVALID_P2M_ENTRY))
+            rc = 1;
     rc = -ENOMEM;
Quoted 2 lines

I used the value 1 on purpose to be able to identify the case by the value printed in the warning below.

Ok.

Quoted 4 lines
Quoted 2 lines
     break;

Why? !rc is already tested in the for() clause.

I prefer an explicit break.

Quoted 9 lines
Quoted 6 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 241

Quoted 4 lines
drivers/xen/balloon.c +4 -0
+
+    if (!rc)
+        rc = add_memory(nid, hotplug_start_paddr,
+                balloon_hotplug << PAGE_SHIFT);

Use else here.

Huh? I want the message to be printed if either set_phys_to_machine() or add_memory() failed.

Ok.

David

#14 lore raw

Re: [PATCH 2/2] xen: before ballooning hotplugged memory, set frames to invalid

In reply to Juergen Gross
Quoted 26 lines · Juergen Gross

Commit 25b884a83d487fd62c3de7ac1ab5549979188482 ("x86/xen: set regions above the end of RAM as 1:1") introduced a regression.

To be able to add memory pages which were added via memory hotplug to a pv domain, the pages must be "invalid" instead of "identity" in the p2m list before they can be added.

---
 drivers/xen/balloon.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
drivers/xen/balloon.c +2 -1
index 0b52d92..52e331f 100644
@@ -221,15 +221,24 @@ static bool balloon_is_inflated(void)
 
 static enum bp_state reserve_additional_memory(long credit)
 {
-	int nid, rc;
+	int nid, rc = 0;
 	u64 hotplug_start_paddr;
 	unsigned long balloon_hotplug = credit;
+	unsigned long pfn;

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 236

Quoted 10 lines
drivers/xen/balloon.c +4 -1
 
 	hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
 	balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
 	nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
 
-	rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT);
+	for (pfn = PFN_DOWN(hotplug_start_paddr);
+	     !rc && pfn < PFN_DOWN(hotplug_start_paddr) + balloon_hotplug;
+	     pfn++)
+		if (!set_phys_to_machine(pfn, INVALID_P2M_ENTRY))

rc = set_phys_to_machine(pfn, INVALID_P2M_ENTRY)?

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 237

Quoted 1 line
drivers/xen/balloon.c +1 -0
+			rc = 1;

I do not think that this stuff is needed for HVM or PVH guests.

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 244

Quoted 6 lines
drivers/xen/balloon.c +3 -0
+	if (!rc)
+		rc = add_memory(nid, hotplug_start_paddr,
+				balloon_hotplug << PAGE_SHIFT);
 
 	if (rc) {
 		pr_warn("Cannot add additional memory (%i)\n", rc);

It will be nice to know what part of infrastructure failed. Could you create separate pr_warn() message for set_phys_to_machine()?

Daniel

author maintainer
#15 lore raw

Re: [PATCH 2/2] xen: before ballooning hotplugged memory, set frames to invalid

Quoted 32 lines · Daniel Kiper
Quoted 31 lines · Juergen Gross

Commit 25b884a83d487fd62c3de7ac1ab5549979188482 ("x86/xen: set regions above the end of RAM as 1:1") introduced a regression.

To be able to add memory pages which were added via memory hotplug to a pv domain, the pages must be "invalid" instead of "identity" in the p2m list before they can be added.

---
  drivers/xen/balloon.c | 13 +++++++++++--
  1 file changed, 11 insertions(+), 2 deletions(-)
drivers/xen/balloon.c +2 -1
index 0b52d92..52e331f 100644
@@ -221,15 +221,24 @@ static bool balloon_is_inflated(void)
 
  static enum bp_state reserve_additional_memory(long credit)
  {
-	int nid, rc;
+	int nid, rc = 0;
  	u64 hotplug_start_paddr;
  	unsigned long balloon_hotplug = credit;
+	unsigned long pfn;
 
  	hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
  	balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
  	nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
 
Quoted 7 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 236

Quoted 5 lines
drivers/xen/balloon.c +4 -1
-	rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT);
+	for (pfn = PFN_DOWN(hotplug_start_paddr);
+	     !rc && pfn < PFN_DOWN(hotplug_start_paddr) + balloon_hotplug;
+	     pfn++)
+		if (!set_phys_to_machine(pfn, INVALID_P2M_ENTRY))

rc = set_phys_to_machine(pfn, INVALID_P2M_ENTRY)?

Not really. set_phys_to_machine returns false on failure...

Quoted 4 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 237

Quoted 2 lines
drivers/xen/balloon.c +1 -0
+			rc = 1;

I do not think that this stuff is needed for HVM or PVH guests.

True.

Quoted 10 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 244

Quoted 7 lines
drivers/xen/balloon.c +3 -0
+	if (!rc)
+		rc = add_memory(nid, hotplug_start_paddr,
+				balloon_hotplug << PAGE_SHIFT);
 
  	if (rc) {
  		pr_warn("Cannot add additional memory (%i)\n", rc);

It will be nice to know what part of infrastructure failed. Could you create separate pr_warn() message for set_phys_to_machine()?

Value 1 for rc is the indicator for that case.

Juergen

#16 lore raw

Re: [PATCH 2/2] xen: before ballooning hotplugged memory, set frames to invalid

Quoted 55 lines · Juergen Gross
Quoted 32 lines · Daniel Kiper
Quoted 31 lines · Juergen Gross

Commit 25b884a83d487fd62c3de7ac1ab5549979188482 ("x86/xen: set regions above the end of RAM as 1:1") introduced a regression.

To be able to add memory pages which were added via memory hotplug to a pv domain, the pages must be "invalid" instead of "identity" in the p2m list before they can be added.

---
 drivers/xen/balloon.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
drivers/xen/balloon.c +2 -1
index 0b52d92..52e331f 100644
@@ -221,15 +221,24 @@ static bool balloon_is_inflated(void)
 
 static enum bp_state reserve_additional_memory(long credit)
 {
-	int nid, rc;
+	int nid, rc = 0;
 	u64 hotplug_start_paddr;
 	unsigned long balloon_hotplug = credit;
+	unsigned long pfn;
 
 	hotplug_start_paddr = PFN_PHYS(SECTION_ALIGN_UP(max_pfn));
 	balloon_hotplug = round_up(balloon_hotplug, PAGES_PER_SECTION);
 	nid = memory_add_physaddr_to_nid(hotplug_start_paddr);
 
Quoted 7 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 236

Quoted 5 lines
drivers/xen/balloon.c +4 -1
-	rc = add_memory(nid, hotplug_start_paddr, balloon_hotplug << PAGE_SHIFT);
+	for (pfn = PFN_DOWN(hotplug_start_paddr);
+	     !rc && pfn < PFN_DOWN(hotplug_start_paddr) + balloon_hotplug;
+	     pfn++)
+		if (!set_phys_to_machine(pfn, INVALID_P2M_ENTRY))

rc = set_phys_to_machine(pfn, INVALID_P2M_ENTRY)?

Not really. set_phys_to_machine returns false on failure...

Quoted 4 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 237

Quoted 2 lines
drivers/xen/balloon.c +1 -0
+			rc = 1;

I do not think that this stuff is needed for HVM or PVH guests.

True.

Quoted 5 lines
Quoted 5 lines
drivers/xen/balloon.c +3 -0
+	if (!rc)
+		rc = add_memory(nid, hotplug_start_paddr,
+				balloon_hotplug << PAGE_SHIFT);
 
Quoted 7 lines
Quoted 5 lines

drivers/xen/balloon.c static bool balloon_is_inflated(void) · line 244

Quoted 2 lines
drivers/xen/balloon.c +0 -0
 	if (rc) {
 		pr_warn("Cannot add additional memory (%i)\n", rc);

It will be nice to know what part of infrastructure failed. Could you create separate pr_warn() message for set_phys_to_machine()?

Value 1 for rc is the indicator for that case.

Well... Personally I prefer explicit messages telling what happened than something which requires digging in a code to understand a problem. Additionally, I think that we should use negative numbers (as David suggested) to signal an error. Most of kernel stuff work in that way.

Daniel