Skip to contentSkip to conversation

xen/mm: limit in-place scrubbing

PATCH In review Roger Pau Monne 2 participants 10 messages Jan 30 – Feb 5 RISC-V

lore
Share & tools
lore
#0 lore raw

v5 cover xen/mm: limit in-place scrubbing

Hello,

In XenServer we have seen the watchdog occasionally triggering during domain creation if 1GB pages are scrubbed in-place during physmap population. The following series attempt to mitigate this by adding preemption to page scrubbing in populate_physmap(). Also a new limit and command line option to signal the maximum allocation order when doing in-place scrubbing. This is set by default to CONFIG_PTDOM_MAX_ORDER.

Thanks, Roger.

Roger Pau Monne (4):
xen/mm: remove aliasing of PGC_need_scrub over PGC_allocated xen/mm: enforce SCRUB_DEBUG checks for MEMF_no_scrub allocations xen/mm: allow deferred scrub of physmap populate allocated pages xen/mm: limit non-scrubbed allocations to a specific order

 docs/misc/xen-command-line.pandoc |  13 ++++
 xen/arch/arm/include/asm/mm.h     |  10 +--
 xen/arch/ppc/include/asm/mm.h     |  10 +--
 xen/arch/riscv/include/asm/mm.h   |  10 +--
 xen/arch/x86/include/asm/mm.h     |  18 ++---
 xen/common/domain.c               |  24 +++++++
 xen/common/memory.c               | 108 ++++++++++++++++++++++++++++--
 xen/common/page_alloc.c           |  32 +++++++--
 xen/include/xen/mm.h              |  14 ++++
 xen/include/xen/sched.h           |   5 ++
 10 files changed, 202 insertions(+), 42 deletions(-)
Signature · 1 line
2.51.0
#1 lore raw

PATCH v5 1/4 xen/mm: remove aliasing of PGC_need_scrub over PGC_allocated

Future changes will care about the state of the PGC_need_scrub flag even when pages have the PGC_allocated set, and hence it's no longer possible to alias both values. Also introduce PGC_need_scrub to the set of preserved flags, so it's not dropped by assign_pages().

No functional change intended, albeit the page counter on x86 looses a bit.

---
Changes since v4:
- New in this version.
---
The PGC space on arches different than x86 is not compact, so I've just used a hole in those.

---
 xen/arch/arm/include/asm/mm.h   | 10 +++-------
 xen/arch/ppc/include/asm/mm.h   | 10 +++-------
 xen/arch/riscv/include/asm/mm.h | 10 +++-------
 xen/arch/x86/include/asm/mm.h   | 18 +++++++-----------
 xen/common/page_alloc.c         |  2 +-
 5 files changed, 17 insertions(+), 33 deletions(-)
xen/arch/arm/include/asm/mm.h +3 -7
index ec2d2dc5372a..72a692862420 100644
@@ -144,6 +144,9 @@ struct page_info
 #define _PGC_colored      PG_shift(4)
 #define PGC_colored       PG_mask(1, 4)
 #endif
+/* Page needs to be scrubbed. */
+#define _PGC_need_scrub   PG_shift(5)
+#define PGC_need_scrub    PG_mask(1, 5)
 /* ... */
 /* Page is broken? */
 #define _PGC_broken       PG_shift(7)
@@ -163,13 +166,6 @@ struct page_info
 #define PGC_count_width   PG_shift(10)
 #define PGC_count_mask    ((1UL<<PGC_count_width)-1)
 
-/*
- * Page needs to be scrubbed. Since this bit can only be set on a page that is
- * free (i.e. in PGC_state_free) we can reuse PGC_allocated bit.
- */
-#define _PGC_need_scrub   _PGC_allocated
-#define PGC_need_scrub    PGC_allocated
-
 #if defined(CONFIG_ARM_64) || defined(CONFIG_MPU)
 #define is_xen_heap_page(page) ((page)->count_info & PGC_xen_heap)
 #define is_xen_heap_mfn(mfn) \
xen/arch/ppc/include/asm/mm.h +3 -7
index 91c405876bd0..402d06bdaa9f 100644
@@ -57,6 +57,9 @@ static inline struct page_info *virt_to_page(const void *v)
 /* Page is Xen heap? */
 #define _PGC_xen_heap     PG_shift(2)
 #define PGC_xen_heap      PG_mask(1, 2)
+/* Page needs to be scrubbed. */
+#define _PGC_need_scrub   PG_shift(3)
+#define PGC_need_scrub    PG_mask(1, 3)
 /* Page is broken? */
 #define _PGC_broken       PG_shift(7)
 #define PGC_broken        PG_mask(1, 7)
@@ -75,13 +78,6 @@ static inline struct page_info *virt_to_page(const void *v)
 #define PGC_count_width   PG_shift(10)
 #define PGC_count_mask    ((1UL<<PGC_count_width)-1)
 
-/*
- * Page needs to be scrubbed. Since this bit can only be set on a page that is
- * free (i.e. in PGC_state_free) we can reuse PGC_allocated bit.
- */
-#define _PGC_need_scrub   _PGC_allocated
-#define PGC_need_scrub    PGC_allocated
-
 #define is_xen_heap_page(page) ((page)->count_info & PGC_xen_heap)
 #define is_xen_heap_mfn(mfn) \
     (mfn_valid(mfn) && is_xen_heap_page(mfn_to_page(mfn)))
xen/arch/riscv/include/asm/mm.h +3 -7
index a005d0247a6f..9e28c2495462 100644
@@ -273,13 +273,6 @@ static inline bool arch_mfns_in_directmap(unsigned long mfn, unsigned long nr)
 #define PGT_count_width   PG_shift(2)
 #define PGT_count_mask    ((1UL << PGT_count_width) - 1)
 
-/*
- * Page needs to be scrubbed. Since this bit can only be set on a page that is
- * free (i.e. in PGC_state_free) we can reuse PGC_allocated bit.
- */
-#define _PGC_need_scrub   _PGC_allocated
-#define PGC_need_scrub    PGC_allocated
-
 /* Cleared when the owning guest 'frees' this page. */
 #define _PGC_allocated    PG_shift(1)
 #define PGC_allocated     PG_mask(1, 1)
@@ -293,6 +286,9 @@ static inline bool arch_mfns_in_directmap(unsigned long mfn, unsigned long nr)
 #else
 #define PGC_static     0
 #endif
+/* Page needs to be scrubbed. */
+#define _PGC_need_scrub   PG_shift(4)
+#define PGC_need_scrub    PG_mask(1, 4)
 /* Page is broken? */
 #define _PGC_broken       PG_shift(7)
 #define PGC_broken        PG_mask(1, 7)
xen/arch/x86/include/asm/mm.h +7 -11
index 419fa17a4373..06c20ab8de33 100644
@@ -83,29 +83,25 @@
 #define PGC_state_offlined  PG_mask(2, 6)
 #define PGC_state_free      PG_mask(3, 6)
 #define page_state_is(pg, st) (((pg)->count_info&PGC_state) == PGC_state_##st)
+/* Page needs to be scrubbed. */
+#define _PGC_need_scrub   PG_shift(7)
+#define PGC_need_scrub    PG_mask(1, 7)
 #ifdef CONFIG_SHADOW_PAGING
  /* Set when a page table page has been shadowed. */
-#define _PGC_shadowed_pt  PG_shift(7)
-#define PGC_shadowed_pt   PG_mask(1, 7)
+#define _PGC_shadowed_pt  PG_shift(8)
+#define PGC_shadowed_pt   PG_mask(1, 8)
 #else
 #define PGC_shadowed_pt   0
 #endif
 
 /* Count of references to this frame. */
 #if PGC_shadowed_pt
-#define PGC_count_width   PG_shift(7)
+#define PGC_count_width   PG_shift(8)
 #else
-#define PGC_count_width   PG_shift(6)
+#define PGC_count_width   PG_shift(7)
 #endif
 #define PGC_count_mask    ((1UL<<PGC_count_width)-1)
 
-/*
- * Page needs to be scrubbed. Since this bit can only be set on a page that is
- * free (i.e. in PGC_state_free) we can reuse PGC_allocated bit.
- */
-#define _PGC_need_scrub   _PGC_allocated
-#define PGC_need_scrub    PGC_allocated
-
 #ifndef CONFIG_BIGMEM
 /*
  * This definition is solely for the use in struct page_info (and
xen/common/page_alloc.c +1 -1
index 2efc11ce095f..2ee249ac365a 100644
@@ -170,7 +170,7 @@
 /*
  * Flags that are preserved in assign_pages() (and only there)
  */
-#define PGC_preserved (PGC_extra | PGC_static | PGC_colored)
+#define PGC_preserved (PGC_extra | PGC_static | PGC_colored | PGC_need_scrub)
 
 #ifndef PGT_TYPE_INFO_INITIALIZER
 #define PGT_TYPE_INFO_INITIALIZER 0
Signature · 1 line
2.51.0
Reviewed-by on v5 1/4
#2 lore raw

Re: [PATCH v5 1/4] xen/mm: remove aliasing of PGC_need_scrub over PGC_allocated

Quoted 9 lines · Roger Pau Monne

Future changes will care about the state of the PGC_need_scrub flag even when pages have the PGC_allocated set, and hence it's no longer possible to alias both values. Also introduce PGC_need_scrub to the set of preserved flags, so it's not dropped by assign_pages().

No functional change intended, albeit the page counter on x86 looses a bit.

#3 lore raw

PATCH v5 2/4 xen/mm: enforce SCRUB_DEBUG checks for MEMF_no_scrub allocations

In reply to Roger Pau Monne

The logic in alloc_heap_pages() only checks for scrubbing pattern correctness when the caller doesn't pass MEMF_no_scrub in memflags. However already scrubbed pages can be checked for correctness, regardless of the caller having requested MEMF_no_scrub.

Relax the checking around the check_one_page() call, to allow for calls with MEMF_no_scrub to also check the correctness of pages marked as already scrubbed when allocated. This widens the checking of scrubbing correctness, so it would also check the scrubbing correctness of MEMF_no_scrub allocations.

---
After discussing with Jan I've deliberately omitted the tag:

Fixes 0c5f2f9cefac ("mm: Make sure pages are scrubbed")

The intended approach might have been to ensure the caller of alloc_heap_pages() gets properly scrubbed pages, rather than asserting the internal state of free pages is as expected.

---
 xen/common/page_alloc.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
xen/common/page_alloc.c +2 -3
index 2ee249ac365a..dcb95309b12a 100644
@@ -1105,8 +1105,7 @@ static struct page_info *alloc_heap_pages(
 
     spin_unlock(&heap_lock);
 
-    if ( first_dirty != INVALID_DIRTY_IDX ||
-         (scrub_debug && !(memflags & MEMF_no_scrub)) )
+    if ( first_dirty != INVALID_DIRTY_IDX || scrub_debug )
     {
         bool cold = d && d != current->domain;
 
@@ -1119,7 +1118,7 @@ static struct page_info *alloc_heap_pages(
 
                 dirty_cnt++;
             }
-            else if ( !(memflags & MEMF_no_scrub) )
+            else
                 check_one_page(&pg[i]);
         }
 
Signature · 1 line
2.51.0
#4 lore raw

PATCH v5 3/4 xen/mm: allow deferred scrub of physmap populate allocated pages

In reply to Roger Pau Monne

Physmap population has the need to use pages as big as possible to reduce p2m shattering. However that triggers issues when big enough pages are not yet scrubbed, and so scrubbing must be done at allocation time. On some scenarios with added contention the watchdog can trigger:

Watchdog timer detects that CPU55 is stuck! ----[ Xen-4.17.5-21 x86_64 debug=n Not tainted ]----
CPU: 55

RIP:    e008:[<ffff82d040204c4a>] clear_page_sse2+0x1a/0x30

RFLAGS: 0000000000000202 CONTEXT: hypervisor (d0v12)
[...]
Xen call trace:
[<ffff82d040204c4a>] R clear_page_sse2+0x1a/0x30 [<ffff82d04022a121>] S clear_domain_page+0x11/0x20 [<ffff82d04022c170>] S common/page_alloc.c#alloc_heap_pages+0x400/0x5a0 [<ffff82d04022d4a7>] S alloc_domheap_pages+0x67/0x180 [<ffff82d040226f9f>] S common/memory.c#populate_physmap+0x22f/0x3b0 [<ffff82d040228ec8>] S do_memory_op+0x728/0x1970

Introduce a mechanism to preempt page scrubbing in populate_physmap(). It relies on stashing the dirty page in the domain struct temporarily to preempt to guest context, so the scrubbing can resume when the domain re-enters the hypercall. The added deferral mechanism will only be used for domain construction, and is designed to be used with a single threaded domain builder. If the toolstack makes concurrent calls to XENMEM_populate_physmap for the same target domain it will trash stashed pages, resulting in slow domain physmap population.

Note a similar issue is present in increase reservation. However that hypercall is likely to only be used once the domain is already running and the known implementations use 4K pages. It will be deal with in a separate patch using a different approach, that will also take care of the allocation in populate_physmap() once the domain is running.

Fixes 74d2e11ccfd2 ("mm: Scrub pages in alloc_heap_pages() if needed")

---
Changes since v4:
- Introduce d->is_dying check in stash_allocation().
- Move call to domain_pending_scrub_free() to domain_teardown().

Changes since v3:
- Introduce helper to free stashed pages.
- Attempt to free stashed pages from domain_unpause_by_systemcontroller() also.
- Free stashed page in get_stashed_allocation() if it doesn't match the requested parameters.

Changes since v2:
- Introduce FREE_DOMHEAP_PAGE{,S}().
- Remove j local counter.
- Free page pending scrub in domain_kill() also.
- Remove BUG_ON().
- Reorder get_stashed_allocation() flow.
- s/dirty/unscrubbed/ in a printk message.

Changes since v1:
- New in this version, different approach than v1.

---
 xen/common/domain.c     |  23 +++++++++
 xen/common/memory.c     | 105 +++++++++++++++++++++++++++++++++++++++-
 xen/common/page_alloc.c |   2 +-
 xen/include/xen/mm.h    |  10 ++++
 xen/include/xen/sched.h |   5 ++
 5 files changed, 143 insertions(+), 2 deletions(-)
xen/common/domain.c +23 -0
index 376351b528c9..163f7fc96658 100644
@@ -615,6 +615,18 @@ static int __init cf_check parse_dom0_param(const char *s)
 }
 custom_param("dom0", parse_dom0_param);
 
+static void domain_pending_scrub_free(struct domain *d)
+{
+    rspin_lock(&d->page_alloc_lock);
+    if ( d->pending_scrub )
+    {
+        FREE_DOMHEAP_PAGES(d->pending_scrub, d->pending_scrub_order);
+        d->pending_scrub_order = 0;
+        d->pending_scrub_index = 0;
+    }
+    rspin_unlock(&d->page_alloc_lock);
+}
+
 /*
  * Release resources held by a domain.  There may or may not be live
  * references to the domain, and it may or may not be fully constructed.
@@ -676,6 +688,7 @@ static int domain_teardown(struct domain *d)
 
         /* Trivial teardown, not long-running enough to need a preemption check. */
         domain_llc_coloring_free(d);
+        domain_pending_scrub_free(d);
 
     PROGRESS(gnttab_mappings):
         rc = gnttab_release_mappings(d);
@@ -719,6 +732,7 @@ static void _domain_destroy(struct domain *d)
 {
     BUG_ON(!d->is_dying);
     BUG_ON(atomic_read(&d->refcnt) != DOMAIN_DESTROYED);
+    ASSERT(!d->pending_scrub);
 
     XVFREE(d->console);
 
@@ -1678,6 +1692,15 @@ int domain_unpause_by_systemcontroller(struct domain *d)
      */
     if ( new == 0 && !d->creation_finished )
     {
+        if ( d->pending_scrub )
+        {
+            printk(XENLOG_ERR
+                   "%pd: cannot be started with pending unscrubbed pages, destroying\n",
+                   d);
+            domain_crash(d);
+            domain_pending_scrub_free(d);
+            return -EBUSY;
+        }
         d->creation_finished = true;
         arch_domain_creation_finished(d);
     }
xen/common/memory.c +104 -1

123 lines not on this page · raw message

xen/common/page_alloc.c +1 -1
index dcb95309b12a..87ebf5a024dc 100644
@@ -792,7 +792,7 @@ static void page_list_add_scrub(struct page_info *pg, unsigned int node,
 # define scrub_page_cold clear_page_cold
 #endif
 
-static void scrub_one_page(const struct page_info *pg, bool cold)
+void scrub_one_page(const struct page_info *pg, bool cold)
 {
     void *ptr;
 
xen/include/xen/mm.h +10 -0
index 426362adb2f4..d80bfba6d393 100644
@@ -145,6 +145,16 @@ unsigned long avail_node_heap_pages(unsigned int nodeid);
 #define alloc_domheap_page(d,f) (alloc_domheap_pages(d,0,f))
 #define free_domheap_page(p)  (free_domheap_pages(p,0))
 
+/* Free an allocation, and zero the pointer to it. */
+#define FREE_DOMHEAP_PAGES(p, o) do { \
+    void *_ptr_ = (p);                \
+    (p) = NULL;                       \
+    free_domheap_pages(_ptr_, o);     \
+} while ( false )
+#define FREE_DOMHEAP_PAGE(p) FREE_DOMHEAP_PAGES(p, 0)
+
+void scrub_one_page(const struct page_info *pg, bool cold);
+
 int online_page(mfn_t mfn, uint32_t *status);
 int offline_page(mfn_t mfn, int broken, uint32_t *status);
 int query_page_offline(mfn_t mfn, uint32_t *status);
xen/include/xen/sched.h +5 -0
index 91d6a49daf16..735d5b76b411 100644
@@ -661,6 +661,11 @@ struct domain
 
     /* Pointer to console settings; NULL for system domains. */
     struct domain_console *console;
+
+    /* Pointer to allocated domheap page that possibly needs scrubbing. */
+    struct page_info *pending_scrub;
+    unsigned int pending_scrub_order;
+    unsigned int pending_scrub_index;
 } __aligned(PAGE_SIZE);
 
 static inline struct page_list_head *page_to_list(
Signature · 1 line
2.51.0
Reviewed-by on v5 3/4
#5 lore raw

Re: [PATCH v5 3/4] xen/mm: allow deferred scrub of physmap populate allocated pages

Quoted 28 lines · Roger Pau Monne

Physmap population has the need to use pages as big as possible to reduce p2m shattering. However that triggers issues when big enough pages are not yet scrubbed, and so scrubbing must be done at allocation time. On some scenarios with added contention the watchdog can trigger:

Watchdog timer detects that CPU55 is stuck! ----[ Xen-4.17.5-21 x86_64 debug=n Not tainted ]----
CPU: 55

RIP:    e008:[<ffff82d040204c4a>] clear_page_sse2+0x1a/0x30
RFLAGS: 0000000000000202   CONTEXT: hypervisor (d0v12)
[...]
Xen call trace:
   [<ffff82d040204c4a>] R clear_page_sse2+0x1a/0x30
   [<ffff82d04022a121>] S clear_domain_page+0x11/0x20
   [<ffff82d04022c170>] S common/page_alloc.c#alloc_heap_pages+0x400/0x5a0
   [<ffff82d04022d4a7>] S alloc_domheap_pages+0x67/0x180
   [<ffff82d040226f9f>] S common/memory.c#populate_physmap+0x22f/0x3b0
   [<ffff82d040228ec8>] S do_memory_op+0x728/0x1970

Introduce a mechanism to preempt page scrubbing in populate_physmap(). It relies on stashing the dirty page in the domain struct temporarily to preempt to guest context, so the scrubbing can resume when the domain re-enters the hypercall. The added deferral mechanism will only be used for domain construction, and is designed to be used with a single threaded domain builder. If the toolstack makes concurrent calls to XENMEM_populate_physmap for the same target domain it will trash stashed pages, resulting in slow domain physmap population.

Quoted 8 lines

Note a similar issue is present in increase reservation. However that hypercall is likely to only be used once the domain is already running and the known implementations use 4K pages. It will be deal with in a separate patch using a different approach, that will also take care of the allocation in populate_physmap() once the domain is running.

Fixes 74d2e11ccfd2 ("mm: Scrub pages in alloc_heap_pages() if needed")

with one minor remark:

Quoted 14 lines
xen/common/memory.c +10 -0
@@ -286,6 +365,30 @@ static void populate_physmap(struct memop_args *a)
                     goto out;
                 }
 
+                if ( memflags & MEMF_no_scrub )
+                {
+                    unsigned int dirty_cnt = 0;
+
+                    /* Check if there's anything to scrub. */
+                    for ( j = scrub_start; j < (1U << a->extent_order); j++ )
+                    {
+                        if ( !test_and_clear_bit(_PGC_need_scrub,
+                                                 &page[j].count_info) )
+                            continue;

xen/common/memory.c static void populate_physmap(struct memop_args *a) · line 387

Quoted 10 lines
xen/common/memory.c +10 -0
+
+                        scrub_one_page(&page[j], true);
+
+                        if ( (j + 1) != (1U << a->extent_order) &&
+                             !(++dirty_cnt & 0xff) &&
+                             hypercall_preempt_check() )
+                        {
+                            a->preempted = 1;
+                            stash_allocation(d, page, a->extent_order, ++j);
+                            goto out;

As j isn't used anymore afterwards, j + 1 may be more natural to use here, if only to avoid raised eyebrows.

Jan

#6 lore raw

PATCH v5 4/4 xen/mm: limit non-scrubbed allocations to a specific order

In reply to Roger Pau Monne

The current logic allows for up to 1G pages to be scrubbed in place, which can cause the watchdog to trigger in practice. Reduce the limit for in-place scrubbed allocations to a newly introduced define:
CONFIG_DIRTY_MAX_ORDER. This currently defaults to CONFIG_PTDOM_MAX_ORDER on all architectures. Also introduce a command line option to set the value.

Fixes 74d2e11ccfd2 ("mm: Scrub pages in alloc_heap_pages() if needed")

---
Changes since v3:
- Note the order limitation is only post-boot.

Changes since v2:
- Move placement of the max-order-dirty option help.
- Add note in memop-max-order about interactions.
- Use CONFIG_PTDOM_MAX_ORDER as the default.

Changes since v1:
- Split from previous patch.
- Introduce a command line option to set the limit.

---
 docs/misc/xen-command-line.pandoc | 13 +++++++++++++
 xen/common/memory.c               |  3 ---
 xen/common/page_alloc.c           | 23 ++++++++++++++++++++++-
 xen/include/xen/mm.h              |  4 ++++
 4 files changed, 39 insertions(+), 4 deletions(-)
docs/misc/xen-command-line.pandoc +13 -0
index 15f7a315a4b5..3577e491e379 100644
@@ -1837,6 +1837,16 @@ presented as the number of bits needed to encode it. This must be at least
 one pending bit to be allocated.
 Defaults to 20 bits (to cover at most 1048576 interrupts).
 
+### max-order-dirty
+> `= <integer>`
+
+Specify the maximum allocation order allowed when scrubbing allocated pages
+in-place.  The allocation is non-preemptive, and hence the value must be keep
+low enough to avoid hogging the CPU for too long.
+
+Defaults to `CONFIG_DIRTY_MAX_ORDER` or if unset to `CONFIG_PTDOM_MAX_ORDER`.
+Note those are internal per-architecture defines not available from Kconfig.
+
 ### mce (x86)
 > `= <boolean>`
 
@@ -1878,6 +1888,9 @@ requests issued by the various kinds of domains (in this order:
 ordinary DomU, control domain, hardware domain, and - when supported
 by the platform - DomU with pass-through device assigned).
 
+Note orders here can be further limited by the value in `max-order-dirty` for
+allocations requesting pages to be scrubbed in-place.
+
 ### mmcfg (x86)
 > `= <boolean>[,amd-fam10]`
 
xen/common/memory.c +0 -3
index 285546298ed9..c2682ecbe5b8 100644
@@ -56,9 +56,6 @@ struct memop_args {
 #ifndef CONFIG_CTLDOM_MAX_ORDER
 #define CONFIG_CTLDOM_MAX_ORDER CONFIG_PAGEALLOC_MAX_ORDER
 #endif
-#ifndef CONFIG_PTDOM_MAX_ORDER
-#define CONFIG_PTDOM_MAX_ORDER CONFIG_HWDOM_MAX_ORDER
-#endif
 
 static unsigned int __read_mostly domu_max_order = CONFIG_DOMU_MAX_ORDER;
 static unsigned int __read_mostly ctldom_max_order = CONFIG_CTLDOM_MAX_ORDER;
xen/common/page_alloc.c +22 -1
index 87ebf5a024dc..725d6ee488a4 100644
@@ -267,6 +267,13 @@ static PAGE_LIST_HEAD(page_offlined_list);
 /* Broken page list, protected by heap_lock. */
 static PAGE_LIST_HEAD(page_broken_list);
 
+/* Maximum order allowed for post-boot allocations with MEMF_no_scrub. */
+#ifndef CONFIG_DIRTY_MAX_ORDER
+# define CONFIG_DIRTY_MAX_ORDER CONFIG_PTDOM_MAX_ORDER
+#endif
+static unsigned int __ro_after_init dirty_max_order = CONFIG_DIRTY_MAX_ORDER;
+integer_param("max-order-dirty", dirty_max_order);
+
 /*************************
  * BOOT-TIME ALLOCATOR
  */
@@ -1008,7 +1015,13 @@ static struct page_info *alloc_heap_pages(
 
     pg = get_free_buddy(zone_lo, zone_hi, order, memflags, d);
     /* Try getting a dirty buddy if we couldn't get a clean one. */
-    if ( !pg && !(memflags & MEMF_no_scrub) )
+    if ( !pg && !(memflags & MEMF_no_scrub) &&
+         /*
+          * Allow any order unscrubbed allocations during boot time, we
+          * compensate by processing softirqs in the scrubbing loop below once
+          * irqs are enabled.
+          */
+         (order <= dirty_max_order || system_state < SYS_STATE_active) )
         pg = get_free_buddy(zone_lo, zone_hi, order,
                             memflags | MEMF_no_scrub, d);
     if ( !pg )
@@ -1117,6 +1130,14 @@ static struct page_info *alloc_heap_pages(
                     scrub_one_page(&pg[i], cold);
 
                 dirty_cnt++;
+
+                /*
+                 * Use SYS_STATE_smp_boot explicitly; ahead of that state
+                 * interrupts are disabled.
+                 */
+                if ( system_state == SYS_STATE_smp_boot &&
+                     !(dirty_cnt & 0xff) )
+                    process_pending_softirqs();
             }
             else
                 check_one_page(&pg[i]);
xen/include/xen/mm.h +4 -0
index d80bfba6d393..cf3796d4286d 100644
@@ -232,6 +232,10 @@ struct npfec {
 #else
 #define MAX_ORDER 20 /* 2^20 contiguous pages */
 #endif
+#ifndef CONFIG_PTDOM_MAX_ORDER
+# define CONFIG_PTDOM_MAX_ORDER CONFIG_HWDOM_MAX_ORDER
+#endif
+
 mfn_t acquire_reserved_page(struct domain *d, unsigned int memflags);
 
 /* Private domain structs for DOMID_XEN, DOMID_IO, etc. */
Signature · 1 line
2.51.0
#7 lore raw

Re: [PATCH v5 4/4] xen/mm: limit non-scrubbed allocations to a specific order

Quoted 5 lines · Roger Pau Monne
xen/common/page_alloc.c +1 -0
@@ -1117,6 +1130,14 @@ static struct page_info *alloc_heap_pages(
                     scrub_one_page(&pg[i], cold);
 
                 dirty_cnt++;
+

xen/common/page_alloc.c static struct page_info *alloc_heap_pages( · line 1143

Quoted 10 lines
xen/common/page_alloc.c +7 -0
+                /*
+                 * Use SYS_STATE_smp_boot explicitly; ahead of that state
+                 * interrupts are disabled.
+                 */
+                if ( system_state == SYS_STATE_smp_boot &&
+                     !(dirty_cnt & 0xff) )
+                    process_pending_softirqs();
             }
             else
                 check_one_page(&pg[i]);

Coming back to this, I see two possible issues. One is that on x86 enabling of IRQs happens a bit earlier than setting of SYS_STATE_smp_boot. Maybe this isn't really a problem right now, but it could become one if the "wrong" thing is inserted into that window in __start_xen().

The other is that only x86 actually ever sets that state.

Jan

#8 lore raw

Re: [PATCH v5 4/4] xen/mm: limit non-scrubbed allocations to a specific order

Quoted 13 lines · Jan Beulich
Quoted 12 lines · Roger Pau Monne
xen/common/page_alloc.c +8 -0
@@ -1117,6 +1130,14 @@ static struct page_info *alloc_heap_pages(
                     scrub_one_page(&pg[i], cold);
 
                 dirty_cnt++;
+
+                /*
+                 * Use SYS_STATE_smp_boot explicitly; ahead of that state
+                 * interrupts are disabled.
+                 */
+                if ( system_state == SYS_STATE_smp_boot &&
+                     !(dirty_cnt & 0xff) )
+                    process_pending_softirqs();
Quoted 8 lines

xen/common/page_alloc.c static struct page_info *alloc_heap_pages( · line 1143

Quoted 3 lines
xen/common/page_alloc.c +0 -0
             }
             else
                 check_one_page(&pg[i]);

Coming back to this, I see two possible issues. One is that on x86 enabling of IRQs happens a bit earlier than setting of SYS_STATE_smp_boot. Maybe this isn't really a problem right now, but it could become one if the "wrong" thing is inserted into that window in __start_xen().

Even if IRQs are enabled a bit earlier, the watchdog is only setup after SYS_STATE_smp_boot state is set.

Quoted 1 line

The other is that only x86 actually ever sets that state.

TBH we have never seen the watchdog triggering during initial domain creation, so the workaround added here was out of caution. There's a certain amount of contention required for the watchdog to trigger, and that scenario doesn't happen during boot, as domain creation is serialized.

I could maybe gate the softirq processing as:

                if ( system_state < SYS_STATE_active && local_irq_enabled() &&
                     !(dirty_cnt & 0xff) )
                    process_pending_softirqs();

Or completely avoid softirq processing in alloc_heap_pages().

Thanks, Roger.

#9 lore raw

Re: [PATCH v5 4/4] xen/mm: limit non-scrubbed allocations to a specific order

Quoted 23 lines · Roger Pau Monné
Quoted 13 lines · Jan Beulich
Quoted 12 lines · Roger Pau Monne
xen/common/page_alloc.c +8 -0
@@ -1117,6 +1130,14 @@ static struct page_info *alloc_heap_pages(
                     scrub_one_page(&pg[i], cold);
 
                 dirty_cnt++;
+
+                /*
+                 * Use SYS_STATE_smp_boot explicitly; ahead of that state
+                 * interrupts are disabled.
+                 */
+                if ( system_state == SYS_STATE_smp_boot &&
+                     !(dirty_cnt & 0xff) )
+                    process_pending_softirqs();
Quoted 8 lines

xen/common/page_alloc.c static struct page_info *alloc_heap_pages( · line 1143

Quoted 3 lines
xen/common/page_alloc.c +0 -0
             }
             else
                 check_one_page(&pg[i]);

Coming back to this, I see two possible issues. One is that on x86 enabling of IRQs happens a bit earlier than setting of SYS_STATE_smp_boot. Maybe this isn't really a problem right now, but it could become one if the "wrong" thing is inserted into that window in __start_xen().

Quoted 2 lines

Even if IRQs are enabled a bit earlier, the watchdog is only setup after SYS_STATE_smp_boot state is set.

Hmm, yes. Adjustments like this, otoh, shouldn't primarily be to silence the watchdog, but to address overly long delays that might trigger it. I.e. they might (generally: would) be as relevant without the watchdog actually being active.

Quoted 7 lines
Quoted 1 line

The other is that only x86 actually ever sets that state.

TBH we have never seen the watchdog triggering during initial domain creation, so the workaround added here was out of caution. There's a certain amount of contention required for the watchdog to trigger, and that scenario doesn't happen during boot, as domain creation is serialized.

It is right now, but really both dom0less and hyperlaunch could benefit from parallelization.

Quoted 5 lines

I could maybe gate the softirq processing as:

                if ( system_state < SYS_STATE_active && local_irq_enabled() &&
                     !(dirty_cnt & 0xff) )
                    process_pending_softirqs();

Yes please.

Quoted 1 line

Or completely avoid softirq processing in alloc_heap_pages().

Rather not, imo.

Jan