Skip to contentSkip to conversation

mm: add __ prefix to page_cpupid_reset_last because it is non-atomic

PATCH In review Nicholas Piggin 2 participants 6 messages Mar 17, 2021 – Jun 8, 2021

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/3 mm: add __ prefix to page_cpupid_reset_last because it is non-atomic

Add a __ prefix to indicate that it should be used in non-atomic situations (where the page is not subject to concurrent flags access, following the pattern of the other page flags modifications).

This prepares for an atomic version.

---
Rebase only. Any comments on this idea?

Thanks,
Nick

 include/linux/mm.h | 6 +++---
 mm/page_alloc.c    | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)
include/linux/mm.h +3 -3
index 64a71bf20536..086ab710383f 100644
@@ -1396,7 +1396,7 @@ static inline int page_cpupid_last(struct page *page)
 {
 	return page->_last_cpupid;
 }
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
 }
@@ -1408,7 +1408,7 @@ static inline int page_cpupid_last(struct page *page)
 
 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
 
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
 }
@@ -1449,7 +1449,7 @@ static inline bool cpupid_pid_unset(int cpupid)
 	return true;
 }
 
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 }
 
mm/page_alloc.c +2 -2
index cfc72873961d..172368357e73 100644
@@ -1266,7 +1266,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
 	if (bad)
 		return false;
 
-	page_cpupid_reset_last(page);
+	__page_cpupid_reset_last(page);
 	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
 	reset_page_owner(page, order);
 
@@ -1463,7 +1463,7 @@ static void __meminit __init_single_page(struct page *page, unsigned long pfn,
 	set_page_links(page, zone, nid, pfn);
 	init_page_count(page);
 	page_mapcount_reset(page);
-	page_cpupid_reset_last(page);
+	__page_cpupid_reset_last(page);
 	page_kasan_tag_reset(page);
 
 	INIT_LIST_HEAD(&page->lru);
Signature · 1 line
2.23.0
#1 lore raw

PATCH v2 2/3 mm: add page_cpupid_reset_last atomic variant

This can be used when there is concurrency on the struct page. If cpupid is not in page flags, it can use a store.

Use this to replace the open-coded reset in wp_page_reuse.

---
 include/linux/mm.h | 15 +++++++++++++++
 mm/memory.c        |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)
include/linux/mm.h +15 -0
index 086ab710383f..2524553ea898 100644
@@ -1400,6 +1400,11 @@ static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
 }
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+	WRITE_ONCE(page->_last_cpupid, -1 & LAST_CPUPID_MASK);
+}
 #else
 static inline int page_cpupid_last(struct page *page)
 {
@@ -1412,6 +1417,12 @@ static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
 }
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+	page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+}
+
 #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
 #else /* !CONFIG_NUMA_BALANCING */
 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
@@ -1453,6 +1464,10 @@ static inline void __page_cpupid_reset_last(struct page *page)
 {
 }
 
+static inline void page_cpupid_reset_last(struct page *page)
+{
+}
+
 static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
 {
 	return false;
mm/memory.c +1 -1
index 5efa07fb6cdc..bb37dc0f4dac 100644
@@ -2804,7 +2804,7 @@ static inline void wp_page_reuse(struct vm_fault *vmf)
 	 * unrelated process.
 	 */
 	if (page)
-		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
+		page_cpupid_reset_last(page);
 
 	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
 	entry = pte_mkyoung(vmf->orig_pte);
Signature · 1 line
2.23.0
#2 lore raw

PATCH v2 3/3 mm: skip page_cpupid_reset_last atomic update if the value is already reset

In reply to Nicholas Piggin

Improves performance of a PROT_READ -> PROT_READ|PROT_WRITE -> store microbenchmark (which goes via the wp_page_reuse fault path) by about 10% on a POWER9.

---
 include/linux/mm.h | 3 +++
 1 file changed, 3 insertions(+)
include/linux/mm.h +3 -0
index 2524553ea898..1e1471235055 100644
@@ -1420,6 +1420,9 @@ static inline void __page_cpupid_reset_last(struct page *page)
 
 static inline void page_cpupid_reset_last(struct page *page)
 {
+	if (page_cpupid_last(page) == (-1 & LAST_CPUPID_MASK))
+		return;
+
 	page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
 }
 
Signature · 1 line
2.23.0
#3 lore raw
In reply to Nicholas Piggin

Ping? Any concerns about these patches?

Thanks,
Nick

Excerpts from Nicholas Piggin's message of March 17, 2021 4:47 pm:

Quoted 74 lines

Add a __ prefix to indicate that it should be used in non-atomic situations (where the page is not subject to concurrent flags access, following the pattern of the other page flags modifications).

This prepares for an atomic version.

---
Rebase only. Any comments on this idea?

Thanks,
Nick

 include/linux/mm.h | 6 +++---
 mm/page_alloc.c    | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)
include/linux/mm.h +3 -3
index 64a71bf20536..086ab710383f 100644
@@ -1396,7 +1396,7 @@ static inline int page_cpupid_last(struct page *page)
 {
 	return page->_last_cpupid;
 }
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
 }
@@ -1408,7 +1408,7 @@ static inline int page_cpupid_last(struct page *page)
 
 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
 
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
 }
@@ -1449,7 +1449,7 @@ static inline bool cpupid_pid_unset(int cpupid)
 	return true;
 }
 
-static inline void page_cpupid_reset_last(struct page *page)
+static inline void __page_cpupid_reset_last(struct page *page)
 {
 }
 
mm/page_alloc.c +2 -2
index cfc72873961d..172368357e73 100644
@@ -1266,7 +1266,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
 	if (bad)
 		return false;
 
-	page_cpupid_reset_last(page);
+	__page_cpupid_reset_last(page);
 	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
 	reset_page_owner(page, order);
 
@@ -1463,7 +1463,7 @@ static void __meminit __init_single_page(struct page *page, unsigned long pfn,
 	set_page_links(page, zone, nid, pfn);
 	init_page_count(page);
 	page_mapcount_reset(page);
-	page_cpupid_reset_last(page);
+	__page_cpupid_reset_last(page);
 	page_kasan_tag_reset(page);
 
 	INIT_LIST_HEAD(&page->lru);
Signature · 1 line
2.23.0