fs/proc/task_mmu: Fix uffd-wp confusion in pagemap_scan_pmd_entry()
Subsystems (3)
PATCH 1/1 fs/proc/task_mmu: Fix uffd-wp confusion in pagemap_scan_pmd_entry()
pagemap_scan_pmd_entry() checks if uffd-wp is set on each pte to avoid unnecessary if set. However it was previously checking with pte_uffd_wp(ptep_get(pte)) without first confirming that the pte was present. It is only valid to call pte_uffd_wp() for present ptes. For swap ptes, pte_swp_uffd_wp() must be called because the uffd-wp bit may be kept in a different position, depending on the arch.
This was leading to test failures in the pagemap_ioctl mm selftest, when bringing up uffd-wp support on arm64 due to incorrectly interpretting the uffd-wp status of migration entries.
Let's fix this by using the correct check based on pte_present(). While we are at it, let's pass the pte to make_uffd_wp_pte() to avoid the pointless extra ptep_get() which can't be optimized out due to READ_ONCE() on many arches.
--- fs/proc/task_mmu.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-)
fs/proc/task_mmu.c +13 -9
@@ -1817,10 +1817,8 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, }static void make_uffd_wp_pte (struct vm_area_struct *vma, -unsigned long addr,pte_t *pte) +unsigned long addr,pte_t *pte,pte_t ptent) { -pte_t ptent =ptep_get (pte); -if (pte_present (ptent)) {pte_t old_pte;
@@ -2175,9 +2173,12 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,if ((p->arg.flags &PM_SCAN_WP_MATCHING ) && !p->vec_out) {/* Fast path for performing exclusive WP */ for (addr = start; addr != end; pte++, addr +=PAGE_SIZE ) { -if (pte_uffd_wp (ptep_get (pte))) +pte_t ptent =ptep_get (pte); + +if ((pte_present (ptent) &&pte_uffd_wp (ptent)) || +pte_swp_uffd_wp_any (ptent))continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = addr +PAGE_SIZE ;
@@ -2190,8 +2191,10 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, p->arg.return_mask == PAGE_IS_WRITTEN) {for (addr = start; addr < end; pte++, addr +=PAGE_SIZE ) {unsigned long next = addr +PAGE_SIZE ; +pte_t ptent =ptep_get (pte); -if (pte_uffd_wp (ptep_get (pte))) +if ((pte_present (ptent) &&pte_uffd_wp (ptent)) || +pte_swp_uffd_wp_any (ptent))continue ; ret =pagemap_scan_output (p->cur_vma_category |PAGE_IS_WRITTEN , p, addr, &next);
@@ -2199,7 +2202,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,break ;if (~p->arg.flags &PM_SCAN_WP_MATCHING )continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = next;
@@ -2208,8 +2211,9 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, }for (addr = start; addr != end; pte++, addr +=PAGE_SIZE ) { +pte_t ptent =ptep_get (pte);unsigned long categories = p->cur_vma_category | -pagemap_page_category (p, vma, addr,ptep_get (pte)); +pagemap_page_category (p, vma, addr, ptent);unsigned long next = addr +PAGE_SIZE ;if (!pagemap_scan_is_interesting_page (categories, p))
@@ -2224,7 +2228,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,if (~categories &PAGE_IS_WRITTEN )continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = next;
--
2.25.1
Quoted 16 lines · Ryan Roberts
pagemap_scan_pmd_entry()checks if uffd-wp is set on each pte to avoid unnecessary if set. However it was previously checking withpte_uffd_wp(ptep_get(pte))without first confirming that the pte was present. It is only valid to callpte_uffd_wp()for present ptes. For swap ptes,pte_swp_uffd_wp()must be called because the uffd-wp bit may be kept in a different position, depending on the arch.This was leading to test failures in the
pagemap_ioctlmm selftest, when bringing up uffd-wp support on arm64 due to incorrectly interpretting the uffd-wp status of migration entries.Let's fix this by using the correct check based on
pte_present(). While we are at it, let's pass the pte tomake_uffd_wp_pte()to avoid the pointless extraptep_get()which can't be optimized out due toREAD_ONCE()on many arches.
Quoted 4 lines
Fixes 12f6b01a0bcb ("fs/proc/task_mmu: add fast paths to get/clear PAGE_IS_WRITTEN flag")Signed-off-by Ryan Roberts <[email protected]>---
Signature · 3 lines
Cheers, David / dhildenb
Thanks for finding and fixing Ryan!
Quoted 11 lines · Ryan Roberts
pagemap_scan_pmd_entry()checks if uffd-wp is set on each pte to avoid unnecessary if set. However it was previously checking withpte_uffd_wp(ptep_get(pte))without first confirming that the pte was present. It is only valid to callpte_uffd_wp()for present ptes. For swap ptes,pte_swp_uffd_wp()must be called because the uffd-wp bit may be kept in a different position, depending on the arch.This was leading to test failures in the
pagemap_ioctlmm selftest, when bringing up uffd-wp support on arm64 due to incorrectly interpretting the uffd-wp status of migration entries.
Quoted 8 lines
Let's fix this by using the correct check based on
pte_present(). While we are at it, let's pass the pte tomake_uffd_wp_pte()to avoid the pointless extraptep_get()which can't be optimized out due toREAD_ONCE()on many arches.Fixes 12f6b01a0bcb ("fs/proc/task_mmu: add fast paths to get/clear PAGE_IS_WRITTEN flag")Signed-off-by Ryan Roberts <[email protected]>
Quoted 79 lines
--- fs/proc/task_mmu.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-)fs/proc/task_mmu.c +13
-10@@ -1817,10 +1817,8 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, }static void make_uffd_wp_pte (struct vm_area_struct *vma, -unsigned long addr,pte_t *pte) +unsigned long addr,pte_t *pte,pte_t ptent) { -pte_t ptent =ptep_get (pte); -if (pte_present (ptent)) {pte_t old_pte;@@ -2175,9 +2173,12 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,if ((p->arg.flags &PM_SCAN_WP_MATCHING ) && !p->vec_out) {/* Fast path for performing exclusive WP */ for (addr = start; addr != end; pte++, addr +=PAGE_SIZE ) { -if (pte_uffd_wp (ptep_get (pte))) +pte_t ptent =ptep_get (pte); + +if ((pte_present (ptent) &&pte_uffd_wp (ptent)) || +pte_swp_uffd_wp_any (ptent))continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = addr +PAGE_SIZE ;@@ -2190,8 +2191,10 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, p->arg.return_mask == PAGE_IS_WRITTEN) {for (addr = start; addr < end; pte++, addr +=PAGE_SIZE ) {unsigned long next = addr +PAGE_SIZE ; +pte_t ptent =ptep_get (pte); -if (pte_uffd_wp (ptep_get (pte))) +if ((pte_present (ptent) &&pte_uffd_wp (ptent)) || +pte_swp_uffd_wp_any (ptent))continue ; ret =pagemap_scan_output (p->cur_vma_category |PAGE_IS_WRITTEN , p, addr, &next);@@ -2199,7 +2202,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,break ;if (~p->arg.flags &PM_SCAN_WP_MATCHING )continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = next;@@ -2208,8 +2211,9 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, }for (addr = start; addr != end; pte++, addr +=PAGE_SIZE ) { +pte_t ptent =ptep_get (pte);unsigned long categories = p->cur_vma_category | -pagemap_page_category (p, vma, addr,ptep_get (pte)); +pagemap_page_category (p, vma, addr, ptent);unsigned long next = addr +PAGE_SIZE ;if (!pagemap_scan_is_interesting_page (categories, p))@@ -2224,7 +2228,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,if (~categories &PAGE_IS_WRITTEN )continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = next; --2.25.1
Signature · 2 lines
BR, Muhammad Usama Anjum
Quoted 11 lines · Ryan Roberts
pagemap_scan_pmd_entry()checks if uffd-wp is set on each pte to avoid unnecessary if set. However it was previously checking withpte_uffd_wp(ptep_get(pte))without first confirming that the pte was present. It is only valid to callpte_uffd_wp()for present ptes. For swap ptes,pte_swp_uffd_wp()must be called because the uffd-wp bit may be kept in a different position, depending on the arch.This was leading to test failures in the
pagemap_ioctlmm selftest, when bringing up uffd-wp support on arm64 due to incorrectly interpretting the uffd-wp status of migration entries.
Quoted 8 lines
Let's fix this by using the correct check based on
pte_present(). While we are at it, let's pass the pte tomake_uffd_wp_pte()to avoid the pointless extraptep_get()which can't be optimized out due toREAD_ONCE()on many arches.Fixes 12f6b01a0bcb ("fs/proc/task_mmu: add fast paths to get/clear PAGE_IS_WRITTEN flag")Signed-off-by Ryan Roberts <[email protected]>
I guess this should have cc'ed stable but I forgot to add it. Are you able to fix this up when you take it, Andrew, or do I need to repost?
Quoted 79 lines
--- fs/proc/task_mmu.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-)fs/proc/task_mmu.c +13
-10@@ -1817,10 +1817,8 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, }static void make_uffd_wp_pte (struct vm_area_struct *vma, -unsigned long addr,pte_t *pte) +unsigned long addr,pte_t *pte,pte_t ptent) { -pte_t ptent =ptep_get (pte); -if (pte_present (ptent)) {pte_t old_pte;@@ -2175,9 +2173,12 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,if ((p->arg.flags &PM_SCAN_WP_MATCHING ) && !p->vec_out) {/* Fast path for performing exclusive WP */ for (addr = start; addr != end; pte++, addr +=PAGE_SIZE ) { -if (pte_uffd_wp (ptep_get (pte))) +pte_t ptent =ptep_get (pte); + +if ((pte_present (ptent) &&pte_uffd_wp (ptent)) || +pte_swp_uffd_wp_any (ptent))continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = addr +PAGE_SIZE ;@@ -2190,8 +2191,10 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, p->arg.return_mask == PAGE_IS_WRITTEN) {for (addr = start; addr < end; pte++, addr +=PAGE_SIZE ) {unsigned long next = addr +PAGE_SIZE ; +pte_t ptent =ptep_get (pte); -if (pte_uffd_wp (ptep_get (pte))) +if ((pte_present (ptent) &&pte_uffd_wp (ptent)) || +pte_swp_uffd_wp_any (ptent))continue ; ret =pagemap_scan_output (p->cur_vma_category |PAGE_IS_WRITTEN , p, addr, &next);@@ -2199,7 +2202,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,break ;if (~p->arg.flags &PM_SCAN_WP_MATCHING )continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = next;@@ -2208,8 +2211,9 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, }for (addr = start; addr != end; pte++, addr +=PAGE_SIZE ) { +pte_t ptent =ptep_get (pte);unsigned long categories = p->cur_vma_category | -pagemap_page_category (p, vma, addr,ptep_get (pte)); +pagemap_page_category (p, vma, addr, ptent);unsigned long next = addr +PAGE_SIZE ;if (!pagemap_scan_is_interesting_page (categories, p))@@ -2224,7 +2228,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,if (~categories &PAGE_IS_WRITTEN )continue ; -make_uffd_wp_pte (vma, addr, pte); +make_uffd_wp_pte (vma, addr, pte, ptent);if (!flush_end) start = addr; flush_end = next; --2.25.1