userfaultfd: snapshot VMA state across UFFDIO_COPY retry
Subsystems (2)
Mike Rapoport
maintainerPATCH RESEND 1/1 userfaultfd: snapshot VMA state across UFFDIO_COPY retry
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry() drops the VMA lock for copy_from_user() and reacquires it afterwards. The destination VMA can be replaced during that window.
The existing check compares vma_uffd_ops() before and after the retry, but if a shmem VMA with MAP_SHARED is replaced with a shmem VMA with MAP_PRIVATE (or vice versa) the replacement goes undetected.
The change from MAP_PRIVATE to MAP_SHARED will treat the folio allocated with shmem_alloc_folio() as anonymous and this will cause BUG() when mfill_atomic_install_pte() will try to folio_add_new_anon_rmap().
The change from MAP_SHARED to MAP_PRIVATE allows injection of folios into the page cache of the original VMA.
Introduce helpers for more comprehensive comparison of VMA state:
- vma_snapshot_get() to save the relevant VMA state into a struct vma_snapshot (original uffd_ops, actual uffd_ops, relevant VMA flags, vm_file and pgoff) before dropping the lock
- vma_snapshot_changed() to compare the saved state with the state of the VMA acquired after retaking the locks
- vma_snapshot_put() to release vm_file pinning.
Use DEFINE_FREE() cleanup to wrap vma_snapshot_put() to avoid complicating error handling paths in mfill_copy_folio_retry().
Add vma_uffd_copy_ops() to avoid code duplication when original ops of shmem VMA with MAP_PRIVATE are replaced with anon_uffd_ops.
--- mm/userfaultfd.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 20 deletions(-)
mm/userfaultfd.c +79 -20
@@ -14,6 +14,8 @@#include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> +#include <linux/file.h> +#include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h"
@@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; } +static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma) +{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ; + +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/ +if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops; + +return ops; +} +static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) {
@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)return ret; } +#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT) + +struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +}; + +static void vma_snapshot_get (struct vma_snapshot *s,struct vm_area_struct *vma) +{ + s->flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); + s->copy_ops =vma_uffd_copy_ops (vma); + s->ops =vma_uffd_ops (vma); + s->pgoff = vma->vm_pgoff; + +if (vma->vm_file) + s->file =get_file (vma->vm_file); +} + +static bool vma_snapshot_changed (struct vma_snapshot *s, +struct vm_area_struct *vma) +{ +vma_flags_t flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); + +if (!vma_flags_same_pair (&s->flags, &flags)) +return true; + +/* VMA type or effective uffd_ops changed while the lock was dropped */ +if (s->ops !=vma_uffd_ops (vma) || s->copy_ops !=vma_uffd_copy_ops (vma)) +return true; + +/* VMA was anonymous before; changed only if it no longer is */ +if (!s->file) +return !vma_is_anonymous (vma); + +/* VMA was file backed, but inode or offset has changed */ +if (!vma->vm_file || vma->vm_file->f_inode != s->file->f_inode || + vma->vm_pgoff != s->pgoff) +return true; + +return false; +} + +static void vma_snapshot_put (struct vma_snapshot *s) +{ +if (s->file) +fput (s->file); +} + +DEFINE_FREE(snapshot_put, struct vma_snapshot *, if (_T) vma_snapshot_put(_T)); +static int mfill_copy_folio_retry (struct mfill_state *state,struct folio *folio) { -const struct vm_uffd_ops *orig_ops =vma_uffd_ops (state->vma); +struct vma_snapshot s = {0 }; + struct vma_snapshot *p __free(snapshot_put) = &s;unsigned long src_addr = state->src_addr;void *kaddr;int err; +vma_snapshot_get (&s, state->vma); +/* retry copying with mm_lock dropped */ mfill_put_vma (state);
@@ -467,12 +543,7 @@ static int mfill_copy_folio_retry(struct mfill_state *state,if (err)return err; -/* -* The VMA type may have changed while the lock was dropped -* (e.g. replaced with a hugetlb mapping), making the caller's -* ops pointer stale. -*/ -if (vma_uffd_ops (state->vma) != orig_ops) +if (vma_snapshot_changed (&s, state->vma))return -EAGAIN ; err =mfill_establish_pmd (state);
@@ -545,19 +616,7 @@ static int __mfill_atomic_pte(struct mfill_state *state,static int mfill_atomic_pte_copy (struct mfill_state *state) { -const struct vm_uffd_ops *ops =vma_uffd_ops (state->vma); - -/* -* The normal page fault path for a MAP_PRIVATE mapping in a -* file-backed VMA will invoke the fault, fill the hole in the file and -* COW it right away. The result generates plain anonymous memory. -* So when we are asked to fill a hole in a MAP_PRIVATE mapping, we'll -* generate anonymous memory directly without actually filling the -* hole. For the MAP_PRIVATE case the robustness check only happens in -* the pagetable (to verify it's still none) and not in the page cache. -*/ -if (!(state->vma->vm_flags &VM_SHARED )) - ops = &anon_uffd_ops; +const struct vm_uffd_ops *ops =vma_uffd_copy_ops (state->vma);return __mfill_atomic_pte (state, ops); }
base-commit: 444fc9435e57157fcf30fc99aee44997f3458641
Signature · 1 line
2.53.0
Hi Mike,
Quoted 59 lines · Mike Rapoport
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry()drops the VMA lock forcopy_from_user()and reacquires it afterwards. The destination VMA can be replaced during that window.The existing check compares
vma_uffd_ops()before and after the retry, but if a shmem VMA withMAP_SHAREDis replaced with a shmem VMA withMAP_PRIVATE(or vice versa) the replacement goes undetected.The change from
MAP_PRIVATEtoMAP_SHAREDwill treat the folio allocated withshmem_alloc_folio()as anonymous and this will causeBUG()whenmfill_atomic_install_pte()will try tofolio_add_new_anon_rmap().The change from
MAP_SHAREDtoMAP_PRIVATEallows injection of folios into the page cache of the original VMA.Introduce helpers for more comprehensive comparison of VMA state:
-vma_snapshot_get()to save the relevant VMA state into a structvma_snapshot(originaluffd_ops, actualuffd_ops, relevant VMA flags,vm_fileand pgoff) before dropping the lock
-vma_snapshot_changed()to compare the saved state with the state of the VMA acquired after retaking the locks
-vma_snapshot_put()to releasevm_filepinning.Use
DEFINE_FREE()cleanup to wrapvma_snapshot_put()to avoid complicating error handling paths inmfill_copy_folio_retry().Add
vma_uffd_copy_ops()to avoid code duplication when original ops of shmem VMA withMAP_PRIVATEare replaced withanon_uffd_ops.Fixes 292411fda25b ("mm/userfaultfd: detect VMA type change after copy retry in mfill_copy_folio_retry()")Fixes 6ab703034f14 ("userfaultfd: mfill_atomic(): remove retry logic")Tested-by Heechan Kang <[email protected]>Suggested-by Peter Xu <[email protected]>Co-developed-by David Carlier <[email protected]>Signed-off-by David Carlier <[email protected]>Co-developed-by Michael Bommarito <[email protected]>Signed-off-by Michael Bommarito <[email protected]>Signed-off-by Mike Rapoport (Microsoft) <[email protected]>--- mm/userfaultfd.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 20 deletions(-)mm/userfaultfd.c +2
-0@@ -14,6 +14,8 @@#include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> +#include <linux/file.h> +#include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h"
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 74
Quoted 4 lines
mm/userfaultfd.c +1
-0@@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; } +static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma)
My only 2 cent, I would name it vma_uffd_effective_copy_ops() instead or a comment to highlight it is about "UFFDIO_COPY into a MAP_PRIVATE file-backed"
Quoted 128 lines
mm/userfaultfd.c +76
-20+{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ; + +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/ +if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops; + +return ops; +} +static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) {@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)return ret; } +#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT) + +struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +}; + +static void vma_snapshot_get (struct vma_snapshot *s,struct vm_area_struct *vma) +{ + s->flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); + s->copy_ops =vma_uffd_copy_ops (vma); + s->ops =vma_uffd_ops (vma); + s->pgoff = vma->vm_pgoff; + +if (vma->vm_file) + s->file =get_file (vma->vm_file); +} + +static bool vma_snapshot_changed (struct vma_snapshot *s, +struct vm_area_struct *vma) +{ +vma_flags_t flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); + +if (!vma_flags_same_pair (&s->flags, &flags)) +return true; + +/* VMA type or effective uffd_ops changed while the lock was dropped */ +if (s->ops !=vma_uffd_ops (vma) || s->copy_ops !=vma_uffd_copy_ops (vma)) +return true; + +/* VMA was anonymous before; changed only if it no longer is */ +if (!s->file) +return !vma_is_anonymous (vma); + +/* VMA was file backed, but inode or offset has changed */ +if (!vma->vm_file || vma->vm_file->f_inode != s->file->f_inode || + vma->vm_pgoff != s->pgoff) +return true; + +return false; +} + +static void vma_snapshot_put (struct vma_snapshot *s) +{ +if (s->file) +fput (s->file); +} + +DEFINE_FREE(snapshot_put, struct vma_snapshot *, if (_T) vma_snapshot_put(_T)); +static int mfill_copy_folio_retry (struct mfill_state *state,struct folio *folio) { -const struct vm_uffd_ops *orig_ops =vma_uffd_ops (state->vma); +struct vma_snapshot s = {0 }; + struct vma_snapshot *p __free(snapshot_put) = &s;unsigned long src_addr = state->src_addr;void *kaddr;int err; +vma_snapshot_get (&s, state->vma); +/* retry copying with mm_lock dropped */ mfill_put_vma (state);@@ -467,12 +543,7 @@ static int mfill_copy_folio_retry(struct mfill_state *state,if (err)return err; -/* -* The VMA type may have changed while the lock was dropped -* (e.g. replaced with a hugetlb mapping), making the caller's -* ops pointer stale. -*/ -if (vma_uffd_ops (state->vma) != orig_ops) +if (vma_snapshot_changed (&s, state->vma))return -EAGAIN ; err =mfill_establish_pmd (state);@@ -545,19 +616,7 @@ static int __mfill_atomic_pte(struct mfill_state *state,static int mfill_atomic_pte_copy (struct mfill_state *state) { -const struct vm_uffd_ops *ops =vma_uffd_ops (state->vma); - -/* -* The normal page fault path for a MAP_PRIVATE mapping in a -* file-backed VMA will invoke the fault, fill the hole in the file and -* COW it right away. The result generates plain anonymous memory. -* So when we are asked to fill a hole in a MAP_PRIVATE mapping, we'll -* generate anonymous memory directly without actually filling the -* hole. For the MAP_PRIVATE case the robustness check only happens in -* the pagetable (to verify it's still none) and not in the page cache. -*/ -if (!(state->vma->vm_flags &VM_SHARED )) - ops = &anon_uffd_ops; +const struct vm_uffd_ops *ops =vma_uffd_copy_ops (state->vma);return __mfill_atomic_pte (state, ops); }
Quoted 4 lines
base-commit:
444fc9435e57157fcf30fc99aee44997f3458641--
2.53.0
Cheers.
Mike Rapoport
maintainerHi David,
Quoted 11 lines · David CARLIER
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 74
Quoted 5 lines · Mike Rapoport
mm/userfaultfd.c +1
-0@@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; } +static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma)My only 2 cent, I would name it
vma_uffd_effective_copy_ops()instead or a comment to highlight it is about "UFFDIO_COPYinto aMAP_PRIVATEfile-backed"
Too long for my taste :)
And comment is useful anyway as it explains why we override the ops at all.
Quoted 16 lines
Quoted 16 lines
mm/userfaultfd.c +16
-0+{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ; + +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/ +if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops; + +return ops; +}
Signature · 2 lines
Sincerely yours, Mike.
Quoted 85 lines · Mike Rapoport
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry()drops the VMA lock forcopy_from_user()and reacquires it afterwards. The destination VMA can be replaced during that window.The existing check compares
vma_uffd_ops()before and after the retry, but if a shmem VMA withMAP_SHAREDis replaced with a shmem VMA withMAP_PRIVATE(or vice versa) the replacement goes undetected.The change from
MAP_PRIVATEtoMAP_SHAREDwill treat the folio allocated withshmem_alloc_folio()as anonymous and this will causeBUG()whenmfill_atomic_install_pte()will try tofolio_add_new_anon_rmap().The change from
MAP_SHAREDtoMAP_PRIVATEallows injection of folios into the page cache of the original VMA.Introduce helpers for more comprehensive comparison of VMA state:
-vma_snapshot_get()to save the relevant VMA state into a structvma_snapshot(originaluffd_ops, actualuffd_ops, relevant VMA flags,vm_fileand pgoff) before dropping the lock
-vma_snapshot_changed()to compare the saved state with the state of the VMA acquired after retaking the locks
-vma_snapshot_put()to releasevm_filepinning.Use
DEFINE_FREE()cleanup to wrapvma_snapshot_put()to avoid complicating error handling paths inmfill_copy_folio_retry().Add
vma_uffd_copy_ops()to avoid code duplication when original ops of shmem VMA withMAP_PRIVATEare replaced withanon_uffd_ops.Fixes 292411fda25b ("mm/userfaultfd: detect VMA type change after copy retry in mfill_copy_folio_retry()")Fixes 6ab703034f14 ("userfaultfd: mfill_atomic(): remove retry logic")Tested-by Heechan Kang <[email protected]>Suggested-by Peter Xu <[email protected]>Co-developed-by David Carlier <[email protected]>Signed-off-by David Carlier <[email protected]>Co-developed-by Michael Bommarito <[email protected]>Signed-off-by Michael Bommarito <[email protected]>Signed-off-by Mike Rapoport (Microsoft) <[email protected]>--- mm/userfaultfd.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 20 deletions(-)mm/userfaultfd.c +20
-0@@ -14,6 +14,8 @@#include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> +#include <linux/file.h> +#include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h" @@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; } +static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma) +{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ; + +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/ +if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops; + +return ops; +} +static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) {@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)return ret; }
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 474
Quoted 10 lines
mm/userfaultfd.c +9
-0+#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT) + +struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +};
As this is all uffd specific, I wonder whether that should be "struct uffd_vma_snapshot"/"vma_uffd_snapshot" etc.
From a high level, this LGTM.
I wish we could identify relevant VMA changes more easily. Like, using a per-MM sequence counter that we simply increment on any VMA changes.
Signature · 3 lines
Cheers, David
Mike Rapoport
maintainerQuoted 87 lines · David Hildenbrand (Arm)
Quoted 86 lines · Mike Rapoport
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry()drops the VMA lock forcopy_from_user()and reacquires it afterwards. The destination VMA can be replaced during that window.The existing check compares
vma_uffd_ops()before and after the retry, but if a shmem VMA withMAP_SHAREDis replaced with a shmem VMA withMAP_PRIVATE(or vice versa) the replacement goes undetected.The change from
MAP_PRIVATEtoMAP_SHAREDwill treat the folio allocated withshmem_alloc_folio()as anonymous and this will causeBUG()whenmfill_atomic_install_pte()will try tofolio_add_new_anon_rmap().The change from
MAP_SHAREDtoMAP_PRIVATEallows injection of folios into the page cache of the original VMA.Introduce helpers for more comprehensive comparison of VMA state:
-vma_snapshot_get()to save the relevant VMA state into a structvma_snapshot(originaluffd_ops, actualuffd_ops, relevant VMA flags,vm_fileand pgoff) before dropping the lock
-vma_snapshot_changed()to compare the saved state with the state of the VMA acquired after retaking the locks
-vma_snapshot_put()to releasevm_filepinning.Use
DEFINE_FREE()cleanup to wrapvma_snapshot_put()to avoid complicating error handling paths inmfill_copy_folio_retry().Add
vma_uffd_copy_ops()to avoid code duplication when original ops of shmem VMA withMAP_PRIVATEare replaced withanon_uffd_ops.Fixes 292411fda25b ("mm/userfaultfd: detect VMA type change after copy retry in mfill_copy_folio_retry()")Fixes 6ab703034f14 ("userfaultfd: mfill_atomic(): remove retry logic")Tested-by Heechan Kang <[email protected]>Suggested-by Peter Xu <[email protected]>Co-developed-by David Carlier <[email protected]>Signed-off-by David Carlier <[email protected]>Co-developed-by Michael Bommarito <[email protected]>Signed-off-by Michael Bommarito <[email protected]>Signed-off-by Mike Rapoport (Microsoft) <[email protected]>--- mm/userfaultfd.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 20 deletions(-)mm/userfaultfd.c +20
-0@@ -14,6 +14,8 @@#include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> +#include <linux/file.h> +#include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h" @@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; } +static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma) +{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ; + +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/ +if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops; + +return ops; +} +static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) {@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)return ret; }
Quoted 12 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 474
Quoted 9 lines
mm/userfaultfd.c +9
-0+#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT) + +struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +};As this is all uffd specific, I wonder whether that should be "struct uffd_vma_snapshot"/"vma_uffd_snapshot" etc.
It's local to this file, I wouldn't worry about namespacing until there's an actual need if it will ever arise.
Quoted 4 lines
From a high level, this LGTM.
I wish we could identify relevant VMA changes more easily. Like, using a per-MM sequence counter that we simply increment on any VMA changes.
Do you mean per-VMA?
Per-MM counter would capture unrelated changes, e.g. an masvise() for unrelated range.
We kinda have an infrastructure to detect VMA changes that affect uffd operation for non-cooperative, we can also hook on that even if UFFD_EVENT_* are not requested, but that's way more involved that this VMA snapshot.
Quoted 4 lines
Signature · 3 lines
Cheers, David
Signature · 2 lines
Sincerely yours, Mike.
Quoted 101 lines · Mike Rapoport
Quoted 87 lines · David Hildenbrand (Arm)
Quoted 86 lines · Mike Rapoport
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry()drops the VMA lock forcopy_from_user()and reacquires it afterwards. The destination VMA can be replaced during that window.The existing check compares
vma_uffd_ops()before and after the retry, but if a shmem VMA withMAP_SHAREDis replaced with a shmem VMA withMAP_PRIVATE(or vice versa) the replacement goes undetected.The change from
MAP_PRIVATEtoMAP_SHAREDwill treat the folio allocated withshmem_alloc_folio()as anonymous and this will causeBUG()whenmfill_atomic_install_pte()will try tofolio_add_new_anon_rmap().The change from
MAP_SHAREDtoMAP_PRIVATEallows injection of folios into the page cache of the original VMA.Introduce helpers for more comprehensive comparison of VMA state:
-vma_snapshot_get()to save the relevant VMA state into a structvma_snapshot(originaluffd_ops, actualuffd_ops, relevant VMA flags,vm_fileand pgoff) before dropping the lock
-vma_snapshot_changed()to compare the saved state with the state of the VMA acquired after retaking the locks
-vma_snapshot_put()to releasevm_filepinning.Use
DEFINE_FREE()cleanup to wrapvma_snapshot_put()to avoid complicating error handling paths inmfill_copy_folio_retry().Add
vma_uffd_copy_ops()to avoid code duplication when original ops of shmem VMA withMAP_PRIVATEare replaced withanon_uffd_ops.Fixes 292411fda25b ("mm/userfaultfd: detect VMA type change after copy retry in mfill_copy_folio_retry()")Fixes 6ab703034f14 ("userfaultfd: mfill_atomic(): remove retry logic")Tested-by Heechan Kang <[email protected]>Suggested-by Peter Xu <[email protected]>Co-developed-by David Carlier <[email protected]>Signed-off-by David Carlier <[email protected]>Co-developed-by Michael Bommarito <[email protected]>Signed-off-by Michael Bommarito <[email protected]>Signed-off-by Mike Rapoport (Microsoft) <[email protected]>--- mm/userfaultfd.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 20 deletions(-)mm/userfaultfd.c +20
-0@@ -14,6 +14,8 @@#include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> +#include <linux/file.h> +#include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h" @@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; } +static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma) +{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ; + +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/ +if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops; + +return ops; +} +static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) {@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)return ret; }Quoted 12 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 474
Quoted 9 lines
mm/userfaultfd.c +9
-0+#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT) + +struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +};As this is all uffd specific, I wonder whether that should be "struct uffd_vma_snapshot"/"vma_uffd_snapshot" etc.
Quoted 2 lines
It's local to this file, I wouldn't worry about namespacing until there's an actual need if it will ever arise.
Indeed, fair enough.
Quoted 9 lines
Quoted 5 lines
From a high level, this LGTM.
I wish we could identify relevant VMA changes more easily. Like, using a per-MM sequence counter that we simply increment on any VMA changes.
Do you mean per-VMA?
Per-MM counter would capture unrelated changes, e.g. anmasvise()for unrelated range.
I was thinking about a per-MM thing for simplicity. If there were any changes, we'd retry (-EAGAIN).
IOW, something like &mm->mm_lock_seq, which we have for per-vma locks already.
Not sure if unrelated changes would really be a problem in practice (another thread gabbing the mmap_lock in write mode until we serviced the fault).
Quoted 5 lines
We kinda have an infrastructure to detect VMA changes that affect uffd operation for non-cooperative, we can also hook on that even if UFFD_EVENT_* are not requested, but that's way more involved that this VMA snapshot.
Yeah, that sounds too complicated and what you have here is simpler.
Signature · 3 lines
Cheers, David
Quoted 21 lines · David Hildenbrand (Arm)
Quoted 7 lines · Mike Rapoport
Quoted 3 lines · David Hildenbrand (Arm)
As this is all uffd specific, I wonder whether that should be "struct uffd_vma_snapshot"/"vma_uffd_snapshot" etc.
It's local to this file, I wouldn't worry about namespacing until there's an actual need if it will ever arise.
Indeed, fair enough.
Quoted 9 lines
Quoted 5 lines
From a high level, this LGTM.
I wish we could identify relevant VMA changes more easily. Like, using a per-MM sequence counter that we simply increment on any VMA changes.
Do you mean per-VMA?
Per-MM counter would capture unrelated changes, e.g. anmasvise()for unrelated range.
Quoted 7 lines
I was thinking about a per-MM thing for simplicity. If there were any changes, we'd retry (
-EAGAIN).IOW, something like
&mm->mm_lock_seq, which we have for per-vma locks already.Not sure if unrelated changes would really be a problem in practice (another thread gabbing the
mmap_lockin write mode until we serviced the fault).
Assume something like the following:
mm/userfaultfd.c +4 -1
@@ -448,9 +448,12 @@ static int mfill_copy_folio_retry(struct mfill_state *state, {const struct vm_uffd_ops *orig_ops =vma_uffd_ops (state->vma);unsigned long src_addr = state->src_addr; +unsigned int seq;void *kaddr;int err; + seq =raw_read_seqcount (&mm->mm_lock_seq); +/* retry copying with mm_lock dropped */ mfill_put_vma (state);
@@ -472,7 +475,7 @@ static int mfill_copy_folio_retry(struct mfill_state *state,* (e.g. replaced with a hugetlb mapping), making the caller's * ops pointer stale. */ -if (vma_uffd_ops (state->vma) != orig_ops) +if (read_seqcount_retry (&mm->mm_lock_seq, seq))return -EAGAIN ; err =mfill_establish_pmd (state);
In case there is concurrent mmap_write lock, we'd retry. But we triggered and resolved the fault. So the retry would just succeed in copying from the page, not triggering another fault here.
So I'd not expect this to matter in practice?
We'd have to implement &mm->mm_lock_seq also without per-VMA locks. Which wouldn't be too bad given that Dave proposed a patch set to just enable per-VMA locks unconditionally.
As a hotfix, what you have is probably better, but a simplification like this might later make sense.
Signature · 3 lines
Cheers, David
Quoted 65 lines · David Hildenbrand (Arm)
Quoted 21 lines · David Hildenbrand (Arm)
Quoted 7 lines · Mike Rapoport
Quoted 3 lines · David Hildenbrand (Arm)
As this is all uffd specific, I wonder whether that should be "struct uffd_vma_snapshot"/"vma_uffd_snapshot" etc.
It's local to this file, I wouldn't worry about namespacing until there's an actual need if it will ever arise.
Indeed, fair enough.
Quoted 9 lines
Quoted 5 lines
From a high level, this LGTM.
I wish we could identify relevant VMA changes more easily. Like, using a per-MM sequence counter that we simply increment on any VMA changes.
Do you mean per-VMA?
Per-MM counter would capture unrelated changes, e.g. anmasvise()for unrelated range.Quoted 7 lines
I was thinking about a per-MM thing for simplicity. If there were any changes, we'd retry (
-EAGAIN).IOW, something like
&mm->mm_lock_seq, which we have for per-vma locks already.Not sure if unrelated changes would really be a problem in practice (another thread gabbing the
mmap_lockin write mode until we serviced the fault).Assume something like the following:
mm/userfaultfd.c +4
-1@@ -448,9 +448,12 @@ static int mfill_copy_folio_retry(struct mfill_state *state, {const struct vm_uffd_ops *orig_ops =vma_uffd_ops (state->vma);unsigned long src_addr = state->src_addr; +unsigned int seq;void *kaddr;int err; + seq =raw_read_seqcount (&mm->mm_lock_seq); +/* retry copying with mm_lock dropped */ mfill_put_vma (state);@@ -472,7 +475,7 @@ static int mfill_copy_folio_retry(struct mfill_state *state,* (e.g. replaced with a hugetlb mapping), making the caller's * ops pointer stale. */ -if (vma_uffd_ops (state->vma) != orig_ops) +if (read_seqcount_retry (&mm->mm_lock_seq, seq))return -EAGAIN ; err =mfill_establish_pmd (state);In case there is concurrent
mmap_writelock, we'd retry. But we triggered and resolved the fault. So the retry would just succeed in copying from the page, not triggering another fault here.
Quoted 6 lines
So I'd not expect this to matter in practice?
We'd have to implement
&mm->mm_lock_seqalso without per-VMA locks. Which wouldn't be too bad given that Dave proposed a patch set to just enable per-VMA locks unconditionally.
That set unfortunately is broken for its ostensible purpose AFAICT, but the idea of making per-VMA locks permanently available is good and I hope that we land that at some point.
Quoted 3 lines
As a hotfix, what you have is probably better, but a simplification like this might later make sense.
Hmmm, but the proposed logic absolutely tolerates changes to the VMA that does not alter the fundamental properties being checked for, so this is an entirely different check no?
I don't think we can both say that we only care about properties X, Y, and Z changing and simultaneously tracking whether a VMA write lock was applied since we dropped the lock?
Also a VMA write lock having been obtained doesn't mean the VMA was necessarily changed?
And then couldn't -EAGAIN cause spurious write locks to abort the operation altogether?
Given reacquisition of the lock will fall back to waiting on the mmap_read_lock() if the VMA was write locked, that makes a race very possible.
I mean maybe we want to tolerate all of the above and just move to a safer world where we conservatively consider any VMA write lock acquisition to be one in which we abort the operation, which I think would be nicer...
Quoted 5 lines
--
Cheers,David
Cheers, Lorenzo
Mike Rapoport
maintainerQuoted 8 lines · David Hildenbrand (Arm)
Quoted 6 lines · Mike Rapoport
Quoted 1 line · David Hildenbrand (Arm)
On 5/19/26 07:25, Mike Rapoport wrote:
Do you mean per-VMA?
Per-MM counter would capture unrelated changes, e.g. anmasvise()for unrelated range.
Quoted 7 lines
I was thinking about a per-MM thing for simplicity. If there were any changes, we'd retry (
-EAGAIN).IOW, something like
&mm->mm_lock_seq, which we have for per-vma locks already.Not sure if unrelated changes would really be a problem in practice (another thread gabbing the
mmap_lockin write mode until we serviced the fault).
Let me reiterate:
A thread doing UFFDIO_COPY releases the VMA in mfill_copy_folio_retry(), re-gets the VMA and checks if the per-MM counter stayed the same.
If another thread makes any change to VMA while mfill_copy_folio_retry() waits to re-get the VMA, the counter would be incremented by another thread. mfill_copy_folio_retry() will see the change after mfill_get_vma() and will bail out with -EAGAIN.
So if another thread does, e.g. MADV_DONTFORK on completely unrelated VMA, mfill_copy_folio_retry() will return -EAGAIN and I'm not sure we'll not break the existing userspace by this.
Quoted 4 lines
Signature · 3 lines
Cheers, David
Signature · 2 lines
Sincerely yours, Mike.
Quoted 17 lines · Mike Rapoport
Quoted 6 lines · David Hildenbrand (Arm)
Quoted 4 lines · Mike Rapoport
Do you mean per-VMA?
Per-MM counter would capture unrelated changes, e.g. anmasvise()for unrelated range.Quoted 7 lines
I was thinking about a per-MM thing for simplicity. If there were any changes, we'd retry (
-EAGAIN).IOW, something like
&mm->mm_lock_seq, which we have for per-vma locks already.Not sure if unrelated changes would really be a problem in practice (another thread gabbing the
mmap_lockin write mode until we serviced the fault).Let me reiterate:
Quoted 8 lines
A thread doing
UFFDIO_COPYreleases the VMA inmfill_copy_folio_retry(), re-gets the VMA and checks if the per-MM counter stayed the same.If another thread makes any change to VMA while
mfill_copy_folio_retry()waits to re-get the VMA, the counter would be incremented by another thread.mfill_copy_folio_retry()will see the change aftermfill_get_vma()and will bail out with-EAGAIN.
Yeah.
Quoted 3 lines
So if another thread does, e.g.
MADV_DONTFORKon completely unrelated VMA,mfill_copy_folio_retry()will return-EAGAINand I'm not sure we'll not break the existing userspace by this.
Ah, you mean that -EAGAIN would not be handled by user space already even though documented in the man page ... because it relies on -EAGAIN not happening in specific situations (i.e., copying a single page).
That's very likely true and we'd have to retry internally, which makes it more complicated indeed.
Signature · 3 lines
Cheers, David
Quoted 21 lines · David Hildenbrand (Arm)
Quoted 20 lines · Mike Rapoport
Quoted 6 lines · David Hildenbrand (Arm)
Quoted 4 lines · Mike Rapoport
Do you mean per-VMA?
Per-MM counter would capture unrelated changes, e.g. anmasvise()for unrelated range.Quoted 7 lines
I was thinking about a per-MM thing for simplicity. If there were any changes, we'd retry (
-EAGAIN).IOW, something like
&mm->mm_lock_seq, which we have for per-vma locks already.Not sure if unrelated changes would really be a problem in practice (another thread gabbing the
mmap_lockin write mode until we serviced the fault).Let me reiterate:
A thread doing
UFFDIO_COPYreleases the VMA inmfill_copy_folio_retry(), re-gets the VMA and checks if the per-MM counter stayed the same.
Quoted 7 lines
Quoted 5 lines
If another thread makes any change to VMA while
mfill_copy_folio_retry()waits to re-get the VMA, the counter would be incremented by another thread.mfill_copy_folio_retry()will see the change aftermfill_get_vma()and will bail out with-EAGAIN.Yeah.
This isn't bulletproof anyways. The sequence count can wrap. So, if someone can replace the vma then cause the sequence counter wrap, then you can be fooled into thinking it's okay (we had this problem years ago with the vmacache using a 32 bit counter, iirc).
Quoted 16 lines
Quoted 4 lines
So if another thread does, e.g.
MADV_DONTFORKon completely unrelated VMA,mfill_copy_folio_retry()will return-EAGAINand I'm not sure we'll not break the existing userspace by this.Ah, you mean that
-EAGAINwould not be handled by user space already even though documented in the man page ... because it relies on-EAGAINnot happening in specific situations (i.e., copying a single page).That's very likely true and we'd have to retry internally, which makes it more complicated indeed.
Signature · 3 lines
Cheers, David
Quoted 14 lines · Liam R. Howlett
Quoted 13 lines · David Hildenbrand (Arm)
Quoted 11 lines · Mike Rapoport
Let me reiterate:
A thread doing
UFFDIO_COPYreleases the VMA inmfill_copy_folio_retry(), re-gets the VMA and checks if the per-MM counter stayed the same.If another thread makes any change to VMA while
mfill_copy_folio_retry()waits to re-get the VMA, the counter would be incremented by another thread.mfill_copy_folio_retry()will see the change aftermfill_get_vma()and will bail out with-EAGAIN.
Quoted 6 lines
Quoted 1 line
Yeah.
This isn't bulletproof anyways. The sequence count can wrap. So, if someone can replace the vma then cause the sequence counter wrap, then you can be fooled into thinking it's okay (we had this problem years ago with the vmacache using a 32 bit counter, iirc).
If you can get it to wrap for such short durations, then how would sequence counters possibly work in any reasonable context?
Signature · 3 lines
Cheers, David
Quoted 17 lines · David Hildenbrand (Arm)
Quoted 16 lines · Liam R. Howlett
Quoted 14 lines · David Hildenbrand (Arm)
Quoted 11 lines · Mike Rapoport
Let me reiterate:
A thread doing
UFFDIO_COPYreleases the VMA inmfill_copy_folio_retry(), re-gets the VMA and checks if the per-MM counter stayed the same.If another thread makes any change to VMA while
mfill_copy_folio_retry()waits to re-get the VMA, the counter would be incremented by another thread.mfill_copy_folio_retry()will see the change aftermfill_get_vma()and will bail out with-EAGAIN.Yeah.
Quoted 7 lines
Quoted 4 lines
This isn't bulletproof anyways. The sequence count can wrap. So, if someone can replace the vma then cause the sequence counter wrap, then you can be fooled into thinking it's okay (we had this problem years ago with the vmacache using a 32 bit counter, iirc).
If you can get it to wrap for such short durations, then how would sequence counters possibly work in any reasonable context?
Surely even for a 32-bit value, we can be pretty confident we're not going to see a wrap that matters (the seqnum will != the prev seqnum unless 4 billion VMA write locks were obtained)?
I may be missing something though!
Quoted 5 lines
--
Cheers,David
Thanks, Lorenzo
Quoted 18 lines · Lorenzo Stoakes
Quoted 17 lines · David Hildenbrand (Arm)
Quoted 16 lines · Liam R. Howlett
Quoted 14 lines · David Hildenbrand (Arm)
Quoted 11 lines · Mike Rapoport
Let me reiterate:
A thread doing
UFFDIO_COPYreleases the VMA inmfill_copy_folio_retry(), re-gets the VMA and checks if the per-MM counter stayed the same.If another thread makes any change to VMA while
mfill_copy_folio_retry()waits to re-get the VMA, the counter would be incremented by another thread.mfill_copy_folio_retry()will see the change aftermfill_get_vma()and will bail out with-EAGAIN.Yeah.
Quoted 7 lines
Quoted 7 lines
Quoted 4 lines
This isn't bulletproof anyways. The sequence count can wrap. So, if someone can replace the vma then cause the sequence counter wrap, then you can be fooled into thinking it's okay (we had this problem years ago with the vmacache using a 32 bit counter, iirc).
If you can get it to wrap for such short durations, then how would sequence counters possibly work in any reasonable context?
I think the answer is because 1. it may not be a short duration once we're asking userspace to do things and 2. because we aren't depending on the sequence number alone, usually.
I don't think we should trust it if we're dropping other locks/guarantees and waiting on userspace.
Quoted 4 lines
Surely even for a 32-bit value, we can be pretty confident we're not going to see a wrap that matters (the seqnum will != the prev seqnum unless 4 billion VMA write locks were obtained)?
The vmacache had a similar issue and it was reported as a potential security problem, which caused it to switch to a 64bit value. [1] I mean, it took 40 minutes, but it was possible..
Note that the vmacache was a bit different in that it was invalidating the cache when the number wrapped.
And the vmacache was mm wide, while the per-vma sequence is mm and per-vma specific, so it's very much harder to hit a false positive. But, I guess if you can register a bunch of UFFDs and fork many processes, you can increase your odds?
If you recall, we did try a 64 bit early on, but switched to a 32bit to avoid a performance regression.
I think we should take a really hard look at the locking mechanics here if we're going to try and depend on the sequence number before trusting it as a correctness guarantee, especially when involving uffd copy.
[1]. https://www.exploit-db.com/exploits/45497
Thanks,
Liam
Quoted 10 lines · Mike Rapoport
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry()drops the VMA lock forcopy_from_user()and reacquires it afterwards. The destination VMA can be replaced during that window.The existing check compares
vma_uffd_ops()before and after the retry, but if a shmem VMA withMAP_SHAREDis replaced with a shmem VMA withMAP_PRIVATE(or vice versa) the replacement goes undetected.
Quoted 6 lines
The change from
MAP_PRIVATEtoMAP_SHAREDwill treat the folio allocated withshmem_alloc_folio()as anonymous and this will causeBUG()whenmfill_atomic_install_pte()will try tofolio_add_new_anon_rmap().The change from
MAP_SHAREDtoMAP_PRIVATEallows injection of folios into the page cache of the original VMA.
OK this seems like a sensible subset of things to check - we are looking to check what might materially impact _this particular operation_, and working to the absolute minimum we need to check.
Quoted 8 lines
Introduce helpers for more comprehensive comparison of VMA state:
-vma_snapshot_get()to save the relevant VMA state into a structvma_snapshot(originaluffd_ops, actualuffd_ops, relevant VMA flags,vm_fileand pgoff) before dropping the lock
-vma_snapshot_changed()to compare the saved state with the state of the VMA acquired after retaking the locks
-vma_snapshot_put()to releasevm_filepinning.
I feel like this is possibly a little misleading, it sort of implies that this is a broader kind of snapshot rather than some uffd-specific thing.
This is more of a naming thing, yes things being defined in mm/userfaultfd.c tells you it's uffd-specific, but people are easily confused by stuff like this.
Will comment inline.
Quoted 3 lines
Use
DEFINE_FREE()cleanup to wrapvma_snapshot_put()to avoid complicating error handling paths inmfill_copy_folio_retry().
That's nice!
Quoted 3 lines
Add
vma_uffd_copy_ops()to avoid code duplication when original ops of shmem VMA withMAP_PRIVATEare replaced withanon_uffd_ops.
I think this is a bit overly brief. And is it only shmem where this might happen? Is MAP_PRIVATE hugetlbfs not also possible?
Quoted 3 lines
Fixes 292411fda25b ("mm/userfaultfd: detect VMA type change after copy retry in mfill_copy_folio_retry()")Fixes 6ab703034f14 ("userfaultfd: mfill_atomic(): remove retry logic")
Hmm this is the second multiple-fixes patch I've seen, maybe that is OK to do :) I guess if this one, indivisble change fixes both then that's valid.
Quoted 25 lines
Tested-by Heechan Kang <[email protected]>Suggested-by Peter Xu <[email protected]>Co-developed-by David Carlier <[email protected]>Signed-off-by David Carlier <[email protected]>Co-developed-by Michael Bommarito <[email protected]>Signed-off-by Michael Bommarito <[email protected]>Signed-off-by Mike Rapoport (Microsoft) <[email protected]>--- mm/userfaultfd.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 20 deletions(-)mm/userfaultfd.c +2
-0@@ -14,6 +14,8 @@#include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> +#include <linux/file.h> +#include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h"
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 74
Quoted 4 lines
mm/userfaultfd.c +1
-0@@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; } +static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma)
I find this a bit confusing, these are the uffd ops for... copy? Or you're copying uffd ops?
I'd maybe rename it to vma_uffd_ops_on_copy() to make it clear which it is, and add a comment like:
/*
* Determine the ops to use when performing a UFFDIO_COPY operation - this
* specifically handles the case of a MAP_PRIVATE file-backed mapping,
* which, upon copy, is CoW'd into anonymous memory.
*/
From the snapshot point of view, I have been pondering if this was the clearest way of checking things because you are, in effect checking a specific case
it, but after much back and forth, perhaps it's not too crazy... you are in most cases doing a duplicate check to the ops check
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 79
Quoted 5 lines
mm/userfaultfd.c +5
-0+{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ;
Is this correct? The code you removed from mfill_atomic_pte_copy() unconditionally replaces the ops with &anon_uffd_ops, regardless of whether any operations were specified, but now not providing ops avoids that?
I mean it's correct if we can never have !ops, and presumably we can't because __mfill_atomic_pte() unconditionally calls ops->alloc_folio() right at the top of the logic?
Doesn't uffd only support shmem, anon + hugetlbfs? So why are we adding !ops checks at all?
Actually claude (:>) tells me that async-WP memory mode (ugh god) can be used for _anything_:
/* * If WP is the only mode enabled and context is wp async, allow any * memory type. */ if (wp_async && (vm_flags == VM_UFFD_WP)) return true;
But. This function is never called in anything but a copy context?
I'm not sure if you were protecting against a possible future usage? But in that case surely you'd just want to VM_WARN_ON_ONCE(!ops)?
We should also have a comment describing the situation like:
/*
* Only async WP allows arbitrary file-backed mappings for UFFD,
* which is the only case in which ops can be NULL. However,
* we are copying here which is not permitted for these
* mappings, so this will never be the case here.
*/
VM_WARN_ON_ONCE(!ops);
Also in:
static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)
{
if (vma_is_anonymous(vma))
return &anon_uffd_ops;
return vma->vm_ops ? vma->vm_ops->uffd_ops : NULL;
}This is doing a redundant check _and_ making life confusing, as if !vma->vm_ops is a condition that can be reached there, it can't, as vma_is_anonymous() is literally a !vma->vm_ops check :)
So surely that should be:
static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)
{
if (vma_is_anonymous(vma))
return &anon_uffd_ops;
return vma->vm_ops->uffd_ops;
}I mean obviously that can be a separate patch, but that's not helped make things clear here! :)
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 85
Quoted 6 lines
mm/userfaultfd.c +6
-0+ +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/
The wording here is super confusing. 'This is an effective ops override' and 'retry validation' and also the 'not just'.
Also it makes zero reference to the invocation of this function from mfill_atomic_pte_copy() - this function is pulling double-duty so this comment isn't really quite correct.
I'd maybe pull up the comment from mfill_atomic_pte_copy() (and adapt it to suit the changes, also mentioning the snapshot use case).
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 87
Quoted 2 lines
mm/userfaultfd.c +2
-0+if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops;
This is inconsistent with the rest of the VMA flags API usage, you want:
if (!vma_test(vma, VMA_SHARED_BIT)) return &anon_uffd_ops;
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 466
Quoted 12 lines
mm/userfaultfd.c +5
-0+ +return ops; +} +static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) {@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)return ret; } +#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT)
Hm it's a bit weird to have a VMA flags define outside of mm.h.
But I definitely appreciate you using the new VMA flags stuff :)
Probably just renaming to VMA_UFFD_COPY_STATE_FLAGS or something?
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 467
Quoted 1 line
mm/userfaultfd.c +1
-0+
I'm a bit of a broken record I know :) but a comment here would be helpful describing what this is for.
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 474
Quoted 7 lines
mm/userfaultfd.c +7
-0+struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +};
Yeah I don't love this naming, you're not snapshotting the VMA at all in any real sense, you're comparing a subset of things specifically for the UFFDIO_COPY case.
Which is fine - but I think we should just make that clear here, because perhaps in the future we'll want a generic version of something like this and this is going to be super confusing if we do...
vma_uffdio_copy_snapshot maybe ?
Maybe ok to leave the functions the same name to avoid a mouthful, the type name should make it clear what's going on.
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 476
Quoted 2 lines
mm/userfaultfd.c +2
-0+ +static void vma_snapshot_get (struct vma_snapshot *s,struct vm_area_struct *vma)
No single letter var please! This isn't plan9 :)
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
Quoted 13 lines
mm/userfaultfd.c +12
-0+{ + s->flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); + s->copy_ops =vma_uffd_copy_ops (vma); + s->ops =vma_uffd_ops (vma); + s->pgoff = vma->vm_pgoff; + +if (vma->vm_file) + s->file =get_file (vma->vm_file); +} + +static bool vma_snapshot_changed (struct vma_snapshot *s, +struct vm_area_struct *vma)
Can the VMA potentially be any arbitrary VMA that might now be remapped in place? Like anon vs. file and same file and offset?
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 491
Quoted 3 lines
mm/userfaultfd.c +3
-0+{ +vma_flags_t flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); +
You have a comment for each of the other cases, so one here is useful I think, e.g.:
/* Have any UFFD flags (missing, WP, minor) changed? */
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
Quoted 5 lines
mm/userfaultfd.c +4
-0+if (!vma_flags_same_pair (&s->flags, &flags)) +return true; + +/* VMA type or effective uffd_ops changed while the lock was dropped */
Can the 'effective' (probably better to say copy or use effective throughout) uffd_ops change though? You already check that VMA_SHARED_BIT is either set or unset across the lock drop above so is the copy_ops check even meaningful here?
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 499
Quoted 2 lines
mm/userfaultfd.c +2
-0+if (s->ops !=vma_uffd_ops (vma) || s->copy_ops !=vma_uffd_copy_ops (vma)) +return true;
Surely we should also check that the UFFD context matches (accounting for NULL cases)?
Could we not otherwise have weird situations across UFFD contexts if both VMAs happen to contain identitical properties otherwise?
This seems to be a canonical part of identifying a UFFD VMA so strange to not see it here?
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 506
Quoted 9 lines
mm/userfaultfd.c +9
-0+ +/* VMA was anonymous before; changed only if it no longer is */ +if (!s->file) +return !vma_is_anonymous (vma); + +/* VMA was file backed, but inode or offset has changed */ +if (!vma->vm_file || vma->vm_file->f_inode != s->file->f_inode || + vma->vm_pgoff != s->pgoff) +return true;
Hm so are we OK with a VMA where the file has been closed, then the same inode re-opened at the exact same offset? (e.g. vma->vm_file differs)
Would this be problematic with hugetlbfs or shmem?
I guess we pin the inode so the f_inode check _should_ be valid, from that side.
Quoted 8 lines
mm/userfaultfd.c +8
-0+ +return false; +} + +static void vma_snapshot_put (struct vma_snapshot *s) +{ +if (s->file) +fput (s->file);
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 523
Quoted 10 lines
mm/userfaultfd.c +6
-1+} + +DEFINE_FREE(snapshot_put, struct vma_snapshot *, if (_T) vma_snapshot_put(_T)); +static int mfill_copy_folio_retry (struct mfill_state *state,struct folio *folio) { -const struct vm_uffd_ops *orig_ops =vma_uffd_ops (state->vma); +struct vma_snapshot s = {0 }; + struct vma_snapshot *p __free(snapshot_put) = &s;
Could we avoid these single letter var names?
Quoted 10 lines
mm/userfaultfd.c +2
-0unsigned long src_addr = state->src_addr;void *kaddr;int err; +vma_snapshot_get (&s, state->vma); +/* retry copying with mm_lock dropped */ mm/userfaultfd.c +0
-0mfill_put_vma (state);
mm/userfaultfd.c static int mfill_copy_folio_retry(struct mfill_state *state, · line 474
Quoted 8 lines
mm/userfaultfd.c +0
-5@@ -467,12 +543,7 @@ static int mfill_copy_folio_retry(struct mfill_state *state,if (err)return err; -/* -* The VMA type may have changed while the lock was dropped -* (e.g. replaced with a hugetlb mapping), making the caller's -* ops pointer stale. -*/
This comment we can lose given we are explicitly comparing snapshots, but I think it's not great to explicitly point out that we're doing this because of the lock being dropped, we should say that somewhere.
Quoted 10 lines
mm/userfaultfd.c +1
-2-if (vma_uffd_ops (state->vma) != orig_ops) +if (vma_snapshot_changed (&s, state->vma))return -EAGAIN ; err =mfill_establish_pmd (state);@@ -545,19 +616,7 @@ static int __mfill_atomic_pte(struct mfill_state *state,static int mfill_atomic_pte_copy (struct mfill_state *state) { -const struct vm_uffd_ops *ops =vma_uffd_ops (state->vma);
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 558
Quoted 10 lines
mm/userfaultfd.c +0
-10- -/* -* The normal page fault path for a MAP_PRIVATE mapping in a -* file-backed VMA will invoke the fault, fill the hole in the file and -* COW it right away. The result generates plain anonymous memory. -* So when we are asked to fill a hole in a MAP_PRIVATE mapping, we'll -* generate anonymous memory directly without actually filling the -* hole. For the MAP_PRIVATE case the robustness check only happens in -* the pagetable (to verify it's still none) and not in the page cache. -*/
(As mentioned above) I hate that we lose this comment (or the gist of it at least), can we please move it up to the copy ops function?
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 560
Quoted 2 lines
mm/userfaultfd.c +0
-2-if (!(state->vma->vm_flags &VM_SHARED )) - ops = &anon_uffd_ops;
A comment here would maybe be useful. But with a function rename and a comment at the function itself maybe OK without.
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 619
Quoted 1 line
mm/userfaultfd.c +1
-0+const struct vm_uffd_ops *ops =vma_uffd_copy_ops (state->vma);
Can we please have a:
/* Only async W/P can have missing ops, and that can't copy. */ VM_WARN_ON_ONCE(!ops);
Check here to make it absolutely clear you're not running straight into a NULL pointer deref in __mfill_atomic_pte()? Maybe worth putting a similar assert there too.
Quoted 8 lines
mm/userfaultfd.c +0
-0return __mfill_atomic_pte (state, ops); }base-commit:
444fc9435e57157fcf30fc99aee44997f3458641--
2.53.0
Thanks, Lorenzo
Mike Rapoport
maintainerQuoted 31 lines · Lorenzo Stoakes
Quoted 10 lines · Mike Rapoport
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry()drops the VMA lock forcopy_from_user()and reacquires it afterwards. The destination VMA can be replaced during that window.The existing check compares
vma_uffd_ops()before and after the retry, but if a shmem VMA withMAP_SHAREDis replaced with a shmem VMA withMAP_PRIVATE(or vice versa) the replacement goes undetected.Quoted 6 lines
The change from
MAP_PRIVATEtoMAP_SHAREDwill treat the folio allocated withshmem_alloc_folio()as anonymous and this will causeBUG()whenmfill_atomic_install_pte()will try tofolio_add_new_anon_rmap().The change from
MAP_SHAREDtoMAP_PRIVATEallows injection of folios into the page cache of the original VMA.OK this seems like a sensible subset of things to check - we are looking to check what might materially impact
_thisparticularoperation_, and working to the absolute minimum we need to check.Quoted 8 lines
Introduce helpers for more comprehensive comparison of VMA state:
-vma_snapshot_get()to save the relevant VMA state into a structvma_snapshot(originaluffd_ops, actualuffd_ops, relevant VMA flags,vm_fileand pgoff) before dropping the lock
-vma_snapshot_changed()to compare the saved state with the state of the VMA acquired after retaking the locks
-vma_snapshot_put()to releasevm_filepinning.
Quoted 8 lines
I feel like this is possibly a little misleading, it sort of implies that this is a broader kind of snapshot rather than some uffd-specific thing.
This is more of a naming thing, yes things being defined in
mm/userfaultfd.ctells you it's uffd-specific, but people are easily confused by stuff like this.Will comment inline.
tl;dr:
* will update comments
* will rename vma_uffd_copy_ops() and replace NULL checks with VM_WARN
* will change some of single letter variable names.
Quoted 4 lines
Quoted 3 lines
Use
DEFINE_FREE()cleanup to wrapvma_snapshot_put()to avoid complicating error handling paths inmfill_copy_folio_retry().
Quoted 8 lines
That's nice!
Quoted 3 lines
Add
vma_uffd_copy_ops()to avoid code duplication when original ops of shmem VMA withMAP_PRIVATEare replaced withanon_uffd_ops.I think this is a bit overly brief. And is it only shmem where this might happen? Is
MAP_PRIVATEhugetlbfs not also possible?
No, hugetlb is not applicabe here, it was not changed to use mfill_copy_folio_retry().
Quoted 35 lines
Quoted 3 lines
Fixes 292411fda25b ("mm/userfaultfd: detect VMA type change after copy retry in mfill_copy_folio_retry()")Fixes 6ab703034f14 ("userfaultfd: mfill_atomic(): remove retry logic")Hmm this is the second multiple-fixes patch I've seen, maybe that is OK to do :) I guess if this one, indivisble change fixes both then that's valid.
Quoted 28 lines
Tested-by Heechan Kang <[email protected]>Suggested-by Peter Xu <[email protected]>Co-developed-by David Carlier <[email protected]>Signed-off-by David Carlier <[email protected]>Co-developed-by Michael Bommarito <[email protected]>Signed-off-by Michael Bommarito <[email protected]>Signed-off-by Mike Rapoport (Microsoft) <[email protected]>--- mm/userfaultfd.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 20 deletions(-)mm/userfaultfd.c +2
-0@@ -14,6 +14,8 @@#include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> +#include <linux/file.h> +#include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h" @@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; }
Quoted 7 lines
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 74
Quoted 1 line
mm/userfaultfd.c +1
-0+static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma)I find this a bit confusing, these are the uffd ops for... copy? Or you're copying uffd ops?
I'd maybe rename it to
vma_uffd_ops_on_copy()to make it clear which it is, and add a comment like:
vma_uffd_ops_for_copy()?
Quoted 5 lines
/*
* Determine the ops to use when performing aUFFDIO_COPYoperation - this
* specifically handles the case of aMAP_PRIVATEfile-backed mapping,
* which, upon copy, is CoW'd into anonymous memory.
*/
I'll pick the large comment that got dropped and stick it here.
Quoted 29 lines
From the snapshot point of view, I have been pondering if this was the clearest way of checking things because you are, in effect checking a specific case
it, but after much back and forth, perhaps it's not too crazy... you are in most cases doing a duplicate check to the ops check
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 79
Quoted 5 lines
mm/userfaultfd.c +5
-0+{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ;Is this correct? The code you removed from
mfill_atomic_pte_copy()unconditionally replaces the ops with&anon_uffd_ops, regardless of whether any operations were specified, but now not providing ops avoids that?I mean it's correct if we can never have !ops, and presumably we can't because
__mfill_atomic_pte()unconditionally callsops->alloc_folio()right at the top of the logic?Doesn't uffd only support shmem, anon + hugetlbfs? So why are we adding !ops checks at all?
Actually claude (:>) tells me that async-WP memory mode (ugh god) can be used for _anything_:
Quoted 8 lines
/* * If WP is the only mode enabled and context is wp async, allow any * memory type. */ if (wp_async && (vm_flags == VM_UFFD_WP)) return true;But. This function is never called in anything but a copy context?
wp_async has nothing to do with copy (or zeropage).
Quoted 11 lines
I'm not sure if you were protecting against a possible future usage? But in that case surely you'd just want to
VM_WARN_ON_ONCE(!ops)?We should also have a comment describing the situation like:
/*
* Only async WP allows arbitrary file-backed mappings for UFFD,
* which is the only case in which ops can be NULL. However,
* we are copying here which is not permitted for these
* mappings, so this will never be the case here.
*/
I can live with one sentence tops here ;-)
Quoted 25 lines
VM_WARN_ON_ONCE(!ops);Also in:
static const struct
vm_uffd_ops*vma_uffd_ops(structvm_area_struct*vma){ if (vma_is_anonymous(vma)) return &anon_uffd_ops; return vma->vm_ops ? vma->vm_ops->uffd_ops : NULL; }This is doing a redundant check _and_ making life confusing, as if !vma->vm_ops is a condition that can be reached there, it can't, as
vma_is_anonymous()is literally a !vma->vm_ops check :)So surely that should be:
static const struct
vm_uffd_ops*vma_uffd_ops(structvm_area_struct*vma){ if (vma_is_anonymous(vma)) return &anon_uffd_ops; return vma->vm_ops->uffd_ops; }
Quoted 2 lines
I mean obviously that can be a separate patch, but that's not helped make things clear here! :)
You are welcome to send it ;-)
Quoted 14 lines
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 85
Quoted 6 lines
mm/userfaultfd.c +6
-0+ +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/ The wording here is super confusing. 'This is an effective ops override' and 'retry validation' and also the 'not just'.
Also it makes zero reference to the invocation of this function from
mfill_atomic_pte_copy()- this function is pulling double-duty so this comment isn't really quite correct.
Quoted 7 lines
I'd maybe pull up the comment from
mfill_atomic_pte_copy()(and adapt it to suit the changes, also mentioning the snapshot use case).mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 87
Quoted 2 lines
mm/userfaultfd.c +2
-0+if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops;This is inconsistent with the rest of the VMA flags API usage, you want:
This is consistent with the rest of VMA usage in uffd, we can convert it all in one go if you'd like.
Quoted 39 lines
if (!vma_test(vma, VMA_SHARED_BIT)) return &anon_uffd_ops;mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 466
Quoted 12 lines
mm/userfaultfd.c +5
-0+ +return ops; +} +static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) {@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)return ret; } +#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT)Hm it's a bit weird to have a VMA flags define outside of
mm.h.But I definitely appreciate you using the new VMA flags stuff :)
Probably just renaming to
VMA_UFFD_COPY_STATE_FLAGSor something?mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 467
Quoted 1 line
mm/userfaultfd.c +1
-0+I'm a bit of a broken record I know :) but a comment here would be helpful describing what this is for.
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 474
Quoted 7 lines
mm/userfaultfd.c +7
-0+struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +};Yeah I don't love this naming, you're not snapshotting the VMA at all in any real sense, you're comparing a subset of things specifically for the
UFFDIO_COPYcase.
Quoted 3 lines
Which is fine - but I think we should just make that clear here, because perhaps in the future we'll want a generic version of something like this and this is going to be super confusing if we do...
Let's deal with it in the future then. This is a local structure, used by a few local functions.
Quoted 9 lines
vma_uffdio_copy_snapshotmaybe ?Maybe ok to leave the functions the same name to avoid a mouthful, the type name should make it clear what's going on.
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 476
Quoted 2 lines
mm/userfaultfd.c +2
-0+ +static void vma_snapshot_get (struct vma_snapshot *s,struct vm_area_struct *vma)No single letter var please! This isn't plan9 :)
"C is a Spartan language, and your naming conventions should follow suit"
https://docs.kernel.org/process/coding-style.html#naming
Here it's perfectly understandable what 's' means.
Quoted 30 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
Quoted 13 lines
mm/userfaultfd.c +12
-0+{ + s->flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); + s->copy_ops =vma_uffd_copy_ops (vma); + s->ops =vma_uffd_ops (vma); + s->pgoff = vma->vm_pgoff; + +if (vma->vm_file) + s->file =get_file (vma->vm_file); +} + +static bool vma_snapshot_changed (struct vma_snapshot *s, +struct vm_area_struct *vma)Can the VMA potentially be any arbitrary VMA that might now be remapped in place? Like anon vs. file and same file and offset?
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 491
Quoted 3 lines
mm/userfaultfd.c +3
-0+{ +vma_flags_t flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); +You have a comment for each of the other cases, so one here is useful I think, e.g.:
/* Have any UFFD flags (missing, WP, minor) changed? */Quoted 2 lines
mm/userfaultfd.c +2
-0+if (!vma_flags_same_pair (&s->flags, &flags)) +return true;
Quoted 7 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 496
Quoted 2 lines
mm/userfaultfd.c +2
-0+ +/* VMA type or effective uffd_ops changed while the lock was dropped */ Can the 'effective' (probably better to say copy or use effective throughout)
uffd_opschange though? You already check thatVMA_SHARED_BITis either set or unset across the lock drop above so is thecopy_opscheck even meaningful here?
Yes, it can change if a shmem VMA changes from SHARED to PRIVATE or vice versa.
Quoted 4 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 498
Quoted 2 lines
mm/userfaultfd.c +2
-0+if (s->ops !=vma_uffd_ops (vma) || s->copy_ops !=vma_uffd_copy_ops (vma)) +return true;Surely we should also check that the UFFD context matches (accounting for NULL cases)?
It's checked in mfill_get_vma().
Quoted 6 lines
Could we not otherwise have weird situations across UFFD contexts if both VMAs happen to contain identitical properties otherwise?
This seems to be a canonical part of identifying a UFFD VMA so strange to not see it here?
Quoted 12 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 506
Quoted 9 lines
mm/userfaultfd.c +9
-0+ +/* VMA was anonymous before; changed only if it no longer is */ +if (!s->file) +return !vma_is_anonymous (vma); + +/* VMA was file backed, but inode or offset has changed */ +if (!vma->vm_file || vma->vm_file->f_inode != s->file->f_inode || + vma->vm_pgoff != s->pgoff) +return true;Hm so are we OK with a VMA where the file has been closed, then the same inode re-opened at the exact same offset? (e.g.
vma->vm_filediffers)
The copy will go to exactly the same place in that case.
Quoted 17 lines
Would this be problematic with hugetlbfs or shmem?
I guess we pin the inode so the
f_inodecheck _should_ be valid, from that side.Quoted 12 lines
mm/userfaultfd.c +12
-0+ +return false; +} + +static void vma_snapshot_put (struct vma_snapshot *s) +{ +if (s->file) +fput (s->file); +} + +DEFINE_FREE(snapshot_put, struct vma_snapshot *, if (_T) vma_snapshot_put(_T)); +
Quoted 8 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 523
Quoted 6 lines
mm/userfaultfd.c +2
-1static int mfill_copy_folio_retry (struct mfill_state *state,struct folio *folio) { -const struct vm_uffd_ops *orig_ops =vma_uffd_ops (state->vma); +struct vma_snapshot s = {0 }; + struct vma_snapshot *p __free(snapshot_put) = &s;Could we avoid these single letter var names?
You suggest to call it a_dummy_pointer_to_allow_use_of_free_cleanup? ;-P
Quoted 31 lines
Quoted 11 lines
mm/userfaultfd.c +2
-0unsigned long src_addr = state->src_addr;void *kaddr;int err; +vma_snapshot_get (&s, state->vma); +/* retry copying with mm_lock dropped */ mfill_put_vma (state);mm/userfaultfd.c static int mfill_copy_folio_retry(struct mfill_state *state, · line 474
Quoted 8 lines
mm/userfaultfd.c +0
-5@@ -467,12 +543,7 @@ static int mfill_copy_folio_retry(struct mfill_state *state,if (err)return err; -/* -* The VMA type may have changed while the lock was dropped -* (e.g. replaced with a hugetlb mapping), making the caller's -* ops pointer stale. -*/ This comment we can lose given we are explicitly comparing snapshots, but I think it's not great to explicitly point out that we're doing this because of the lock being dropped, we should say that somewhere.
Quoted 7 lines
mm/userfaultfd.c +1
-1-if (vma_uffd_ops (state->vma) != orig_ops) +if (vma_snapshot_changed (&s, state->vma))return -EAGAIN ; err =mfill_establish_pmd (state);@@ -545,19 +616,7 @@ static int __mfill_atomic_pte(struct mfill_state *state,
Quoted 16 lines
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 558
Quoted 13 lines
mm/userfaultfd.c +0
-11static int mfill_atomic_pte_copy (struct mfill_state *state) { -const struct vm_uffd_ops *ops =vma_uffd_ops (state->vma); - -/* -* The normal page fault path for a MAP_PRIVATE mapping in a -* file-backed VMA will invoke the fault, fill the hole in the file and -* COW it right away. The result generates plain anonymous memory. -* So when we are asked to fill a hole in a MAP_PRIVATE mapping, we'll -* generate anonymous memory directly without actually filling the -* hole. For the MAP_PRIVATE case the robustness check only happens in -* the pagetable (to verify it's still none) and not in the page cache. -*/ (As mentioned above) I hate that we lose this comment (or the gist of it at least), can we please move it up to the copy ops function?
Will move it before the copy_ops()
Quoted 6 lines
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 560
Quoted 2 lines
mm/userfaultfd.c +0
-2-if (!(state->vma->vm_flags &VM_SHARED )) - ops = &anon_uffd_ops;A comment here would maybe be useful. But with a function rename and a comment at the function itself maybe OK without.
Quoted 6 lines
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 619
Quoted 1 line
mm/userfaultfd.c +1
-0+const struct vm_uffd_ops *ops =vma_uffd_copy_ops (state->vma);Can we please have a:
/* Only async W/P can have missing ops, and that can't copy. */ VM_WARN_ON_ONCE(!ops);
It would be the third time :-D
Quoted 14 lines
Check here to make it absolutely clear you're not running straight into a NULL pointer deref in
__mfill_atomic_pte()? Maybe worth putting a similar assert there too.Quoted 8 lines
mm/userfaultfd.c +0
-0return __mfill_atomic_pte (state, ops); }base-commit:
444fc9435e57157fcf30fc99aee44997f3458641--
2.53.0Thanks, Lorenzo
Signature · 2 lines
Sincerely yours, Mike.
Quoted 39 lines · Mike Rapoport
Quoted 38 lines · Lorenzo Stoakes
Quoted 10 lines · Mike Rapoport
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry()drops the VMA lock forcopy_from_user()and reacquires it afterwards. The destination VMA can be replaced during that window.The existing check compares
vma_uffd_ops()before and after the retry, but if a shmem VMA withMAP_SHAREDis replaced with a shmem VMA withMAP_PRIVATE(or vice versa) the replacement goes undetected.Quoted 6 lines
The change from
MAP_PRIVATEtoMAP_SHAREDwill treat the folio allocated withshmem_alloc_folio()as anonymous and this will causeBUG()whenmfill_atomic_install_pte()will try tofolio_add_new_anon_rmap().The change from
MAP_SHAREDtoMAP_PRIVATEallows injection of folios into the page cache of the original VMA.OK this seems like a sensible subset of things to check - we are looking to check what might materially impact
_thisparticularoperation_, and working to the absolute minimum we need to check.Quoted 8 lines
Introduce helpers for more comprehensive comparison of VMA state:
-vma_snapshot_get()to save the relevant VMA state into a structvma_snapshot(originaluffd_ops, actualuffd_ops, relevant VMA flags,vm_fileand pgoff) before dropping the lock
-vma_snapshot_changed()to compare the saved state with the state of the VMA acquired after retaking the locks
-vma_snapshot_put()to releasevm_filepinning.I feel like this is possibly a little misleading, it sort of implies that this is a broader kind of snapshot rather than some uffd-specific thing.
This is more of a naming thing, yes things being defined in
mm/userfaultfd.ctells you it's uffd-specific, but people are easily confused by stuff like this.
Quoted 6 lines
Quoted 1 line
Will comment inline.
tl;dr:
* will update comments
* will renamevma_uffd_copy_ops()and replace NULL checks withVM_WARN
* will change some of single letter variable names.
Ack, thanks!
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Use
DEFINE_FREE()cleanup to wrapvma_snapshot_put()to avoid complicating error handling paths inmfill_copy_folio_retry().That's nice!
Quoted 1 line
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Add
vma_uffd_copy_ops()to avoid code duplication when original ops of shmem VMA withMAP_PRIVATEare replaced withanon_uffd_ops.I think this is a bit overly brief. And is it only shmem where this might happen? Is
MAP_PRIVATEhugetlbfs not also possible?No, hugetlb is not applicabe here, it was not changed to use
mfill_copy_folio_retry().
OK, maybe worth a mention?
Quoted 41 lines
Quoted 41 lines
Quoted 4 lines
Fixes 292411fda25b ("mm/userfaultfd: detect VMA type change after copy retry in mfill_copy_folio_retry()")Fixes 6ab703034f14 ("userfaultfd: mfill_atomic(): remove retry logic")Hmm this is the second multiple-fixes patch I've seen, maybe that is OK to do :) I guess if this one, indivisble change fixes both then that's valid.
Quoted 25 lines
Tested-by Heechan Kang <[email protected]>Suggested-by Peter Xu <[email protected]>Co-developed-by David Carlier <[email protected]>Signed-off-by David Carlier <[email protected]>Co-developed-by Michael Bommarito <[email protected]>Signed-off-by Michael Bommarito <[email protected]>Signed-off-by Mike Rapoport (Microsoft) <[email protected]>--- mm/userfaultfd.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 20 deletions(-)mm/userfaultfd.c +2
-0@@ -14,6 +14,8 @@#include <linux/userfaultfd_k.h> #include <linux/mmu_notifier.h> #include <linux/hugetlb.h> +#include <linux/file.h> +#include <linux/cleanup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include "internal.h" mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 74
Quoted 4 lines
mm/userfaultfd.c +1
-0@@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)return vma->vm_ops ? vma->vm_ops->uffd_ops :NULL ; } +static const struct vm_uffd_ops *vma_uffd_copy_ops (struct vm_area_struct *vma)I find this a bit confusing, these are the uffd ops for... copy? Or you're copying uffd ops?
Quoted 4 lines
Quoted 2 lines
I'd maybe rename it to
vma_uffd_ops_on_copy()to make it clear which it is, and add a comment like:
vma_uffd_ops_for_copy()?
Works for me!
Quoted 8 lines
Quoted 6 lines
/*
* Determine the ops to use when performing aUFFDIO_COPYoperation - this
* specifically handles the case of aMAP_PRIVATEfile-backed mapping,
* which, upon copy, is CoW'd into anonymous memory.
*/I'll pick the large comment that got dropped and stick it here.
Ack thanks.
Quoted 37 lines
Quoted 37 lines
From the snapshot point of view, I have been pondering if this was the clearest way of checking things because you are, in effect checking a specific case
it, but after much back and forth, perhaps it's not too crazy... you are in most cases doing a duplicate check to the ops check
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 79
Quoted 5 lines
mm/userfaultfd.c +5
-0+{ +const struct vm_uffd_ops *ops =vma_uffd_ops (vma); + +if (!ops) +return NULL ;Is this correct? The code you removed from
mfill_atomic_pte_copy()unconditionally replaces the ops with&anon_uffd_ops, regardless of whether any operations were specified, but now not providing ops avoids that?I mean it's correct if we can never have !ops, and presumably we can't because
__mfill_atomic_pte()unconditionally callsops->alloc_folio()right at the top of the logic?Doesn't uffd only support shmem, anon + hugetlbfs? So why are we adding !ops checks at all?
Actually claude (:>) tells me that async-WP memory mode (ugh god) can be used for _anything_:
/* * If WP is the only mode enabled and context is wp async, allow any * memory type. */ if (wp_async && (vm_flags == VM_UFFD_WP)) return true;
Quoted 3 lines
Quoted 1 line
But. This function is never called in anything but a copy context?
wp_asynchas nothing to do with copy (or zeropage).
Yup, so ops can't be NULL, and all the NULL checks are confusing. Covered elsewhere.
Quoted 6 lines
Quoted 6 lines
I'm not sure if you were protecting against a possible future usage? But in that case surely you'd just want to
VM_WARN_ON_ONCE(!ops)?We should also have a comment describing the situation like:
Quoted 8 lines
Quoted 6 lines
/*
* Only async WP allows arbitrary file-backed mappings for UFFD,
* which is the only case in which ops can be NULL. However,
* we are copying here which is not permitted for these
* mappings, so this will never be the case here.
*/I can live with one sentence tops here ;-)
It took me a _long_ time to understand what you were doing here, there's a balance to be had in commenting, but right now it's tilted towards 'nobody but Mike understands this'.
I can tell, because nobody else bothered to review this in detail ;)
I guess I'll see what the respin looks like.
Quoted 23 lines
Quoted 23 lines
VM_WARN_ON_ONCE(!ops);Also in:
static const struct
vm_uffd_ops*vma_uffd_ops(structvm_area_struct*vma){ if (vma_is_anonymous(vma)) return &anon_uffd_ops; return vma->vm_ops ? vma->vm_ops->uffd_ops : NULL; }This is doing a redundant check _and_ making life confusing, as if !vma->vm_ops is a condition that can be reached there, it can't, as
vma_is_anonymous()is literally a !vma->vm_ops check :)So surely that should be:
static const struct
vm_uffd_ops*vma_uffd_ops(structvm_area_struct*vma){ if (vma_is_anonymous(vma)) return &anon_uffd_ops;
Quoted 7 lines
Quoted 5 lines
return vma->vm_ops->uffd_ops; }I mean obviously that can be a separate patch, but that's not helped make things clear here! :)
You are welcome to send it ;-)
Well you asked for it ;)
Quoted 18 lines
Quoted 18 lines
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 85
Quoted 7 lines
mm/userfaultfd.c +6
-0+ +/* +* UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous +* memory. This is an effective ops override, so retry validation must +* compare the override result, not just vma->vm_ops->uffd_ops. +*/ The wording here is super confusing. 'This is an effective ops override' and 'retry validation' and also the 'not just'.
Also it makes zero reference to the invocation of this function from
mfill_atomic_pte_copy()- this function is pulling double-duty so this comment isn't really quite correct.I'd maybe pull up the comment from
mfill_atomic_pte_copy()(and adapt it to suit the changes, also mentioning the snapshot use case).
Quoted 7 lines
Quoted 4 lines
mm/userfaultfd.c static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) · line 87
Quoted 2 lines
mm/userfaultfd.c +2
-0+if (!(vma->vm_flags &VM_SHARED )) +return &anon_uffd_ops;This is inconsistent with the rest of the VMA flags API usage, you want:
This is consistent with the rest of VMA usage in uffd, we can convert it all in one go if you'd like.
It's new code, inconsistent with literally _the rest of the patch_. I literally wrote the code for you and still you resist? :)
Also your patch makes uffd mix both methods so it feels like you're arguing with yourself here? :P
Quoted 40 lines
Quoted 40 lines
if (!vma_test(vma, VMA_SHARED_BIT)) return &anon_uffd_ops;mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 466
Quoted 12 lines
mm/userfaultfd.c +5
-0+ +return ops; +} +static __always_inline bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) {@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)return ret; } +#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT)Hm it's a bit weird to have a VMA flags define outside of
mm.h.But I definitely appreciate you using the new VMA flags stuff :)
Probably just renaming to
VMA_UFFD_COPY_STATE_FLAGSor something?mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 467
Quoted 1 line
mm/userfaultfd.c +1
-0+I'm a bit of a broken record I know :) but a comment here would be helpful describing what this is for.
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 474
Quoted 7 lines
mm/userfaultfd.c +7
-0+struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +};Yeah I don't love this naming, you're not snapshotting the VMA at all in any real sense, you're comparing a subset of things specifically for the
UFFDIO_COPYcase.
Quoted 6 lines
Quoted 3 lines
Which is fine - but I think we should just make that clear here, because perhaps in the future we'll want a generic version of something like this and this is going to be super confusing if we do...
Let's deal with it in the future then. This is a local structure, used by a few local functions.
I'd really rather we rename it. This is asking for confusion...
Quoted 11 lines
Quoted 10 lines
vma_uffdio_copy_snapshotmaybe ?Maybe ok to leave the functions the same name to avoid a mouthful, the type name should make it clear what's going on.
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 476
Quoted 2 lines
mm/userfaultfd.c +2
-0+ +static void vma_snapshot_get (struct vma_snapshot *s,struct vm_area_struct *vma)No single letter var please! This isn't plan9 :)
Quoted 4 lines
"C is a Spartan language, and your naming conventions should follow suit"
https://docs.kernel.org/process/coding-style.html#namingHere it's perfectly understandable what 's' means.
That very document goes on to give suggestions that aren't single letter :)
I'm not going to die on this hill for a small function like this though, if you are adamant!
Hey - let's not miss the wood for the trees? ;)
Quoted 39 lines
Quoted 31 lines
Quoted 4 lines
mm/userfaultfd.c +3
-0+{ + s->flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); + s->copy_ops =vma_uffd_copy_ops (vma);mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
Quoted 10 lines
mm/userfaultfd.c +9
-0+ s->ops =vma_uffd_ops (vma); + s->pgoff = vma->vm_pgoff; + +if (vma->vm_file) + s->file =get_file (vma->vm_file); +} + +static bool vma_snapshot_changed (struct vma_snapshot *s, +struct vm_area_struct *vma)Can the VMA potentially be any arbitrary VMA that might now be remapped in place? Like anon vs. file and same file and offset?
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 491
Quoted 3 lines
mm/userfaultfd.c +3
-0+{ +vma_flags_t flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); +You have a comment for each of the other cases, so one here is useful I think, e.g.:
/* Have any UFFD flags (missing, WP, minor) changed? */Quoted 2 lines
mm/userfaultfd.c +2
-0+if (!vma_flags_same_pair (&s->flags, &flags)) +return true;Quoted 7 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 496
Quoted 2 lines
mm/userfaultfd.c +2
-0+ +/* VMA type or effective uffd_ops changed while the lock was dropped */ Can the 'effective' (probably better to say copy or use effective throughout)
uffd_opschange though? You already check thatVMA_SHARED_BITis either set or unset across the lock drop above so is thecopy_opscheck even meaningful here?
Quoted 2 lines
Yes, it can change if a shmem VMA changes from SHARED to PRIVATE or vice versa.
How?
if (!vma_flags_same_pair(&s->flags, &flags)) return true;
Checks against VMA_SNAPSHOT_FLAGS which includes VMA_SHARED_BIT. So that would catch it already no?
Quoted 7 lines
Quoted 5 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 498
Quoted 3 lines
mm/userfaultfd.c +2
-0+if (s->ops !=vma_uffd_ops (vma) || s->copy_ops !=vma_uffd_copy_ops (vma)) +return true;Surely we should also check that the UFFD context matches (accounting for NULL cases)?
It's checked in
mfill_get_vma().
Yeah it's a bit strange not to do that on a validation step though, would prefer we live with the duplication or at least add a comment saying it's checked.
Quoted 20 lines
Quoted 7 lines
Could we not otherwise have weird situations across UFFD contexts if both VMAs happen to contain identitical properties otherwise?
This seems to be a canonical part of identifying a UFFD VMA so strange to not see it here?
Quoted 12 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 506
Quoted 9 lines
mm/userfaultfd.c +9
-0+ +/* VMA was anonymous before; changed only if it no longer is */ +if (!s->file) +return !vma_is_anonymous (vma); + +/* VMA was file backed, but inode or offset has changed */ +if (!vma->vm_file || vma->vm_file->f_inode != s->file->f_inode || + vma->vm_pgoff != s->pgoff) +return true;Hm so are we OK with a VMA where the file has been closed, then the same inode re-opened at the exact same offset? (e.g.
vma->vm_filediffers)
Quoted 1 line
The copy will go to exactly the same place in that case.
I mean OK, but maybe it's better to be conserative? Is just a bit weird to tolerate that when we pin the file we could just check file equality?
Quoted 27 lines
Quoted 18 lines
Would this be problematic with hugetlbfs or shmem?
I guess we pin the inode so the
f_inodecheck _should_ be valid, from that side.Quoted 12 lines
mm/userfaultfd.c +12
-0+ +return false; +} + +static void vma_snapshot_put (struct vma_snapshot *s) +{ +if (s->file) +fput (s->file); +} + +DEFINE_FREE(snapshot_put, struct vma_snapshot *, if (_T) vma_snapshot_put(_T)); +Quoted 8 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 523
Quoted 6 lines
mm/userfaultfd.c +2
-1static int mfill_copy_folio_retry (struct mfill_state *state,struct folio *folio) { -const struct vm_uffd_ops *orig_ops =vma_uffd_ops (state->vma); +struct vma_snapshot s = {0 }; + struct vma_snapshot *p __free(snapshot_put) = &s;Could we avoid these single letter var names?
Quoted 1 line
You suggest to call it
a_dummy_pointer_to_allow_use_of_free_cleanup? ;-P
Or, and I know it's _crazy_, but like 'snapshot' and 'prev'(or whatever p is supposed to be - see the problem??), or something vaguely greppable? :)
Quoted 48 lines
Quoted 31 lines
Quoted 11 lines
mm/userfaultfd.c +2
-0unsigned long src_addr = state->src_addr;void *kaddr;int err; +vma_snapshot_get (&s, state->vma); +/* retry copying with mm_lock dropped */ mfill_put_vma (state);mm/userfaultfd.c static int mfill_copy_folio_retry(struct mfill_state *state, · line 474
Quoted 8 lines
mm/userfaultfd.c +0
-5@@ -467,12 +543,7 @@ static int mfill_copy_folio_retry(struct mfill_state *state,if (err)return err; -/* -* The VMA type may have changed while the lock was dropped -* (e.g. replaced with a hugetlb mapping), making the caller's -* ops pointer stale. -*/ This comment we can lose given we are explicitly comparing snapshots, but I think it's not great to explicitly point out that we're doing this because of the lock being dropped, we should say that somewhere.
Quoted 7 lines
mm/userfaultfd.c +1
-1-if (vma_uffd_ops (state->vma) != orig_ops) +if (vma_snapshot_changed (&s, state->vma))return -EAGAIN ; err =mfill_establish_pmd (state);@@ -545,19 +616,7 @@ static int __mfill_atomic_pte(struct mfill_state *state,Quoted 16 lines
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 558
Quoted 13 lines
mm/userfaultfd.c +0
-11static int mfill_atomic_pte_copy (struct mfill_state *state) { -const struct vm_uffd_ops *ops =vma_uffd_ops (state->vma); - -/* -* The normal page fault path for a MAP_PRIVATE mapping in a -* file-backed VMA will invoke the fault, fill the hole in the file and -* COW it right away. The result generates plain anonymous memory. -* So when we are asked to fill a hole in a MAP_PRIVATE mapping, we'll -* generate anonymous memory directly without actually filling the -* hole. For the MAP_PRIVATE case the robustness check only happens in -* the pagetable (to verify it's still none) and not in the page cache. -*/ (As mentioned above) I hate that we lose this comment (or the gist of it at least), can we please move it up to the copy ops function?
Quoted 1 line
Will move it before the
copy_ops()
Thanks
Quoted 7 lines
Quoted 7 lines
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 560
Quoted 3 lines
mm/userfaultfd.c +0
-2-if (!(state->vma->vm_flags &VM_SHARED )) - ops = &anon_uffd_ops;A comment here would maybe be useful. But with a function rename and a comment at the function itself maybe OK without.
Quoted 8 lines
Quoted 6 lines
mm/userfaultfd.c static int __mfill_atomic_pte(struct mfill_state *state, · line 619
Quoted 1 line
mm/userfaultfd.c +1
-0+const struct vm_uffd_ops *ops =vma_uffd_copy_ops (state->vma);Can we please have a:
/* Only async W/P can have missing ops, and that can't copy. */ VM_WARN_ON_ONCE(!ops);It would be the third time :-D
<insert old addage about repetition etc. etc.>
Quoted 14 lines
Quoted 14 lines
Check here to make it absolutely clear you're not running straight into a NULL pointer deref in
__mfill_atomic_pte()? Maybe worth putting a similar assert there too.Quoted 8 lines
mm/userfaultfd.c +0
-0return __mfill_atomic_pte (state, ops); }base-commit:
444fc9435e57157fcf30fc99aee44997f3458641--
2.53.0
Quoted 5 lines
Quoted 1 line
Thanks, Lorenzo
--
Sincerely yours,
Mike.
Cheers, Lorenzo
Mike Rapoport
maintainerQuoted 60 lines · Lorenzo Stoakes
Quoted 39 lines · Mike Rapoport
Quoted 38 lines · Lorenzo Stoakes
Quoted 10 lines · Mike Rapoport
From: "Mike Rapoport (Microsoft)" <[email protected]>
mfill_copy_folio_retry()drops the VMA lock forcopy_from_user()and reacquires it afterwards. The destination VMA can be replaced during that window.The existing check compares
vma_uffd_ops()before and after the retry, but if a shmem VMA withMAP_SHAREDis replaced with a shmem VMA withMAP_PRIVATE(or vice versa) the replacement goes undetected.Quoted 6 lines
The change from
MAP_PRIVATEtoMAP_SHAREDwill treat the folio allocated withshmem_alloc_folio()as anonymous and this will causeBUG()whenmfill_atomic_install_pte()will try tofolio_add_new_anon_rmap().The change from
MAP_SHAREDtoMAP_PRIVATEallows injection of folios into the page cache of the original VMA.OK this seems like a sensible subset of things to check - we are looking to check what might materially impact
_thisparticularoperation_, and working to the absolute minimum we need to check.Quoted 8 lines
Introduce helpers for more comprehensive comparison of VMA state:
-vma_snapshot_get()to save the relevant VMA state into a structvma_snapshot(originaluffd_ops, actualuffd_ops, relevant VMA flags,vm_fileand pgoff) before dropping the lock
-vma_snapshot_changed()to compare the saved state with the state of the VMA acquired after retaking the locks
-vma_snapshot_put()to releasevm_filepinning.I feel like this is possibly a little misleading, it sort of implies that this is a broader kind of snapshot rather than some uffd-specific thing.
This is more of a naming thing, yes things being defined in
mm/userfaultfd.ctells you it's uffd-specific, but people are easily confused by stuff like this.Quoted 6 lines
Quoted 1 line
Will comment inline.
tl;dr:
* will update comments
* will renamevma_uffd_copy_ops()and replace NULL checks withVM_WARN
* will change some of single letter variable names.Ack, thanks!
Quoted 11 lines
Quoted 11 lines
Quoted 4 lines
Use
DEFINE_FREE()cleanup to wrapvma_snapshot_put()to avoid complicating error handling paths inmfill_copy_folio_retry().That's nice!
Quoted 3 lines
Add
vma_uffd_copy_ops()to avoid code duplication when original ops of shmem VMA withMAP_PRIVATEare replaced withanon_uffd_ops.
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
I think this is a bit overly brief. And is it only shmem where this might happen? Is
MAP_PRIVATEhugetlbfs not also possible?No, hugetlb is not applicabe here, it was not changed to use
mfill_copy_folio_retry().OK, maybe worth a mention?
Ok.
Quoted 11 lines
Quoted 9 lines
Quoted 7 lines
Quoted 1 line
+static const struct
vm_uffd_ops*vma_uffd_copy_ops(structvm_area_struct*vma)I find this a bit confusing, these are the uffd ops for... copy? Or you're copying uffd ops?
I'd maybe rename it to
vma_uffd_ops_on_copy()to make it clear which it is, and add a comment like:
vma_uffd_ops_for_copy()?Works for me!
It's not needed actually, so I dropped it completely.
Quoted 9 lines
Quoted 7 lines
Quoted 5 lines
/*
* Determine the ops to use when performing aUFFDIO_COPYoperation - this
* specifically handles the case of aMAP_PRIVATEfile-backed mapping,
* which, upon copy, is CoW'd into anonymous memory.
*/I'll pick the large comment that got dropped and stick it here.
Ack thanks.
And that remains in place.
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
I'm not sure if you were protecting against a possible future usage? But in that case surely you'd just want to
VM_WARN_ON_ONCE(!ops)?We should also have a comment describing the situation like:
/*
* Only async WP allows arbitrary file-backed mappings for UFFD,
* which is the only case in which ops can be NULL. However,
* we are copying here which is not permitted for these
* mappings, so this will never be the case here.
*/
Quoted 5 lines
Quoted 1 line
I can live with one sentence tops here ;-)
It took me a _long_ time to understand what you were doing here, there's a balance to be had in commenting, but right now it's tilted towards 'nobody but Mike understands this'.
That's job security isn't it? ;-P
Quoted 1 line
I can tell, because nobody else bothered to review this in detail ;)
Were you on strike? ;-)
Quoted 23 lines
Quoted 23 lines
Quoted 23 lines
Also in:
static const struct
vm_uffd_ops*vma_uffd_ops(structvm_area_struct*vma){ if (vma_is_anonymous(vma)) return &anon_uffd_ops; return vma->vm_ops ? vma->vm_ops->uffd_ops : NULL; }This is doing a redundant check _and_ making life confusing, as if !vma->vm_ops is a condition that can be reached there, it can't, as
vma_is_anonymous()is literally a !vma->vm_ops check :)So surely that should be:
static const struct
vm_uffd_ops*vma_uffd_ops(structvm_area_struct*vma){ if (vma_is_anonymous(vma)) return &anon_uffd_ops; return vma->vm_ops->uffd_ops; }
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
I mean obviously that can be a separate patch, but that's not helped make things clear here! :)
You are welcome to send it ;-)
Well you asked for it ;)
Did it myself.
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 474
Quoted 7 lines
mm/userfaultfd.c +7
-0+struct vma_snapshot { +const struct vm_uffd_ops *copy_ops; +const struct vm_uffd_ops *ops; +struct file *file; +vma_flags_t flags; +pgoff_t pgoff; +};Yeah I don't love this naming, you're not snapshotting the VMA at all in any real sense, you're comparing a subset of things specifically for the
UFFDIO_COPYcase.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Which is fine - but I think we should just make that clear here, because perhaps in the future we'll want a generic version of something like this and this is going to be super confusing if we do...
Let's deal with it in the future then. This is a local structure, used by a few local functions.
I'd really rather we rename it. This is asking for confusion...
Renamed everything to mfill_retry_state.
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
vma_uffdio_copy_snapshotmaybe ?Maybe ok to leave the functions the same name to avoid a mouthful, the type name should make it clear what's going on.
Review
Quoted 2 lines
(quoted diff) +2
-0+ +static void vma_snapshot_get(struct vma_snapshot *s, struct vm_area_struct *vma)No single letter var please! This isn't plan9 :)
Quoted 6 lines
Quoted 4 lines
"C is a Spartan language, and your naming conventions should follow suit"
https://docs.kernel.org/process/coding-style.html#namingHere it's perfectly understandable what 's' means.
That very document goes on to give suggestions that aren't single letter :)
There's 'i' there a paragraph below ;)
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
Quoted 3 lines
mm/userfaultfd.c +2
-0+static bool vma_snapshot_changed (struct vma_snapshot *s, +struct vm_area_struct *vma)Can the VMA potentially be any arbitrary VMA that might now be remapped in place? Like anon vs. file and same file and offset?
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 491
Quoted 3 lines
mm/userfaultfd.c +3
-0+{ +vma_flags_t flags =vma_flags_and_mask (&vma->flags,VMA_SNAPSHOT_FLAGS ); +You have a comment for each of the other cases, so one here is useful I think, e.g.:
/* Have any UFFD flags (missing, WP, minor) changed? */
Added it, although the flags are just one line above.
Quoted 15 lines
Quoted 12 lines
Quoted 11 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
Quoted 6 lines
mm/userfaultfd.c +4
-0+if (!vma_flags_same_pair (&s->flags, &flags)) +return true; + +/* VMA type or effective uffd_ops changed while the lock was dropped */ Can the 'effective' (probably better to say copy or use effective throughout)
uffd_opschange though? You already check thatVMA_SHARED_BITis either set or unset across the lock drop above so is thecopy_opscheck even meaningful here?Quoted 2 lines
Yes, it can change if a shmem VMA changes from SHARED to PRIVATE or vice versa.
Quoted 7 lines
How?
if (!vma_flags_same_pair(&s->flags, &flags)) return true;Checks against
VMA_SNAPSHOT_FLAGSwhich includesVMA_SHARED_BIT. So that would catch it already no?
Yes, you are right. It's enough to check for flags and the original ops. And with the copy_ops check dropped there is no need in vma_uffd_copy_ops().
Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ if (s->ops != vma_uffd_ops(vma) || s->copy_ops != vma_uffd_copy_ops(vma)) + return true;Surely we should also check that the UFFD context matches (accounting for NULL cases)?
It's checked in
mfill_get_vma().Yeah it's a bit strange not to do that on a validation step though, would prefer we live with the duplication or at least add a comment saying it's checked.
There's a bunch of validation checks on mfill_get_vma() path, let's leave them there.
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 506
Quoted 4 lines
mm/userfaultfd.c +4
-0+/* VMA was file backed, but inode or offset has changed */ +if (!vma->vm_file || vma->vm_file->f_inode != s->file->f_inode || + vma->vm_pgoff != s->pgoff) +return true;Hm so are we OK with a VMA where the file has been closed, then the same inode re-opened at the exact same offset? (e.g.
vma->vm_filediffers)
Quoted 4 lines
Quoted 1 line
The copy will go to exactly the same place in that case.
I mean OK, but maybe it's better to be conserative? Is just a bit weird to tolerate that when we pin the file we could just check file equality?
Yes, we could :)
Quoted 9 lines
Quoted 9 lines
Quoted 8 lines
mm/userfaultfd.c static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) · line 523
Quoted 6 lines
static int mfill_copy_folio_retry(struct
mfill_state*state,mm/userfaultfd.c +2
-1struct folio *folio) { -const struct vm_uffd_ops *orig_ops =vma_uffd_ops (state->vma); +struct vma_snapshot s = {0 }; + struct vma_snapshot *p __free(snapshot_put) = &s;Could we avoid these single letter var names?
Quoted 4 lines
Quoted 1 line
You suggest to call it
a_dummy_pointer_to_allow_use_of_free_cleanup? ;-POr, and I know it's _crazy_, but like 'snapshot' and 'prev'(or whatever p is supposed to be - see the problem??), or something vaguely greppable? :)
p is pointer, prev has nothing to do with it.
But renamed anyway.
Quoted 1 line
Cheers, Lorenzo
Signature · 2 lines
Sincerely yours, Mike.