mm: softdirty: write protect PTEs created for read faults after VM_SOFTDIRTY cleared
Subsystems (5)
PATCH 1/1 mm: softdirty: write protect PTEs created for read faults after VM_SOFTDIRTY cleared
In readable+writable+shared VMAs, PTEs created for read faults have their write bit set. If the read fault happens after VM_SOFTDIRTY is cleared, then the PTE's softdirty bit will remain clear after subsequent writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, new PTEs created for read faults are write protected if the VMA has VM_SOFTDIRTY clear.
--- mm/memory.c | 2 ++ 1 file changed, 2 insertions(+)
mm/memory.c +2 -0
@@ -2755,6 +2755,8 @@ void do_set_pte(struct vm_area_struct *vma, unsigned long address, entry =maybe_mkwrite (pte_mkdirty (entry), vma); else if (pte_file(*pte) && pte_file_soft_dirty(*pte)) entry =pte_mksoft_dirty (entry); + else if (!(vma->vm_flags & VM_SOFTDIRTY)) + entry =pte_wrprotect (entry);if (anon) {inc_mm_counter_fast (vma->vm_mm,MM_ANONPAGES );page_add_new_anon_rmap (page, vma, address);
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 28 lines · Peter Feiner
In readable+writable+shared VMAs, PTEs created for read faults have their write bit set. If the read fault happens after
VM_SOFTDIRTYis cleared, then the PTE's softdirty bit will remain clear after subsequent writes.Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL,
getpagesize(),PROT_READ|PROT_WRITE,MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clearVM_SOFTDIRTY*/
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */With this patch, new PTEs created for read faults are write protected if the VMA has
VM_SOFTDIRTYclear.Signed-off-by Peter Feiner <[email protected]>--- mm/memory.c | 2 ++ 1 file changed, 2 insertions(+)mm/memory.c +0
-0
mm/memory.c void do_set_pte(struct vm_area_struct *vma, unsigned long address, · line 2759
Quoted 5 lines
mm/memory.c +2
-0@@ -2755,6 +2755,8 @@ void do_set_pte(struct vm_area_struct *vma, unsigned long address, entry =maybe_mkwrite (pte_mkdirty (entry), vma); else if (pte_file(*pte) && pte_file_soft_dirty(*pte)) entry =pte_mksoft_dirty (entry); + else if (!(vma->vm_flags & VM_SOFTDIRTY)) + entry =pte_wrprotect (entry);
It basically means VM_SOFTDIRTY require writenotify on the vma.
What about patch below? Untested. And it seems it'll introduce bug similar to bug fixed by c9d0bf241451, *but* IIUC we have it already in mprotect() code path.
I'll look more careful tomorrow.
fs/proc/task_mmu.c +3 -2
@@ -851,8 +851,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,if (type ==CLEAR_REFS_MAPPED && !vma->vm_file)continue ;if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; + vma->vm_flags &= ~VM_SOFTDIRTY ; + vma->vm_page_prot =vm_get_page_prot ( + vma->vm_flags & ~VM_SHARED ); }walk_page_range (vma->vm_start, vma->vm_end, &clear_refs_walk);
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 29 lines · Kirill A. Shutemov
On Wed, Aug 20, 2014 at 05:46:22PM -0400, Peter Feiner wrote:
It basically meansVM_SOFTDIRTYrequire writenotify on the vma.What about patch below? Untested. And it seems it'll introduce bug similar to bug fixed by
c9d0bf241451, *but* IIUC we have it already inmprotect()code path.I'll look more careful tomorrow.
Not-signed-off-by Kirill A. Shutemov <[email protected]>fs/proc/task_mmu.c +3
-2@@ -851,8 +851,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,if (type ==CLEAR_REFS_MAPPED && !vma->vm_file)continue ;if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; + vma->vm_flags &= ~VM_SOFTDIRTY ; + vma->vm_page_prot =vm_get_page_prot ( + vma->vm_flags & ~VM_SHARED ); }walk_page_range (vma->vm_start, vma->vm_end, &clear_refs_walk);Signature · 1 line
Kirill A. Shutemov
Thanks Kirill, I prefer your approach. I'll send a v2.
I believe you're right about c9d0bf241451. It seems like passing the old & new pgprot through pgprot_modify would handle the problem. Furthermore, as you suggest, mprotect_fixup should use pgprot_modify when it turns write notification on. I think a patch like this is in order:
mm/mmap.c +3 -6
@@ -1611,18 +1611,15 @@ munmap_back: }if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; -/* Can vma->vm_page_prot have changed?? * * Answer: Yes, drivers may have changed it in their * f_op->mmap method. * -* Ensures that vmas marked as uncached stay that way. +* Ensures that vmas marked with special bits stay that way. */ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, + vm_get_page_prot(vm_flags & ~VM_SHARED); }vma_link (mm, vma, prev, rb_link, rb_parent);
mm/mprotect.c +2 -1
@@ -324,7 +324,8 @@ success: vm_get_page_prot(newflags));if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (newflags & ~VM_SHARED )); dirty_accountable =1 ; }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 41 lines · Peter Feiner
Thanks Kirill, I prefer your approach. I'll send a v2.
I believe you're right about
c9d0bf241451. It seems like passing the old & new pgprot throughpgprot_modifywould handle the problem. Furthermore, as you suggest,mprotect_fixupshould usepgprot_modifywhen it turns write notification on. I think a patch like this is in order:Not-signed-off-by Peter Feiner <[email protected]>mm/mmap.c +3
-6@@ -1611,18 +1611,15 @@ munmap_back: }if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; -/* Can vma->vm_page_prot have changed?? * * Answer: Yes, drivers may have changed it in their * f_op->mmap method. * -* Ensures that vmas marked as uncached stay that way. +* Ensures that vmas marked with special bits stay that way. */ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, + vm_get_page_prot(vm_flags & ~VM_SHARED); }vma_link (mm, vma, prev, rb_link, rb_parent);mm/mprotect.c +0
-0
mm/mprotect.c success: · line 330
Quoted 8 lines
mm/mprotect.c +2
-1@@ -324,7 +324,8 @@ success: vm_get_page_prot(newflags));if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (newflags & ~VM_SHARED )); dirty_accountable =1 ; }
Thanks a lot Peter and Kirill for catching it and providing the prelim. fixup. (Initial patch doesn't look that right for me because vm-softdirty should involve into account for newly created/expaned vmas only but not into some deep code such as fault handlings). Peter does the patch above helps? (out of testing machine at the moment so cant test myself).
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 8 lines · Cyrill Gorcunov
Quoted 7 lines · Peter Feiner
Thanks Kirill, I prefer your approach. I'll send a v2.
I believe you're right about
c9d0bf241451. It seems like passing the old & new pgprot throughpgprot_modifywould handle the problem. Furthermore, as you suggest,mprotect_fixupshould usepgprot_modifywhen it turns write notification on. I think a patch like this is in order:
Looks good to me.
Would you mind to apply the same pgprot_modify() approach on the clear_refs_write(), test and post the patch?
Feel free to use my singed-off-by (or suggested-by if you prefer) once it's tested (see merge case below).
Quoted 35 lines
Quoted 35 lines
Not-signed-off-by Peter Feiner <[email protected]>mm/mmap.c +3
-6@@ -1611,18 +1611,15 @@ munmap_back: }if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; -/* Can vma->vm_page_prot have changed?? * * Answer: Yes, drivers may have changed it in their * f_op->mmap method. * -* Ensures that vmas marked as uncached stay that way. +* Ensures that vmas marked with special bits stay that way. */ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, + vm_get_page_prot(vm_flags & ~VM_SHARED); }vma_link (mm, vma, prev, rb_link, rb_parent);mm/mprotect.c +0
-0@@ -324,7 +324,8 @@ success: vm_get_page_prot(newflags));
Quoted 12 lines
mm/mprotect.c success: · line 330
Quoted 6 lines
mm/mprotect.c +2
-1if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (newflags & ~VM_SHARED )); dirty_accountable =1 ; }Thanks a lot Peter and Kirill for catching it and providing the prelim. fixup. (Initial patch doesn't look that right for me because vm-softdirty should involve into account for newly created/expaned vmas only but not into some deep code such as fault handlings). Peter does the patch above helps? (out of testing machine at the moment so cant test myself).
One thing: there could be (I haven't checked) complications on vma_merge(): since vm_flags are identical it assumes that it can reuse vma->vm_page_prot of expanded vma. But VM_SOFTDIRTY is excluded from vm_flags compatibility check. What should we do with vm_page_prot there?
Signature · 1 line
Kirill A. Shutemov
Quoted 9 lines · Kirill A. Shutemov
On Fri, Aug 22, 2014 at 12:51:15AM +0400, Cyrill Gorcunov wrote:
Looks good to me.
Would you mind to apply the same
pgprot_modify()approach on theclear_refs_write(), test and post the patch?Feel free to use my singed-off-by (or suggested-by if you prefer) once it's tested (see merge case below).
Sure thing :-)
Quoted 4 lines
One thing: there could be (I haven't checked) complications on
vma_merge(): sincevm_flagsare identical it assumes that it can reusevma->vm_page_protof expanded vma. ButVM_SOFTDIRTYis excluded fromvm_flagscompatibility check. What should we do withvm_page_protthere?
Since the merged VMA will have VM_SOFTDIRTY set, it's OK that it's vm_page_prot won't be setup for write notifications. For the purpose of process migration, you'll just get some false positives, which is tolerable.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 13 lines · Peter Feiner
Quoted 9 lines · Kirill A. Shutemov
On Fri, Aug 22, 2014 at 12:51:15AM +0400, Cyrill Gorcunov wrote:
Looks good to me.
Would you mind to apply the same
pgprot_modify()approach on theclear_refs_write(), test and post the patch?Feel free to use my singed-off-by (or suggested-by if you prefer) once it's tested (see merge case below).
Sure thing :-)
Quoted 8 lines
Quoted 4 lines
One thing: there could be (I haven't checked) complications on
vma_merge(): sincevm_flagsare identical it assumes that it can reusevma->vm_page_protof expanded vma. ButVM_SOFTDIRTYis excluded fromvm_flagscompatibility check. What should we do withvm_page_protthere?Since the merged VMA will have
VM_SOFTDIRTYset, it's OK that it'svm_page_protwon't be setup for write notifications. For the purpose of process migration, you'll just get some false positives, which is tolerable.
Right. But should we disable writenotify back to avoid exessive wp-faults if it was enabled due to soft-dirty (the case when expanded vma is soft-dirty)?
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 5 lines · Kirill A. Shutemov
Quoted 5 lines
Quoted 4 lines
One thing: there could be (I haven't checked) complications on
vma_merge(): sincevm_flagsare identical it assumes that it can reusevma->vm_page_protof expanded vma. ButVM_SOFTDIRTYis excluded fromvm_flagscompatibility check. What should we do withvm_page_protthere?
Quoted 7 lines
Quoted 3 lines
Since the merged VMA will have
VM_SOFTDIRTYset, it's OK that it'svm_page_protwon't be setup for write notifications. For the purpose of process migration, you'll just get some false positives, which is tolerable.Right. But should we disable writenotify back to avoid exessive wp-faults if it was enabled due to soft-dirty (the case when expanded vma is soft-dirty)?
Ah, I understand now. I've got a patch in the works that disables the write faults when a VMA is merged. I'll send a series with all of the changes tomorrow.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 10 lines · Peter Feiner
Quoted 9 lines · Kirill A. Shutemov
Quoted 8 lines
Quoted 4 lines
One thing: there could be (I haven't checked) complications on
vma_merge(): sincevm_flagsare identical it assumes that it can reusevma->vm_page_protof expanded vma. ButVM_SOFTDIRTYis excluded fromvm_flagscompatibility check. What should we do withvm_page_protthere?Since the merged VMA will have
VM_SOFTDIRTYset, it's OK that it'svm_page_protwon't be setup for write notifications. For the purpose of process migration, you'll just get some false positives, which is tolerable.
Quoted 7 lines
Quoted 3 lines
Right. But should we disable writenotify back to avoid exessive wp-faults if it was enabled due to soft-dirty (the case when expanded vma is soft-dirty)?
Ah, I understand now. I've got a patch in the works that disables the write faults when a VMA is merged. I'll send a series with all of the changes tomorrow.
Cool! Thanks a lot, guys!
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
v2 cover softdirty fix and write notification cleanup
In reply to Peter FeinerHere's the new patch that uses Kirill's approach of setting write notifications on the VMA. I also included write notification cleanups and fixes per our discussion.
Peter Feiner (3):
mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY
cleared
mm: mprotect: preserve special page protection bits
mm: mmap: cleanup code that preserves special vm_page_prot bits
fs/proc/task_mmu.c | 17 ++++++++++++++++- include/linux/mm.h | 15 +++++++++++++++ mm/mmap.c | 26 +++++++++++--------------- mm/mprotect.c | 2 +- 4 files changed, 43 insertions(+), 17 deletions(-)
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v2 1/3 mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
For VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after VM_SOFTDIRTY is cleared, then the PTE's softdirty bit will remain clear after subsequent writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is cleared. Furthermore, to avoid faults, write notifications are disabled when VM_SOFTDIRTY is reset.
---
v1 -> v2: Instead of checking VM_SOFTDIRTY in the fault handler, enable write
notifications on vm_page_prot when we clear VM_SOFTDIRTY.
fs/proc/task_mmu.c | 17 ++++++++++++++++- include/linux/mm.h | 15 +++++++++++++++ mm/mmap.c | 10 +++++++++- 3 files changed, 40 insertions(+), 2 deletions(-)
fs/proc/task_mmu.c +16 -1
@@ -851,8 +851,23 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,if (type ==CLEAR_REFS_MAPPED && !vma->vm_file)continue ;if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) +if (vma->vm_flags &VM_SOFTDIRTY ) { vma->vm_flags &= ~VM_SOFTDIRTY ; +/* +* We don't have a write lock on +* mm->mmap_sem, so we race with the +* fault handler reading vm_page_prot. +* Therefore writable PTEs (that won't +* have soft-dirty set) can be created +* for read faults. However, since the +* PTE lock is held while vm_page_prot +* is read and while we write protect +* PTEs during our walk, any writable +* PTEs that slipped through will be +* write protected. +*/ +vma_enable_writenotify (vma); + } }walk_page_range (vma->vm_start, vma->vm_end, &clear_refs_walk);
include/linux/mm.h +15 -0
@@ -1946,6 +1946,21 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif +/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{ + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (vma->vm_flags & + ~VM_SHARED )); +} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma) +{ + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (vma->vm_flags)); +} +#ifdef CONFIG_NUMA_BALANCING unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end);
mm/mmap.c +9 -1
@@ -1549,8 +1549,16 @@ munmap_back:* Can we just expand an old mapping? */ vma =vma_merge (mm, prev, addr, addr + len, vm_flags,NULL , file, pgoff,NULL ); -if (vma) +if (vma) { +if (!vma_wants_writenotify (vma)) { +/* +* We're going to reset VM_SOFTDIRTY, so we can disable +* write notifications. +*/ +vma_disable_writenotify (vma); + } goto out; + }/* * Determine the object being mapped and call the appropriate
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v2 1/3] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 34 lines · Peter Feiner
For VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after
VM_SOFTDIRTYis cleared, then the PTE's softdirty bit will remain clear after subsequent writes.Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL,
getpagesize(),PROT_READ|PROT_WRITE,MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clearVM_SOFTDIRTY*/
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */With this patch, write notifications are enabled when
VM_SOFTDIRTYis cleared. Furthermore, to avoid faults, write notifications are disabled whenVM_SOFTDIRTYis reset.Signed-off-by Peter Feiner <[email protected]>---
v1 -> v2: Instead of checkingVM_SOFTDIRTYin the fault handler, enable write
notifications onvm_page_protwhen we clearVM_SOFTDIRTY.fs/proc/task_mmu.c | 17 ++++++++++++++++- include/linux/mm.h | 15 +++++++++++++++ mm/mmap.c | 10 +++++++++- 3 files changed, 40 insertions(+), 2 deletions(-)fs/proc/task_mmu.c +0
-0
fs/proc/task_mmu.c static ssize_t clear_refs_write(struct file *file, const char __user *buf, · line 854
Quoted 5 lines
fs/proc/task_mmu.c +1
-1@@ -851,8 +851,23 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,if (type ==CLEAR_REFS_MAPPED && !vma->vm_file)continue ;if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) +if (vma->vm_flags &VM_SOFTDIRTY ) {
Why do we need the branch here. Does it save us anything? Looks like we can update vm_flags and enable writenotify unconditionally. Indentation level is high enough already.
Quoted 4 lines
fs/proc/task_mmu.c +3
-0vma->vm_flags &= ~VM_SOFTDIRTY ; +/* +* We don't have a write lock on +* mm->mmap_sem, so we race with the
fs/proc/task_mmu.c static ssize_t clear_refs_write(struct file *file, const char __user *buf, · line 868
Quoted 10 lines
fs/proc/task_mmu.c +10
-0+* fault handler reading vm_page_prot. +* Therefore writable PTEs (that won't +* have soft-dirty set) can be created +* for read faults. However, since the +* PTE lock is held while vm_page_prot +* is read and while we write protect +* PTEs during our walk, any writable +* PTEs that slipped through will be +* write protected. +*/
Hm.. Isn't this yet another bug?
Updating vma->vm_flags without down_write(&mm->mmap_sem) looks troublesome to me. Am I wrong?
Quoted 10 lines
fs/proc/task_mmu.c +2
-0+vma_enable_writenotify (vma); + } }walk_page_range (vma->vm_start, vma->vm_end, &clear_refs_walk);include/linux/mm.h +0
-0
include/linux/mm.h static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) · line 1954
Quoted 9 lines
include/linux/mm.h +6
-0@@ -1946,6 +1946,21 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif +/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{ + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (vma->vm_flags & + ~VM_SHARED ));
I think this way is more readable:
pgprot_t newprot; newprot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED); vma->vm_page_prot = pgprot_modify(vma->vm_page_prot, newprot);
include/linux/mm.h static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) · line 1961
Quoted 7 lines
include/linux/mm.h +7
-0+} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma) +{ + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (vma->vm_flags));
ditto.
Quoted 30 lines
include/linux/mm.h +2
-0+} +#ifdef CONFIG_NUMA_BALANCING unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end);mm/mmap.c +9
-1@@ -1549,8 +1549,16 @@ munmap_back:* Can we just expand an old mapping? */ vma =vma_merge (mm, prev, addr, addr + len, vm_flags,NULL , file, pgoff,NULL ); -if (vma) +if (vma) { +if (!vma_wants_writenotify (vma)) { +/* +* We're going to reset VM_SOFTDIRTY, so we can disable +* write notifications. +*/ +vma_disable_writenotify (vma); + } goto out; + }/* * Determine the object being mapped and call the appropriate Signature · 1 line
2.1.0.rc2.206.gedb03e5
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v2 1/3] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 15 lines · Kirill A. Shutemov
fs/proc/task_mmu.c static ssize_t clear_refs_write(struct file *file, const char __user *buf, · line 854
Quoted 10 lines · Peter Feiner
fs/proc/task_mmu.c +1
-1@@ -851,8 +851,23 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,if (type ==CLEAR_REFS_MAPPED && !vma->vm_file)continue ;if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) +if (vma->vm_flags &VM_SOFTDIRTY ) {Why do we need the branch here. Does it save us anything? Looks like we can update
vm_flagsand enable writenotify unconditionally. Indentation level is high enough already.
You're right, we don't need the branch here. I'll change for v3.
Quoted 10 lines
Quoted 10 lines
fs/proc/task_mmu.c +9
-0vma->vm_flags &= ~VM_SOFTDIRTY ; +/* +* We don't have a write lock on +* mm->mmap_sem, so we race with the +* fault handler reading vm_page_prot. +* Therefore writable PTEs (that won't +* have soft-dirty set) can be created +* for read faults. However, since the +* PTE lock is held while vm_page_prot +* is read and while we write protect
Quoted 8 lines
fs/proc/task_mmu.c static ssize_t clear_refs_write(struct file *file, const char __user *buf, · line 868
Quoted 4 lines
fs/proc/task_mmu.c +4
-0+* PTEs during our walk, any writable +* PTEs that slipped through will be +* write protected. +*/ Hm.. Isn't this yet another bug?
Updatingvma->vm_flagswithoutdown_write(&mm->mmap_sem)looks troublesome to me. Am I wrong?
As I said in the comment, it looks fishy but we're still fixing the bug. That is, no writable PTEs will sneak by that don't have soft-dirty set.
I was originally going to submit something that dropped the mmap_sem and re-took it in write mode before manipulating vm_page_prot. The control flow was slightly hairy, so I convinced myself that the race is benign :-)
If I'm right and the race is benign, it still might be worth having the more straightforward & obviously correct implementation since this isn't performance critical code.
Quoted 9 lines
fs/proc/task_mmu.c static ssize_t clear_refs_write(struct file *file, const char __user *buf,
Quoted 6 lines
fs/proc/task_mmu.c +6
-0+/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{ + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (vma->vm_flags & + ~VM_SHARED ));I think this way is more readable:
Quoted 4 lines
pgprot_t newprot; newprot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED); vma->vm_page_prot = pgprot_modify(vma->vm_page_prot, newprot);
Looks good. I'll update.
Quoted 9 lines
fs/proc/task_mmu.c
Quoted 7 lines
fs/proc/task_mmu.c +7
-0+} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma) +{ + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, +vm_get_page_prot (vma->vm_flags));ditto.
I'll change this too.
Peter
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v2 1/3] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
In reply to Kirill A. ShutemovQuoted 16 lines · Kirill A. Shutemov
Quoted 15 lines · Peter Feiner
For VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after
VM_SOFTDIRTYis cleared, then the PTE's softdirty bit will remain clear after subsequent writes.Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL,
getpagesize(),PROT_READ|PROT_WRITE,MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clearVM_SOFTDIRTY*/
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
Quoted 5 lines
Quoted 5 lines
With this patch, write notifications are enabled when
VM_SOFTDIRTYis cleared. Furthermore, to avoid faults, write notifications are disabled whenVM_SOFTDIRTYis reset.Signed-off-by Peter Feiner <[email protected]>
One more case to consider: mprotect() which doesn't trigger successful vma_merge() will not set VM_SOFTDIRTY and will not enable write-protect on the vma.
It's probably better to take VM_SOFTDIRTY into account in vma_wants_writenotify() and re-think logic in other corners.
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v2 1/3] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 6 lines · Kirill A. Shutemov
One more case to consider:
mprotect()which doesn't trigger successfulvma_merge()will not setVM_SOFTDIRTYand will not enable write-protect on the vma.It's probably better to take
VM_SOFTDIRTYinto account invma_wants_writenotify()and re-think logic in other corners.
Merge or not, write notifications get disabled! I'll fix this too :-)
Peter
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v2 2/3 mm: mprotect: preserve special page protection bits
In reply to Peter FeinerWe don't want to zap special page protection bits on mprotect. Analogous to the bug fixed in c9d0bf241451a3ab7d02e1652c22b80cd7d93e8f where vm_page_prot bits set by drivers were zapped when write notifications were enabled on new VMAs.
--- mm/mprotect.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mm/mprotect.c +1 -1
@@ -324,7 +324,7 @@ success: vm_get_page_prot(newflags));if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); +vma_enable_writenotify (vma); dirty_accountable =1 ; }
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v2 3/3 mm: mmap: cleanup code that preserves special vm_page_prot bits
In reply to Peter FeinerReplace logic that has been factored out into a utility method.
--- mm/mmap.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-)
mm/mmap.c +2 -14
@@ -1618,20 +1618,8 @@ munmap_back: goto free_vma; } -if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; - -/* Can vma->vm_page_prot have changed?? -* -* Answer: Yes, drivers may have changed it in their -* f_op->mmap method. -* -* Ensures that vmas marked as uncached stay that way. -*/ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); - } +if (vma_wants_writenotify (vma)) +vma_enable_writenotify (vma);vma_link (mm, vma, prev, rb_link, rb_parent);/* Once vma denies write, undo our temporary denial count */
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 1/1 mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
In reply to Peter FeinerFor VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after VM_SOFTDIRTY is cleared, then the PTE's softdirty bit will remain clear after subsequent writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is cleared. Furthermore, to avoid unnecessary faults, write notifications are disabled when VM_SOFTDIRTY is reset.
As a side effect of enabling and disabling write notifications with care, this patch fixes a bug in mprotect where vm_page_prot bits set by drivers were zapped on mprotect. An analogous bug was fixed in mmap by c9d0bf241451a3ab7d02e1652c22b80cd7d93e8f.
---
v1 -> v2: Instead of checking VM_SOFTDIRTY in the fault handler,
enable write notifications on vm_page_prot when we clear VM_SOFTDIRTY.
v2 -> v3: * Grab the mmap_sem in write mode if any VMAs have
VM_SOFTDIRTY set. This involved refactoring clear_refs_write to make it less unwieldy.
* In mprotect, don't inadvertently disable write notifications on VMAs that have had VM_SOFTDIRTY cleared
* The mprotect fix and mmap cleanup that comprised the second and third patches in v2 were swallowed by the main patch because of vm_page_prot corner case handling.
--- fs/proc/task_mmu.c | 113 +++++++++++++++++++++++++++++++++-------------------- include/linux/mm.h | 14 +++++++ mm/mmap.c | 24 +++++------- mm/mprotect.c | 6 +-- 4 files changed, 97 insertions(+), 60 deletions(-)
fs/proc/task_mmu.c +71 -42
@@ -785,13 +785,80 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,return 0 ; } +static int clear_refs (struct mm_struct *mm,enum clear_refs_types type, +int write) +{ +int r =0 ; +struct vm_area_struct *vma; +struct clear_refs_private cp = { + .type = type, + }; +struct mm_walk clear_refs_walk = { + .pmd_entry = clear_refs_pte_range, + .mm = mm, + .private = &cp, + }; + +if (write) +down_write (&mm->mmap_sem); +else +down_read (&mm->mmap_sem); + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_start (mm,0 ,-1 ); + +for (vma = mm->mmap; vma; vma = vma->vm_next) { + cp.vma = vma; +if (is_vm_hugetlb_page (vma)) +continue ; +/* +* Writing 1 to /proc/pid/clear_refs affects all pages. +* +* Writing 2 to /proc/pid/clear_refs only affects +* Anonymous pages. +* +* Writing 3 to /proc/pid/clear_refs only affects file +* mapped pages. +* +* Writing 4 to /proc/pid/clear_refs affects all pages. +*/ +if (type ==CLEAR_REFS_ANON && vma->vm_file) +continue ; +if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) +continue ; +if (type ==CLEAR_REFS_SOFT_DIRTY && + (vma->vm_flags &VM_SOFTDIRTY )) { +if (!write) { + r = -EAGAIN ; +break ; + } + vma->vm_flags &= ~VM_SOFTDIRTY ; +vma_enable_writenotify (vma); + } +walk_page_range (vma->vm_start, vma->vm_end, + &clear_refs_walk); + } + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_end (mm,0 ,-1 ); + +if (!r) +flush_tlb_mm (mm); + +if (write) +up_write (&mm->mmap_sem); +else +up_read (&mm->mmap_sem); + +return r; +} +static ssize_t clear_refs_write (struct file *file,const char __user *buf,size_t count,loff_t *ppos) {struct task_struct *task;char buffer[PROC_NUMBUF ];struct mm_struct *mm; -struct vm_area_struct *vma;enum clear_refs_types type;int itype;int rv;
@@ -820,47 +887,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,return -ESRCH ; mm =get_task_mm (task);if (mm) { -struct clear_refs_private cp = { - .type = type, - }; -struct mm_walk clear_refs_walk = { - .pmd_entry = clear_refs_pte_range, - .mm = mm, - .private = &cp, - }; -down_read (&mm->mmap_sem); -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_start (mm,0 ,-1 ); -for (vma = mm->mmap; vma; vma = vma->vm_next) { - cp.vma = vma; -if (is_vm_hugetlb_page (vma)) -continue ; -/* -* Writing 1 to /proc/pid/clear_refs affects all pages. -* -* Writing 2 to /proc/pid/clear_refs only affects -* Anonymous pages. -* -* Writing 3 to /proc/pid/clear_refs only affects file -* mapped pages. -* -* Writing 4 to /proc/pid/clear_refs affects all pages. -*/ -if (type ==CLEAR_REFS_ANON && vma->vm_file) -continue ; -if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) -continue ; -if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; - } -walk_page_range (vma->vm_start, vma->vm_end, - &clear_refs_walk); - } -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_end (mm,0 ,-1 ); -flush_tlb_mm (mm); -up_read (&mm->mmap_sem); + rv =clear_refs (mm, type,0 ); +if (rv) +clear_refs (mm, type,1 );mmput (mm); }put_task_struct (task);
include/linux/mm.h +14 -0
@@ -1946,6 +1946,20 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif +/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} +#ifdef CONFIG_NUMA_BALANCING unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end);
mm/mmap.c +9 -15
@@ -1470,6 +1470,10 @@ int vma_wants_writenotify(struct vm_area_struct *vma)if (vma->vm_ops && vma->vm_ops->page_mkwrite)return 1 ; +/* Do we need to track softdirty? */ +if (!(vm_flags &VM_SOFTDIRTY )) +return 1 ; +/* The open routine did something to the protections already? */ if (pgprot_val (vma->vm_page_prot) !=pgprot_val (vm_get_page_prot (vm_flags)))
@@ -1610,21 +1614,6 @@ munmap_back: goto free_vma; } -if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; - -/* Can vma->vm_page_prot have changed?? -* -* Answer: Yes, drivers may have changed it in their -* f_op->mmap method. -* -* Ensures that vmas marked as uncached stay that way. -*/ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); - } -vma_link (mm, vma, prev, rb_link, rb_parent);/* Once vma denies write, undo our temporary denial count */ if (file) {
@@ -1658,6 +1647,11 @@ out:*/ vma->vm_flags |=VM_SOFTDIRTY ; +if (vma_wants_writenotify (vma)) +vma_enable_writenotify (vma); +else +vma_disable_writenotify (vma); +return addr; unmap_and_free_vma:
mm/mprotect.c +3 -3
@@ -320,12 +320,12 @@ success:* held in write mode. */ vma->vm_flags = newflags; - vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, -vm_get_page_prot (newflags));if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); +vma_enable_writenotify (vma); dirty_accountable =1 ; + }else { +vma_disable_writenotify (vma); } change_protection(vma, start, end, vma->vm_page_prot,
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 214 lines · Peter Feiner
For VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after
VM_SOFTDIRTYis cleared, then the PTE's softdirty bit will remain clear after subsequent writes.Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL,
getpagesize(),PROT_READ|PROT_WRITE,MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clearVM_SOFTDIRTY*/
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */With this patch, write notifications are enabled when
VM_SOFTDIRTYis cleared. Furthermore, to avoid unnecessary faults, write notifications are disabled whenVM_SOFTDIRTYis reset.As a side effect of enabling and disabling write notifications with care, this patch fixes a bug in mprotect where
vm_page_protbits set by drivers were zapped on mprotect. An analogous bug was fixed in mmap byc9d0bf241451a3ab7d02e1652c22b80cd7d93e8f.---
v1 -> v2: Instead of checking
VM_SOFTDIRTYin the fault handler,
enable write notifications onvm_page_protwhen we clearVM_SOFTDIRTY.v2 -> v3: * Grab the
mmap_semin write mode if any VMAs have
VM_SOFTDIRTYset. This involved refactoringclear_refs_writeto make it less unwieldy.* In mprotect, don't inadvertently disable write notifications on VMAs that have had
VM_SOFTDIRTYcleared* The mprotect fix and mmap cleanup that comprised the second and third patches in v2 were swallowed by the main patch because of
vm_page_protcorner case handling.--- fs/proc/task_mmu.c | 113 +++++++++++++++++++++++++++++++++-------------------- include/linux/mm.h | 14 +++++++ mm/mmap.c | 24 +++++------- mm/mprotect.c | 6 +-- 4 files changed, 97 insertions(+), 60 deletions(-)fs/proc/task_mmu.c +71
-42@@ -785,13 +785,80 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,return 0 ; } +static int clear_refs (struct mm_struct *mm,enum clear_refs_types type, +int write) +{ +int r =0 ; +struct vm_area_struct *vma; +struct clear_refs_private cp = { + .type = type, + }; +struct mm_walk clear_refs_walk = { + .pmd_entry = clear_refs_pte_range, + .mm = mm, + .private = &cp, + }; + +if (write) +down_write (&mm->mmap_sem); +else +down_read (&mm->mmap_sem); + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_start (mm,0 ,-1 ); + +for (vma = mm->mmap; vma; vma = vma->vm_next) { + cp.vma = vma; +if (is_vm_hugetlb_page (vma)) +continue ; +/* +* Writing 1 to /proc/pid/clear_refs affects all pages. +* +* Writing 2 to /proc/pid/clear_refs only affects +* Anonymous pages. +* +* Writing 3 to /proc/pid/clear_refs only affects file +* mapped pages. +* +* Writing 4 to /proc/pid/clear_refs affects all pages. +*/ +if (type ==CLEAR_REFS_ANON && vma->vm_file) +continue ; +if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) +continue ; +if (type ==CLEAR_REFS_SOFT_DIRTY && + (vma->vm_flags &VM_SOFTDIRTY )) { +if (!write) { + r = -EAGAIN ; +break ; + } + vma->vm_flags &= ~VM_SOFTDIRTY ; +vma_enable_writenotify (vma); + } +walk_page_range (vma->vm_start, vma->vm_end, + &clear_refs_walk); + } + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_end (mm,0 ,-1 ); + +if (!r) +flush_tlb_mm (mm); + +if (write) +up_write (&mm->mmap_sem); +else +up_read (&mm->mmap_sem); + +return r; +} +static ssize_t clear_refs_write (struct file *file,const char __user *buf,size_t count,loff_t *ppos) {struct task_struct *task;char buffer[PROC_NUMBUF ];struct mm_struct *mm; -struct vm_area_struct *vma;enum clear_refs_types type;int itype;int rv;@@ -820,47 +887,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,return -ESRCH ; mm =get_task_mm (task);if (mm) { -struct clear_refs_private cp = { - .type = type, - }; -struct mm_walk clear_refs_walk = { - .pmd_entry = clear_refs_pte_range, - .mm = mm, - .private = &cp, - }; -down_read (&mm->mmap_sem); -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_start (mm,0 ,-1 ); -for (vma = mm->mmap; vma; vma = vma->vm_next) { - cp.vma = vma; -if (is_vm_hugetlb_page (vma)) -continue ; -/* -* Writing 1 to /proc/pid/clear_refs affects all pages. -* -* Writing 2 to /proc/pid/clear_refs only affects -* Anonymous pages. -* -* Writing 3 to /proc/pid/clear_refs only affects file -* mapped pages. -* -* Writing 4 to /proc/pid/clear_refs affects all pages. -*/ -if (type ==CLEAR_REFS_ANON && vma->vm_file) -continue ; -if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) -continue ; -if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; - } -walk_page_range (vma->vm_start, vma->vm_end, - &clear_refs_walk); - } -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_end (mm,0 ,-1 ); -flush_tlb_mm (mm); -up_read (&mm->mmap_sem); + rv =clear_refs (mm, type,0 ); +if (rv) +clear_refs (mm, type,1 );mmput (mm); }put_task_struct (task);include/linux/mm.h +14
-0@@ -1946,6 +1946,20 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif +/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} +#ifdef CONFIG_NUMA_BALANCING unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end);mm/mmap.c +0
-0
mm/mmap.c int vma_wants_writenotify(struct vm_area_struct *vma) · line 1474
Quoted 5 lines
mm/mmap.c +2
-0@@ -1470,6 +1470,10 @@ int vma_wants_writenotify(struct vm_area_struct *vma)if (vma->vm_ops && vma->vm_ops->page_mkwrite)return 1 ; +/* Do we need to track softdirty? */ +if (!(vm_flags &VM_SOFTDIRTY ))
This will give false-positive if CONFIG_MEM_SOFT_DIRTY is disabled, since VM_SOFTDIRTY is 0 in this case:
if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))
Otherwise looks good to me.
Quoted 67 lines
mm/mmap.c +7
-15+return 1 ; +/* The open routine did something to the protections already? */ if (pgprot_val (vma->vm_page_prot) !=pgprot_val (vm_get_page_prot (vm_flags)))@@ -1610,21 +1614,6 @@ munmap_back: goto free_vma; } -if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; - -/* Can vma->vm_page_prot have changed?? -* -* Answer: Yes, drivers may have changed it in their -* f_op->mmap method. -* -* Ensures that vmas marked as uncached stay that way. -*/ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); - } -vma_link (mm, vma, prev, rb_link, rb_parent);/* Once vma denies write, undo our temporary denial count */ if (file) {@@ -1658,6 +1647,11 @@ out:*/ vma->vm_flags |=VM_SOFTDIRTY ; +if (vma_wants_writenotify (vma)) +vma_enable_writenotify (vma); +else +vma_disable_writenotify (vma); +return addr; unmap_and_free_vma:mm/mprotect.c +3
-3@@ -320,12 +320,12 @@ success:* held in write mode. */ vma->vm_flags = newflags; - vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, -vm_get_page_prot (newflags));if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); +vma_enable_writenotify (vma); dirty_accountable =1 ; + }else { +vma_disable_writenotify (vma); } change_protection(vma, start, end, vma->vm_page_prot,Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Cyrill Gorcunov
Reviewed-byRe: [PATCH v3] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
On Sun, Aug 24, 2014 at 10:59:24AM +0300, Kirill A. Shutemov wrote:
...
Quoted 14 lines
mm/mmap.c int vma_wants_writenotify(struct vm_area_struct *vma) · line 1474
Quoted 10 lines
mm/mmap.c +2
-0@@ -1470,6 +1470,10 @@ int vma_wants_writenotify(struct vm_area_struct *vma)if (vma->vm_ops && vma->vm_ops->page_mkwrite)return 1 ; +/* Do we need to track softdirty? */ +if (!(vm_flags &VM_SOFTDIRTY ))This will give false-positive if
CONFIG_MEM_SOFT_DIRTYis disabled, sinceVM_SOFTDIRTYis 0 in this case:
Quoted 6 lines
if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))Otherwise looks good to me.
Suggested-by Kirill A. Shutemov <[email protected]>Reviewed-by Kirill A. Shutemov <[email protected]>
Really sorry for delay. Thanks a huge, guys!
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v4 1/1 mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
In reply to Peter FeinerFor VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after VM_SOFTDIRTY is cleared, then the PTE's softdirty bit will remain clear after subsequent writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is cleared. Furthermore, to avoid unnecessary faults, write notifications are disabled when VM_SOFTDIRTY is reset.
As a side effect of enabling and disabling write notifications with care, this patch fixes a bug in mprotect where vm_page_prot bits set by drivers were zapped on mprotect. An analogous bug was fixed in mmap by c9d0bf241451a3ab7d02e1652c22b80cd7d93e8f.
---
v1 -> v2: Instead of checking VM_SOFTDIRTY in the fault handler,
enable write notifications on vm_page_prot when we clear VM_SOFTDIRTY.
v2 -> v3: * Grab the mmap_sem in write mode if any VMAs have
VM_SOFTDIRTY set. This involved refactoring clear_refs_write to make it less unwieldy.
* In mprotect, don't inadvertently disable write notifications on VMAs that have had VM_SOFTDIRTY cleared
* The mprotect fix and mmap cleanup that comprised the second and third patches in v2 were swallowed by the main patch because of vm_page_prot corner case handling.
v3 -> v4: Handle !defined(CONFIG_MEM_SOFT_DIRTY): old patch would have
enabled write notifications for all VMAs in this case.
--- fs/proc/task_mmu.c | 113 +++++++++++++++++++++++++++++++++-------------------- include/linux/mm.h | 14 +++++++ mm/mmap.c | 26 ++++++------ mm/mprotect.c | 6 +-- 4 files changed, 99 insertions(+), 60 deletions(-)
fs/proc/task_mmu.c +71 -42
@@ -785,13 +785,80 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,return 0 ; } +static int clear_refs (struct mm_struct *mm,enum clear_refs_types type, +int write) +{ +int r =0 ; +struct vm_area_struct *vma; +struct clear_refs_private cp = { + .type = type, + }; +struct mm_walk clear_refs_walk = { + .pmd_entry = clear_refs_pte_range, + .mm = mm, + .private = &cp, + }; + +if (write) +down_write (&mm->mmap_sem); +else +down_read (&mm->mmap_sem); + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_start (mm,0 ,-1 ); + +for (vma = mm->mmap; vma; vma = vma->vm_next) { + cp.vma = vma; +if (is_vm_hugetlb_page (vma)) +continue ; +/* +* Writing 1 to /proc/pid/clear_refs affects all pages. +* +* Writing 2 to /proc/pid/clear_refs only affects +* Anonymous pages. +* +* Writing 3 to /proc/pid/clear_refs only affects file +* mapped pages. +* +* Writing 4 to /proc/pid/clear_refs affects all pages. +*/ +if (type ==CLEAR_REFS_ANON && vma->vm_file) +continue ; +if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) +continue ; +if (type ==CLEAR_REFS_SOFT_DIRTY && + (vma->vm_flags &VM_SOFTDIRTY )) { +if (!write) { + r = -EAGAIN ; +break ; + } + vma->vm_flags &= ~VM_SOFTDIRTY ; +vma_enable_writenotify (vma); + } +walk_page_range (vma->vm_start, vma->vm_end, + &clear_refs_walk); + } + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_end (mm,0 ,-1 ); + +if (!r) +flush_tlb_mm (mm); + +if (write) +up_write (&mm->mmap_sem); +else +up_read (&mm->mmap_sem); + +return r; +} +static ssize_t clear_refs_write (struct file *file,const char __user *buf,size_t count,loff_t *ppos) {struct task_struct *task;char buffer[PROC_NUMBUF ];struct mm_struct *mm; -struct vm_area_struct *vma;enum clear_refs_types type;int itype;int rv;
@@ -820,47 +887,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,return -ESRCH ; mm =get_task_mm (task);if (mm) { -struct clear_refs_private cp = { - .type = type, - }; -struct mm_walk clear_refs_walk = { - .pmd_entry = clear_refs_pte_range, - .mm = mm, - .private = &cp, - }; -down_read (&mm->mmap_sem); -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_start (mm,0 ,-1 ); -for (vma = mm->mmap; vma; vma = vma->vm_next) { - cp.vma = vma; -if (is_vm_hugetlb_page (vma)) -continue ; -/* -* Writing 1 to /proc/pid/clear_refs affects all pages. -* -* Writing 2 to /proc/pid/clear_refs only affects -* Anonymous pages. -* -* Writing 3 to /proc/pid/clear_refs only affects file -* mapped pages. -* -* Writing 4 to /proc/pid/clear_refs affects all pages. -*/ -if (type ==CLEAR_REFS_ANON && vma->vm_file) -continue ; -if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) -continue ; -if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; - } -walk_page_range (vma->vm_start, vma->vm_end, - &clear_refs_walk); - } -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_end (mm,0 ,-1 ); -flush_tlb_mm (mm); -up_read (&mm->mmap_sem); + rv =clear_refs (mm, type,0 ); +if (rv) +clear_refs (mm, type,1 );mmput (mm); }put_task_struct (task);
include/linux/mm.h +14 -0
@@ -1946,6 +1946,20 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif +/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} +#ifdef CONFIG_NUMA_BALANCING unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end);
mm/mmap.c +11 -15
@@ -1470,6 +1470,12 @@ int vma_wants_writenotify(struct vm_area_struct *vma)if (vma->vm_ops && vma->vm_ops->page_mkwrite)return 1 ; +#ifdef CONFIG_MEM_SOFT_DIRTY +/* Do we need to track softdirty? */ +if (!(vm_flags &VM_SOFTDIRTY )) +return 1 ; +#endif +/* The open routine did something to the protections already? */ if (pgprot_val (vma->vm_page_prot) !=pgprot_val (vm_get_page_prot (vm_flags)))
@@ -1610,21 +1616,6 @@ munmap_back: goto free_vma; } -if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; - -/* Can vma->vm_page_prot have changed?? -* -* Answer: Yes, drivers may have changed it in their -* f_op->mmap method. -* -* Ensures that vmas marked as uncached stay that way. -*/ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); - } -vma_link (mm, vma, prev, rb_link, rb_parent);/* Once vma denies write, undo our temporary denial count */ if (file) {
@@ -1658,6 +1649,11 @@ out:*/ vma->vm_flags |=VM_SOFTDIRTY ; +if (vma_wants_writenotify (vma)) +vma_enable_writenotify (vma); +else +vma_disable_writenotify (vma); +return addr; unmap_and_free_vma:
mm/mprotect.c +3 -3
@@ -320,12 +320,12 @@ success:* held in write mode. */ vma->vm_flags = newflags; - vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, -vm_get_page_prot (newflags));if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); +vma_enable_writenotify (vma); dirty_accountable =1 ; + }else { +vma_disable_writenotify (vma); } change_protection(vma, start, end, vma->vm_page_prot,
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v5 1/1 mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
In reply to Peter FeinerFor VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after VM_SOFTDIRTY is cleared, then the PTE's softdirty bit will remain clear after subsequent writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is cleared. Furthermore, to avoid unnecessary faults, write notifications are disabled when VM_SOFTDIRTY is reset.
As a side effect of enabling and disabling write notifications with care, this patch fixes a bug in mprotect where vm_page_prot bits set by drivers were zapped on mprotect. An analogous bug was fixed in mmap by c9d0bf241451a3ab7d02e1652c22b80cd7d93e8f.
---
v1 -> v2: Instead of checking VM_SOFTDIRTY in the fault handler,
enable write notifications on vm_page_prot when we clear VM_SOFTDIRTY.
v2 -> v3: * Grab the mmap_sem in write mode if any VMAs have
VM_SOFTDIRTY set. This involved refactoring clear_refs_write to make it less unwieldy.
* In mprotect, don't inadvertently disable write notifications on VMAs that have had VM_SOFTDIRTY cleared
* The mprotect fix and mmap cleanup that comprised the second and third patches in v2 were swallowed by the main patch because of vm_page_prot corner case handling.
v3 -> v4: Handle !defined(CONFIG_MEM_SOFT_DIRTY): old patch would have
enabled write notifications for all VMAs in this case.
v4 -> v5: IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) instead of #ifdef ...
--- fs/proc/task_mmu.c | 113 +++++++++++++++++++++++++++++++++-------------------- include/linux/mm.h | 14 +++++++ mm/mmap.c | 24 +++++------- mm/mprotect.c | 6 +-- 4 files changed, 97 insertions(+), 60 deletions(-)
fs/proc/task_mmu.c +71 -42
@@ -785,13 +785,80 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,return 0 ; } +static int clear_refs (struct mm_struct *mm,enum clear_refs_types type, +int write) +{ +int r =0 ; +struct vm_area_struct *vma; +struct clear_refs_private cp = { + .type = type, + }; +struct mm_walk clear_refs_walk = { + .pmd_entry = clear_refs_pte_range, + .mm = mm, + .private = &cp, + }; + +if (write) +down_write (&mm->mmap_sem); +else +down_read (&mm->mmap_sem); + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_start (mm,0 ,-1 ); + +for (vma = mm->mmap; vma; vma = vma->vm_next) { + cp.vma = vma; +if (is_vm_hugetlb_page (vma)) +continue ; +/* +* Writing 1 to /proc/pid/clear_refs affects all pages. +* +* Writing 2 to /proc/pid/clear_refs only affects +* Anonymous pages. +* +* Writing 3 to /proc/pid/clear_refs only affects file +* mapped pages. +* +* Writing 4 to /proc/pid/clear_refs affects all pages. +*/ +if (type ==CLEAR_REFS_ANON && vma->vm_file) +continue ; +if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) +continue ; +if (type ==CLEAR_REFS_SOFT_DIRTY && + (vma->vm_flags &VM_SOFTDIRTY )) { +if (!write) { + r = -EAGAIN ; +break ; + } + vma->vm_flags &= ~VM_SOFTDIRTY ; +vma_enable_writenotify (vma); + } +walk_page_range (vma->vm_start, vma->vm_end, + &clear_refs_walk); + } + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_end (mm,0 ,-1 ); + +if (!r) +flush_tlb_mm (mm); + +if (write) +up_write (&mm->mmap_sem); +else +up_read (&mm->mmap_sem); + +return r; +} +static ssize_t clear_refs_write (struct file *file,const char __user *buf,size_t count,loff_t *ppos) {struct task_struct *task;char buffer[PROC_NUMBUF ];struct mm_struct *mm; -struct vm_area_struct *vma;enum clear_refs_types type;int itype;int rv;
@@ -820,47 +887,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,return -ESRCH ; mm =get_task_mm (task);if (mm) { -struct clear_refs_private cp = { - .type = type, - }; -struct mm_walk clear_refs_walk = { - .pmd_entry = clear_refs_pte_range, - .mm = mm, - .private = &cp, - }; -down_read (&mm->mmap_sem); -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_start (mm,0 ,-1 ); -for (vma = mm->mmap; vma; vma = vma->vm_next) { - cp.vma = vma; -if (is_vm_hugetlb_page (vma)) -continue ; -/* -* Writing 1 to /proc/pid/clear_refs affects all pages. -* -* Writing 2 to /proc/pid/clear_refs only affects -* Anonymous pages. -* -* Writing 3 to /proc/pid/clear_refs only affects file -* mapped pages. -* -* Writing 4 to /proc/pid/clear_refs affects all pages. -*/ -if (type ==CLEAR_REFS_ANON && vma->vm_file) -continue ; -if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) -continue ; -if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; - } -walk_page_range (vma->vm_start, vma->vm_end, - &clear_refs_walk); - } -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_end (mm,0 ,-1 ); -flush_tlb_mm (mm); -up_read (&mm->mmap_sem); + rv =clear_refs (mm, type,0 ); +if (rv) +clear_refs (mm, type,1 );mmput (mm); }put_task_struct (task);
include/linux/mm.h +14 -0
@@ -1946,6 +1946,20 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif +/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} +#ifdef CONFIG_NUMA_BALANCING unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end);
mm/mmap.c +9 -15
@@ -1470,6 +1470,10 @@ int vma_wants_writenotify(struct vm_area_struct *vma)if (vma->vm_ops && vma->vm_ops->page_mkwrite)return 1 ; +/* Do we need to track softdirty? */ +if (IS_ENABLED (CONFIG_MEM_SOFT_DIRTY ) && !(vm_flags &VM_SOFTDIRTY )) +return 1 ; +/* The open routine did something to the protections already? */ if (pgprot_val (vma->vm_page_prot) !=pgprot_val (vm_get_page_prot (vm_flags)))
@@ -1610,21 +1614,6 @@ munmap_back: goto free_vma; } -if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; - -/* Can vma->vm_page_prot have changed?? -* -* Answer: Yes, drivers may have changed it in their -* f_op->mmap method. -* -* Ensures that vmas marked as uncached stay that way. -*/ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); - } -vma_link (mm, vma, prev, rb_link, rb_parent);/* Once vma denies write, undo our temporary denial count */ if (file) {
@@ -1658,6 +1647,11 @@ out:*/ vma->vm_flags |=VM_SOFTDIRTY ; +if (vma_wants_writenotify (vma)) +vma_enable_writenotify (vma); +else +vma_disable_writenotify (vma); +return addr; unmap_and_free_vma:
mm/mprotect.c +3 -3
@@ -320,12 +320,12 @@ success:* held in write mode. */ vma->vm_flags = newflags; - vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, -vm_get_page_prot (newflags));if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); +vma_enable_writenotify (vma); dirty_accountable =1 ; + }else { +vma_disable_writenotify (vma); } change_protection(vma, start, end, vma->vm_page_prot,
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 4 lines · Peter Feiner
For VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after
VM_SOFTDIRTYis cleared, then the PTE's softdirty bit will remain clear after subsequent writes.
Good catch. Worrying that it's not been noticed until now. But I find quite a lot that needs thinking about in the fix.
Quoted 11 lines
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL,
getpagesize(),PROT_READ|PROT_WRITE,MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clearVM_SOFTDIRTY*/
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
Quoted 3 lines
With this patch, write notifications are enabled when
VM_SOFTDIRTYis cleared. Furthermore, to avoid unnecessary faults, write notifications are disabled whenVM_SOFTDIRTYis reset.
"reset" is often a synonym for "cleared": "whenever VM_SOFTDIRTY is set"?
Quoted 5 lines
As a side effect of enabling and disabling write notifications with care, this patch fixes a bug in mprotect where
vm_page_protbits set by drivers were zapped on mprotect. An analogous bug was fixed in mmap byc9d0bf241451a3ab7d02e1652c22b80cd7d93e8f.
Commit c9d0bf241451 ("mm: uncached vma support with writenotify"). Adding Magnus to the Cc list: I have some doubt as to whether his bugfix is in fact preserved below, and would like him to check.
Quoted 6 lines
Reported-by Peter Feiner <[email protected]>Suggested-by Kirill A. Shutemov <[email protected]>Reviewed-by Kirill A. Shutemov <[email protected]>Reviewed-by Cyrill Gorcunov <[email protected]>Signed-off-by Peter Feiner <[email protected]>
I like Kirill's suggestion to approach this via writenotify, but find the disable/enable rather confusing (partly because enabling writenotify amounts to disabling write access). I may be alone in my confusion.
Quoted 73 lines
---
v1 -> v2: Instead of checking
VM_SOFTDIRTYin the fault handler,
enable write notifications onvm_page_protwhen we clearVM_SOFTDIRTY.v2 -> v3: * Grab the
mmap_semin write mode if any VMAs have
VM_SOFTDIRTYset. This involved refactoringclear_refs_writeto make it less unwieldy.* In mprotect, don't inadvertently disable write notifications on VMAs that have had
VM_SOFTDIRTYcleared* The mprotect fix and mmap cleanup that comprised the second and third patches in v2 were swallowed by the main patch because of
vm_page_protcorner case handling.v3 -> v4: Handle !defined(CONFIG_MEM_SOFT_DIRTY): old patch would have
enabled write notifications for all VMAs in this case.v4 -> v5:
IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)instead of #ifdef ...--- fs/proc/task_mmu.c | 113 +++++++++++++++++++++++++++++++++-------------------- include/linux/mm.h | 14 +++++++ mm/mmap.c | 24 +++++------- mm/mprotect.c | 6 +-- 4 files changed, 97 insertions(+), 60 deletions(-)fs/proc/task_mmu.c +36
-0@@ -785,13 +785,80 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,return 0 ; } +static int clear_refs (struct mm_struct *mm,enum clear_refs_types type, +int write) +{ +int r =0 ; +struct vm_area_struct *vma; +struct clear_refs_private cp = { + .type = type, + }; +struct mm_walk clear_refs_walk = { + .pmd_entry = clear_refs_pte_range, + .mm = mm, + .private = &cp, + }; + +if (write) +down_write (&mm->mmap_sem); +else +down_read (&mm->mmap_sem); + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_start (mm,0 ,-1 ); + +for (vma = mm->mmap; vma; vma = vma->vm_next) { + cp.vma = vma; +if (is_vm_hugetlb_page (vma)) +continue ; +/* +* Writing 1 to /proc/pid/clear_refs affects all pages. +* +* Writing 2 to /proc/pid/clear_refs only affects +* Anonymous pages. +* +* Writing 3 to /proc/pid/clear_refs only affects file +* mapped pages. +* +* Writing 4 to /proc/pid/clear_refs affects all pages.
fs/proc/task_mmu.c static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, · line 833
Quoted 10 lines
fs/proc/task_mmu.c +10
-0+*/ +if (type ==CLEAR_REFS_ANON && vma->vm_file) +continue ; +if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) +continue ; +if (type ==CLEAR_REFS_SOFT_DIRTY && + (vma->vm_flags &VM_SOFTDIRTY )) { +if (!write) { + r = -EAGAIN ; +break ;
Hmm. For a long time I thought you were fixing another important bug with down_write, since we "always" use down_write to modify vm_flags.
But now I'm realizing that if this is the _only_ place which modifies vm_flags with down_read, then it's "probably" safe. I've a vague feeling that this was discussed before - is that so, Cyrill?
It certainly feels fragile to depend on this; but conversely, I don't like replacing a long down_read scan by an indefinite down_read scan followed by a long down_write scan.
I see that you earlier persuaded yourself that the races are benign if you stick with down_read. I can't confirm or deny that at present:
seems more important right now to get this mail out to you than think through that aspect.
fs/proc/task_mmu.c static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, · line 836
Quoted 3 lines
fs/proc/task_mmu.c +3
-0+ } + vma->vm_flags &= ~VM_SOFTDIRTY ; +vma_enable_writenotify (vma);
That's an example of how the vma_enable_writenotify() interface may be confusing. I thought for a while that that line was unsafe, there being quite other reasons why write protection may be needed; then realized it's okay because "enable" is the restrictive one.
Quoted 91 lines
fs/proc/task_mmu.c +22
-42+ } +walk_page_range (vma->vm_start, vma->vm_end, + &clear_refs_walk); + } + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_end (mm,0 ,-1 ); + +if (!r) +flush_tlb_mm (mm); + +if (write) +up_write (&mm->mmap_sem); +else +up_read (&mm->mmap_sem); + +return r; +} +static ssize_t clear_refs_write (struct file *file,const char __user *buf,size_t count,loff_t *ppos) {struct task_struct *task;char buffer[PROC_NUMBUF ];struct mm_struct *mm; -struct vm_area_struct *vma;enum clear_refs_types type;int itype;int rv;@@ -820,47 +887,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,return -ESRCH ; mm =get_task_mm (task);if (mm) { -struct clear_refs_private cp = { - .type = type, - }; -struct mm_walk clear_refs_walk = { - .pmd_entry = clear_refs_pte_range, - .mm = mm, - .private = &cp, - }; -down_read (&mm->mmap_sem); -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_start (mm,0 ,-1 ); -for (vma = mm->mmap; vma; vma = vma->vm_next) { - cp.vma = vma; -if (is_vm_hugetlb_page (vma)) -continue ; -/* -* Writing 1 to /proc/pid/clear_refs affects all pages. -* -* Writing 2 to /proc/pid/clear_refs only affects -* Anonymous pages. -* -* Writing 3 to /proc/pid/clear_refs only affects file -* mapped pages. -* -* Writing 4 to /proc/pid/clear_refs affects all pages. -*/ -if (type ==CLEAR_REFS_ANON && vma->vm_file) -continue ; -if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) -continue ; -if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; - } -walk_page_range (vma->vm_start, vma->vm_end, - &clear_refs_walk); - } -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_end (mm,0 ,-1 ); -flush_tlb_mm (mm); -up_read (&mm->mmap_sem); + rv =clear_refs (mm, type,0 ); +if (rv) +clear_refs (mm, type,1 );mmput (mm); }put_task_struct (task);include/linux/mm.h +3
-0@@ -1946,6 +1946,20 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif +/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{
include/linux/mm.h static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) · line 1961
Quoted 10 lines
include/linux/mm.h +10
-0+pgprot_t newprot =vm_get_page_prot (vma->vm_flags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +}
As mentioned above, I find that enable and disable confusing. Might it be better just to have a vma_set_page_prot(vma), which does the "if vma_wants_writenotify(vma) blah; else blah;" internally?
And does what you have there build on any architecture other than x86 and tile? Because pgprot_modify() was only used in mm/mprotect.c before, we declare the fallback version there, and so far as I can see, only x86 and tile declare the pgprot_modify() they need in a header file.
Quoted 9 lines
include/linux/mm.h +1
-0+#ifdef CONFIG_NUMA_BALANCING unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end);mm/mmap.c +0
-0
mm/mmap.c int vma_wants_writenotify(struct vm_area_struct *vma) · line 1479
Quoted 10 lines
mm/mmap.c +4
-0@@ -1470,6 +1470,10 @@ int vma_wants_writenotify(struct vm_area_struct *vma)if (vma->vm_ops && vma->vm_ops->page_mkwrite)return 1 ; +/* Do we need to track softdirty? */ +if (IS_ENABLED (CONFIG_MEM_SOFT_DIRTY ) && !(vm_flags &VM_SOFTDIRTY )) +return 1 ; +/* The open routine did something to the protections already? */ if (pgprot_val (vma->vm_page_prot) !=pgprot_val (vm_get_page_prot (vm_flags)))
That sets me wondering: have you placed the VM_SOFTDIRTY check in the right place in this series of tests?
I think, once pgprot_modify() is correct on all architectures, it should be possible to drop that pgprot_val() check from vma_wants_writenotify() - which would be a welcome simplification.
But what about the VM_PFNMAP test below it? If that test was necessary, then having your VM_SOFTDIRTY check before it seems dangerous. But I'm hoping we can persuade ourselves that the VM_PFNMAP test was unnecessary, and simply delete it.
Quoted 7 lines
mm/mmap.c +0
-3@@ -1610,21 +1614,6 @@ munmap_back: goto free_vma; } -if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; -
mm/mmap.c munmap_back: · line 1625
Quoted 10 lines
mm/mmap.c +0
-10-/* Can vma->vm_page_prot have changed?? -* -* Answer: Yes, drivers may have changed it in their -* f_op->mmap method. -* -* Ensures that vmas marked as uncached stay that way. -*/ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot);
So, this is where Magnus's bugfix gets deleted: but I'm afraid that with pgprot_modify() properly implemented only on x86 and tile, we cannot delete this so easily.
It's going to be tedious and error-prone to devise a proper pgprot_modify() for each of N unfamiliar architectures. I wonder if we can take a hint from Magnus's code there, to get a suitable default going, which may not be perfect for each, but will avoid introducing regression.
Or am I simply confused about the lack of proper pgprot_modify()s?
Quoted 22 lines
mm/mmap.c +5
-2- } -vma_link (mm, vma, prev, rb_link, rb_parent);/* Once vma denies write, undo our temporary denial count */ if (file) {@@ -1658,6 +1647,11 @@ out:*/ vma->vm_flags |=VM_SOFTDIRTY ; +if (vma_wants_writenotify (vma)) +vma_enable_writenotify (vma); +else +vma_disable_writenotify (vma); +return addr; unmap_and_free_vma:mm/mprotect.c +0
-0
mm/mprotect.c success: · line 326
Quoted 10 lines
mm/mprotect.c +1
-3@@ -320,12 +320,12 @@ success:* held in write mode. */ vma->vm_flags = newflags; - vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, -vm_get_page_prot (newflags));if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); +vma_enable_writenotify (vma); dirty_accountable =1 ;
Not an issue coming from your patch, but please take a look at how dirty_accountable gets used in change_pte_range(): I suspect we have a similar soft-dirty bug there, do you agree? Or does it work out safely?
Quoted 7 lines
mm/mprotect.c +2
-0+ }else { +vma_disable_writenotify (vma); } change_protection(vma, start, end, vma->vm_page_prot,Signature · 1 line
2.1.0.rc2.206.gedb03e5
scripts/checkpatch.pl has a few complaints too. Personally, I like to make very simple functions as brief as possible, ignoring the rule about a blank line between declarations and body. So I like your style, but others will disagree: I suppose we should bow to checkpatch there.
Hugh
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 3 lines · Hugh Dickins
Quoted 3 lines
fs/proc/task_mmu.c +3
-0+static int clear_refs (struct mm_struct *mm,enum clear_refs_types type, +int write) +{
...
Quoted 13 lines
Quoted 13 lines
fs/proc/task_mmu.c +13
-0+ +if (write) +down_write (&mm->mmap_sem); +else +down_read (&mm->mmap_sem); + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_start (mm,0 ,-1 ); + +for (vma = mm->mmap; vma; vma = vma->vm_next) { + cp.vma = vma; +if (is_vm_hugetlb_page (vma)) +continue ;
...
Quoted 13 lines
fs/proc/task_mmu.c static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, · line 833
Quoted 9 lines
fs/proc/task_mmu.c +9
-0+if (type ==CLEAR_REFS_ANON && vma->vm_file) +continue ; +if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) +continue ; +if (type ==CLEAR_REFS_SOFT_DIRTY && + (vma->vm_flags &VM_SOFTDIRTY )) { +if (!write) { + r = -EAGAIN ; +break ;Hmm. For a long time I thought you were fixing another important bug with
down_write, since we "always" usedown_writeto modifyvm_flags.
Quoted 3 lines
But now I'm realizing that if this is the _only_ place which modifies
vm_flagswithdown_read, then it's "probably" safe. I've a vague feeling that this was discussed before - is that so, Cyrill?
Well, as far as I remember we were not talking before about vm_flags and read-lock in this function, maybe it was on some unrelated lkml thread without me CC'ed? Until I miss something obvious using read-lock here for vm_flags modification should be safe, since the only thing which is important (in context of vma-softdirty) is the vma's presence. Hugh, mind to refresh my memory, how long ago the discussion took place?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 36 lines · Cyrill Gorcunov
Quoted 3 lines · Hugh Dickins
Quoted 3 lines
fs/proc/task_mmu.c +3
-0+static int clear_refs (struct mm_struct *mm,enum clear_refs_types type, +int write) +{...
Quoted 13 lines
Quoted 13 lines
fs/proc/task_mmu.c +13
-0+ +if (write) +down_write (&mm->mmap_sem); +else +down_read (&mm->mmap_sem); + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_start (mm,0 ,-1 ); + +for (vma = mm->mmap; vma; vma = vma->vm_next) { + cp.vma = vma; +if (is_vm_hugetlb_page (vma)) +continue ;...
Quoted 13 lines
fs/proc/task_mmu.c static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, · line 833
Quoted 9 lines
fs/proc/task_mmu.c +9
-0+if (type ==CLEAR_REFS_ANON && vma->vm_file) +continue ; +if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) +continue ; +if (type ==CLEAR_REFS_SOFT_DIRTY && + (vma->vm_flags &VM_SOFTDIRTY )) { +if (!write) { + r = -EAGAIN ; +break ;Hmm. For a long time I thought you were fixing another important bug with
down_write, since we "always" usedown_writeto modifyvm_flags.Quoted 3 lines
But now I'm realizing that if this is the _only_ place which modifies
vm_flagswithdown_read, then it's "probably" safe. I've a vague feeling that this was discussed before - is that so, Cyrill?
Quoted 6 lines
Well, as far as I remember we were not talking before about
vm_flagsand read-lock in this function, maybe it was on some unrelated lkml thread without me CC'ed? Until I miss something obvious using read-lock here forvm_flagsmodification should be safe, since the only thing which is important (in context of vma-softdirty) is the vma's presence. Hugh, mind to refresh my memory, how long ago the discussion took place?
It seems safe in vma-softdirty context. But if somebody else will decide that it's fine to modify vm_flags without down_write (in their context), we will get trouble. Sasha will come with weird bug report one day ;)
At least vm_flags must be updated atomically to avoid race in middle of load-modify-store.
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 12 lines · Kirill A. Shutemov
Quoted 11 lines
Quoted 4 lines
But now I'm realizing that if this is the _only_ place which modifies
vm_flagswithdown_read, then it's "probably" safe. I've a vague feeling that this was discussed before - is that so, Cyrill?Well, as far as I remember we were not talking before about
vm_flagsand read-lock in this function, maybe it was on some unrelated lkml thread without me CC'ed? Until I miss something obvious using read-lock here forvm_flagsmodification should be safe, since the only thing which is important (in context of vma-softdirty) is the vma's presence. Hugh, mind to refresh my memory, how long ago the discussion took place?
Quoted 6 lines
It seems safe in vma-softdirty context. But if somebody else will decide that it's fine to modify
vm_flagswithoutdown_write(in their context), we will get trouble. Sasha will come with weird bug report one day ;)At least
vm_flagsmust be updated atomically to avoid race in middle of load-modify-store.
Which race you mean here? Two concurrent clear-refs?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 13 lines · Cyrill Gorcunov
Quoted 12 lines · Kirill A. Shutemov
Quoted 11 lines
Quoted 4 lines
But now I'm realizing that if this is the _only_ place which modifies
vm_flagswithdown_read, then it's "probably" safe. I've a vague feeling that this was discussed before - is that so, Cyrill?Well, as far as I remember we were not talking before about
vm_flagsand read-lock in this function, maybe it was on some unrelated lkml thread without me CC'ed? Until I miss something obvious using read-lock here forvm_flagsmodification should be safe, since the only thing which is important (in context of vma-softdirty) is the vma's presence. Hugh, mind to refresh my memory, how long ago the discussion took place?
Quoted 8 lines
Quoted 6 lines
It seems safe in vma-softdirty context. But if somebody else will decide that it's fine to modify
vm_flagswithoutdown_write(in their context), we will get trouble. Sasha will come with weird bug report one day ;)At least
vm_flagsmust be updated atomically to avoid race in middle of load-modify-store.Which race you mean here? Two concurrent clear-refs?
Two concurent clear-refs is fine. But if somebody else will exploit the same approch to set/clear other VM_FOO and it will race with clear-refs we get trouble: some modifications can be lost.
Basically, it's safe if only soft-dirty is allowed to modify vm_flags without down_write(). But why is soft-dirty so special?
Should we consider moving protection of some vma fields under per-vma lock rather use over-loaded mmap_sem?
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 5 lines · Kirill A. Shutemov
Quoted 5 lines
Quoted 5 lines
It seems safe in vma-softdirty context. But if somebody else will decide that it's fine to modify
vm_flagswithoutdown_write(in their context), we will get trouble. Sasha will come with weird bug report one day ;)
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
At least
vm_flagsmust be updated atomically to avoid race in middle of load-modify-store.Which race you mean here? Two concurrent clear-refs?
Two concurent clear-refs is fine. But if somebody else will exploit the same approch to set/clear other
VM_FOOand it will race with clear-refs we get trouble: some modifications can be lost.
yup, i see
Quoted 2 lines
Basically, it's safe if only soft-dirty is allowed to modify
vm_flagswithoutdown_write(). But why is soft-dirty so special?
because how we use this bit, i mean in normal workload this bit won't be used intensively i think so it's not widespread in kernel code
Quoted 2 lines
Should we consider moving protection of some vma fields under per-vma lock rather use over-loaded
mmap_sem?
Hard to say, if vma-softdirty bit is the reason then I guess no, probably it worth to estimate how much profit we would have if using per-vma lock.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 5 lines · Cyrill Gorcunov
Quoted 2 lines
Basically, it's safe if only soft-dirty is allowed to modify
vm_flagswithoutdown_write(). But why is soft-dirty so special?because how we use this bit, i mean in normal workload this bit won't be used intensively i think so it's not widespread in kernel code
Weak argument to me.
What about walk through vmas twice: first with down_write() to modify vm_flags and vm_page_prot, then downgrade_write() and do walk_page_range() on every vma?
Signature · 7 lines
Kirill A. Shutemov -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 4 lines · Kirill A. Shutemov
Quoted 3 lines · Cyrill Gorcunov
Quoted 2 lines
Basically, it's safe if only soft-dirty is allowed to modify
vm_flagswithoutdown_write(). But why is soft-dirty so special?
Quoted 8 lines
Quoted 2 lines
because how we use this bit, i mean in normal workload this bit won't be used intensively i think so it's not widespread in kernel code
Weak argument to me.
What about walk through vmas twice: first with
down_write()to modifyvm_flagsandvm_page_prot, thendowngrade_write()and dowalk_page_range()on every vma?
I still it's undeeded, but for sure using write-lock/downgrade won't hurt, so no argues from my side.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 9 lines · Cyrill Gorcunov
Quoted 8 lines · Kirill A. Shutemov
Quoted 5 lines · Cyrill Gorcunov
Quoted 2 lines
Basically, it's safe if only soft-dirty is allowed to modify
vm_flagswithoutdown_write(). But why is soft-dirty so special?because how we use this bit, i mean in normal workload this bit won't be used intensively i think so it's not widespread in kernel code
Weak argument to me.
Yes. However rarely it's modified, we don't want any chance of it corrupting another flag.
VM_SOFTDIRTY is special in the sense that it's maintained in a very different way from the other VM_flags. If we had a little alignment padding space somewhere in struct vm_area_struct, I think I'd jump at Kirill's suggestion to move it out of vm_flags and into a new field:
that would remove some other special casing, like the vma merge issue.
But I don't think we have such padding space, and we'd prefer not to bloat struct vm_area_struct for it; so maybe it should stay for now. Besides, with Peter's patch, we're also talking about the locking on modifications to vm_page_prot, aren't we?
Quoted 6 lines
Quoted 4 lines
What about walk through vmas twice: first with
down_write()to modifyvm_flagsandvm_page_prot, thendowngrade_write()and dowalk_page_range()on every vma?I still it's undeeded,
Yes, so long as nothing else is doing the same. No bug yet, that we can see, but a bug in waiting.
Quoted 2 lines
but for sure using write-lock/downgrade won't hurt, so no argues from my side.
Yes, Kirill's two-stage suggestion seems the best:
down_write
quickly scan vmas clearing VM_SOFT_DIRTY and updating vm_page_prot downgrade_write (or up_write, down_read?)
slowly walk page tables write protecting and clearing soft-dirty on ptes up_read
But please don't mistake me for someone who has a good grasp of
soft-dirty: I don't.
Hugh
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 12 lines · Hugh Dickins
Quoted 2 lines
Quoted 2 lines
Weak argument to me.
Yes. However rarely it's modified, we don't want any chance of it corrupting another flag.
VM_SOFTDIRTYis special in the sense that it's maintained in a very different way from the otherVM_flags. If we had a little alignment padding space somewhere in structvm_area_struct, I think I'd jump at Kirill's suggestion to move it out ofvm_flagsand into a new field:
that would remove some other special casing, like the vma merge issue.
Quoted 4 lines
But I don't think we have such padding space, and we'd prefer not to bloat struct
vm_area_structfor it; so maybe it should stay for now. Besides, with Peter's patch, we're also talking about the locking on modifications tovm_page_prot, aren't we?
I think so.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
What about walk through vmas twice: first with
down_write()to modifyvm_flagsandvm_page_prot, thendowngrade_write()and dowalk_page_range()on every vma?I still it's undeeded,
Yes, so long as nothing else is doing the same. No bug yet, that we can see, but a bug in waiting.
:-)
Quoted 6 lines
Quoted 3 lines
but for sure using write-lock/downgrade won't hurt, so no argues from my side.
Yes, Kirill's two-stage suggestion seems the best:
Quoted 8 lines
down_write
quickly scan vmas clearingVM_SOFT_DIRTYand updatingvm_page_protdowngrade_write(orup_write,down_read?)
slowly walk page tables write protecting and clearing soft-dirty on ptesup_readBut please don't mistake me for someone who has a good grasp of
soft-dirty: I don't.
Thanks for sharing opinion, Hugh! (And thanks for second email about vma-flags) So lets move it to Kirill's way, otherwise indeed one day it might end up in a bug which for sure will not be easy to catch.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
In reply to Cyrill GorcunovQuoted 9 lines · Cyrill Gorcunov
Quoted 7 lines · Hugh Dickins
Hmm. For a long time I thought you were fixing another important bug with
down_write, since we "always" usedown_writeto modifyvm_flags.But now I'm realizing that if this is the _only_ place which modifies
vm_flagswithdown_read, then it's "probably" safe. I've a vague feeling that this was discussed before - is that so, Cyrill?
Quoted 6 lines
Well, as far as I remember we were not talking before about
vm_flagsand read-lock in this function, maybe it was on some unrelated lkml thread without me CC'ed? Until I miss something obvious using read-lock here forvm_flagsmodification should be safe, since the only thing which is important (in context of vma-softdirty) is the vma's presence. Hugh, mind to refresh my memory, how long ago the discussion took place?
Sorry for making you think you were losing your mind, Cyrill.
I myself have no recollection of any such conversation with you; but afraid that I might have lost _my_ memory of it - I didn't want to get too strident about how fragile (though probably not yet buggy) this down_read-for-updating-VM_SOFTDIRTY-onlyi is, if there had already been such a discussion, coming to the conclusion that it is okay for now.
I am fairly sure that I have had some such discussion before; but probably with someone else, probably still about mmap_sem and vm_flags, but probably some other VM_flag: the surprising realization that it may be safe but fragile to use just down_read for updating one particular flag.
Hugh
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
In reply to Hugh DickinsQuoted 6 lines · Hugh Dickins
Quoted 3 lines · Peter Feiner
With this patch, write notifications are enabled when
VM_SOFTDIRTYis cleared. Furthermore, to avoid unnecessary faults, write notifications are disabled whenVM_SOFTDIRTYis reset."reset" is often a synonym for "cleared": "whenever
VM_SOFTDIRTYis set"?
Agreed, "set" sounds good.
Quoted 9 lines
Quoted 4 lines
As a side effect of enabling and disabling write notifications with care, this patch fixes a bug in mprotect where
vm_page_protbits set by drivers were zapped on mprotect. An analogous bug was fixed in mmap byc9d0bf241451a3ab7d02e1652c22b80cd7d93e8f.Commit
c9d0bf241451("mm: uncached vma support with writenotify"). Adding Magnus to the Cc list: I have some doubt as to whether his bugfix is in fact preserved below, and would like him to check.
I believe the fix is preserved as long as pgprot_modify preserves cache flags. As you explain below, pgprot_modify only does this on x86 and tile. So this patch does indeed break c9d0bf241451 on most architectures. Furthermore, as you said below, this patch would break the build on the other architectures :-)
Quoted 4 lines
I like Kirill's suggestion to approach this via writenotify, but find the disable/enable rather confusing (partly because enabling writenotify amounts to disabling write access). I may be alone in my confusion.
I agree about the confusion. I wasn't too happy with the names myself. Furthermore, I only really use vma_disable_writenotify to set vm_page_prot from vm_flags. So I think the enable / disable idea is pretty broken. As you suggest below, I'm going to give vma_set_page_prot a try.
Quoted 13 lines
fs/proc/task_mmu.c static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, · line 833
Quoted 5 lines
fs/proc/task_mmu.c +5
-0+if (type ==CLEAR_REFS_SOFT_DIRTY && + (vma->vm_flags &VM_SOFTDIRTY )) { +if (!write) { + r = -EAGAIN ; +break ;Hmm. For a long time I thought you were fixing another important bug with
down_write, since we "always" usedown_writeto modifyvm_flags.But now I'm realizing that if this is the _only_ place which modifies
vm_flagswithdown_read, then it's "probably" safe. I've a vague feeling that this was discussed before - is that so, Cyrill?
Quoted 8 lines
It certainly feels fragile to depend on this; but conversely, I don't like replacing a long
down_readscan by an indefinitedown_readscan followed by a longdown_writescan.I see that you earlier persuaded yourself that the races are benign if you stick with
down_read. I can't confirm or deny that at present:
seems more important right now to get this mail out to you than think through that aspect.
Your observation is correct: clear_refs_write is the only place that vm_flags is modified without an exclusive lock on mmap_sem.
I was wrong about the race between clear_refs_write modifying vm_flags and the fault handler reading vm_flags being benign. I had thought that since clear_refs_write zaps all of the PTEs in the VMA after it modifies vm_flags, it was ok for a writable PTE to be temporarily installed to handle a read fault. However, if a write happened after the read fault and before clear_refs_write zapped the PTE, then we'd miss the write. Therefore I'm convinced that its necessary to serialize changes to vm_flags and fault handling.
There are a few ways to accomplish this serialization, all with their pros and cons:
* One down_read scan followed by a down_write scan, if necessary. This is the current implementation.
Pros: won't take exclusive lock when VMAs haven't changed.
Cons: might hold exclusive lock during page table walk.
* Per-vma lock, as Cyrill and Kirill were discussing.
Pros: handle faults on other VMAs when vm_flags is changing.
Cons: another lock acquired in fault path.
* Iterate over VMAs and modify vm_flags with down_write, then downgrade to down_read for page table scan, as Kirill suggested.
Pros: won't hold exclusive lock during page table walk.
Cons: clear_refs_write always grabs exclusive lock.
I think the extra lock in the fault handling path rules the per-vma lock out. Whether the first or third approach is better depends on whether or not VMAs are changing, which is obviously application specific behavior. A hybrid approach offers the best of both worlds (i.e., an optimistic down_read scan that bails out if there's a VM_SOFTDIRTY VMA and falls back to the downgrading approach).
Quoted 9 lines
fs/proc/task_mmu.c static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, · line 836
Quoted 4 lines
fs/proc/task_mmu.c +3
-0+ } + vma->vm_flags &= ~VM_SOFTDIRTY ; +vma_enable_writenotify (vma);That's an example of how the
vma_enable_writenotify()interface may be confusing. I thought for a while that that line was unsafe, there being quite other reasons why write protection may be needed; then realized it's okay because "enable" is the restrictive one.
Yep, agreed. It'll look like
vma->vm_flags &= ~VM_SOFTDIRTY; vma_set_page_prot(vma);
after implementing your suggestion.
Quoted 98 lines
Quoted 98 lines
fs/proc/task_mmu.c +22
-42+ } +walk_page_range (vma->vm_start, vma->vm_end, + &clear_refs_walk); + } + +if (type ==CLEAR_REFS_SOFT_DIRTY ) +mmu_notifier_invalidate_range_end (mm,0 ,-1 ); + +if (!r) +flush_tlb_mm (mm); + +if (write) +up_write (&mm->mmap_sem); +else +up_read (&mm->mmap_sem); + +return r; +} +static ssize_t clear_refs_write (struct file *file,const char __user *buf,size_t count,loff_t *ppos) {struct task_struct *task;char buffer[PROC_NUMBUF ];struct mm_struct *mm; -struct vm_area_struct *vma;enum clear_refs_types type;int itype;int rv;@@ -820,47 +887,9 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,return -ESRCH ; mm =get_task_mm (task);if (mm) { -struct clear_refs_private cp = { - .type = type, - }; -struct mm_walk clear_refs_walk = { - .pmd_entry = clear_refs_pte_range, - .mm = mm, - .private = &cp, - }; -down_read (&mm->mmap_sem); -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_start (mm,0 ,-1 ); -for (vma = mm->mmap; vma; vma = vma->vm_next) { - cp.vma = vma; -if (is_vm_hugetlb_page (vma)) -continue ; -/* -* Writing 1 to /proc/pid/clear_refs affects all pages. -* -* Writing 2 to /proc/pid/clear_refs only affects -* Anonymous pages. -* -* Writing 3 to /proc/pid/clear_refs only affects file -* mapped pages. -* -* Writing 4 to /proc/pid/clear_refs affects all pages. -*/ -if (type ==CLEAR_REFS_ANON && vma->vm_file) -continue ; -if (type ==CLEAR_REFS_MAPPED && !vma->vm_file) -continue ; -if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; - } -walk_page_range (vma->vm_start, vma->vm_end, - &clear_refs_walk); - } -if (type ==CLEAR_REFS_SOFT_DIRTY ) -mmu_notifier_invalidate_range_end (mm,0 ,-1 ); -flush_tlb_mm (mm); -up_read (&mm->mmap_sem); + rv =clear_refs (mm, type,0 ); +if (rv) +clear_refs (mm, type,1 );mmput (mm); }put_task_struct (task);include/linux/mm.h +9
-0@@ -1946,6 +1946,20 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) }#endif +/* Enable write notifications without blowing away special flags. */ +static inline void vma_enable_writenotify (struct vm_area_struct *vma) +{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags & ~VM_SHARED ); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +} + +/* Disable write notifications without blowing away special flags. */ +static inline void vma_disable_writenotify (struct vm_area_struct *vma)
Quoted 8 lines
include/linux/mm.h static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) · line 1961
Quoted 4 lines
include/linux/mm.h +4
-0+{ +pgprot_t newprot =vm_get_page_prot (vma->vm_flags); + vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, newprot); +}As mentioned above, I find that enable and disable confusing. Might it be better just to have a
vma_set_page_prot(vma), which does the "ifvma_wants_writenotify(vma)blah; else blah;" internally?
I like that idea. I'll think it through and give it a try.
Quoted 28 lines
And does what you have there build on any architecture other than x86 and tile? Because
pgprot_modify()was only used inmm/mprotect.cbefore, we declare the fallback version there, and so far as I can see, only x86 and tile declare thepgprot_modify()they need in a header file.Quoted 9 lines
include/linux/mm.h +1
-0+#ifdef CONFIG_NUMA_BALANCING unsigned long change_prot_numa (struct vm_area_struct *vma,unsigned long start,unsigned long end);mm/mmap.c +0
-0mm/mmap.c int vma_wants_writenotify(struct vm_area_struct *vma) · line 1479
Quoted 10 lines
mm/mmap.c +4
-0@@ -1470,6 +1470,10 @@ int vma_wants_writenotify(struct vm_area_struct *vma)if (vma->vm_ops && vma->vm_ops->page_mkwrite)return 1 ; +/* Do we need to track softdirty? */ +if (IS_ENABLED (CONFIG_MEM_SOFT_DIRTY ) && !(vm_flags &VM_SOFTDIRTY )) +return 1 ; +/* The open routine did something to the protections already? */ if (pgprot_val (vma->vm_page_prot) !=pgprot_val (vm_get_page_prot (vm_flags)))That sets me wondering: have you placed the
VM_SOFTDIRTYcheck in the right place in this series of tests?
Quoted 8 lines
I think, once
pgprot_modify()is correct on all architectures, it should be possible to drop thatpgprot_val()check fromvma_wants_writenotify()- which would be a welcome simplification.But what about the
VM_PFNMAPtest below it? If that test was necessary, then having yourVM_SOFTDIRTYcheck before it seems dangerous. But I'm hoping we can persuade ourselves that theVM_PFNMAPtest was unnecessary, and simply delete it.
If VM_PFNMAP is necessary, then I definitely put the VM_SOFTDIRTY check in the wrong spot :-) I don't know much (i.e., anything) about VM_PFNMAP, so I'll have to bone up on a lot of code before I have an informed opinion about the necessity of the check.
I had erroneously reasoned that it was necessary to put the VM_SOFTDIRTY check before the pgprot_val check in order to handle VMA merging correctly. I'll give this another think and look into dropping the pgprot_val check altogether, as you suggest.
Quoted 23 lines
Quoted 8 lines
mm/mmap.c +0
-3@@ -1610,21 +1614,6 @@ munmap_back: goto free_vma; } -if (vma_wants_writenotify (vma)) { -pgprot_t pprot = vma->vm_page_prot; -mm/mmap.c munmap_back: · line 1625
Quoted 10 lines
mm/mmap.c +0
-10-/* Can vma->vm_page_prot have changed?? -* -* Answer: Yes, drivers may have changed it in their -* f_op->mmap method. -* -* Ensures that vmas marked as uncached stay that way. -*/ - vma->vm_page_prot =vm_get_page_prot (vm_flags & ~VM_SHARED ); -if (pgprot_val (pprot) ==pgprot_val (pgprot_noncached (pprot))) - vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot);So, this is where Magnus's bugfix gets deleted: but I'm afraid that with
pgprot_modify()properly implemented only on x86 and tile, we cannot delete this so easily.
Quoted 7 lines
It's going to be tedious and error-prone to devise a proper
pgprot_modify()for each of N unfamiliar architectures. I wonder if we can take a hint from Magnus's code there, to get a suitable default going, which may not be perfect for each, but will avoid introducing regression.Or am I simply confused about the lack of proper pgprot_modify()s?
No, I think you're right about pgprot_modify. I like your idea for a best-effort implementation of a generic pgprot_modify. I'll give it a try and see if everything fits together nicely for VM_SOFTDIRTY.
Quoted 11 lines
Quoted 11 lines
mm/mprotect.c +0
-2@@ -320,12 +320,12 @@ success:* held in write mode. */ vma->vm_flags = newflags; - vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, -vm_get_page_prot (newflags));
Quoted 8 lines
mm/mprotect.c success: · line 326
Quoted 4 lines
mm/mprotect.c +1
-1if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); +vma_enable_writenotify (vma); dirty_accountable =1 ;Not an issue coming from your patch, but please take a look at how
dirty_accountablegets used inchange_pte_range(): I suspect we have a similar soft-dirty bug there, do you agree? Or does it work out safely?
Indeed, there is a similar bug in change_pte_range. If a PTE is dirty but not soft-dirty and dirty_accountable is true, then the PTE will be made writable and we'll never get to mark the PTE softdirty. Good catch! I'll submit another patch to fix this.
Quoted 4 lines
scripts/checkpatch.plhas a few complaints too. Personally, I like to make very simple functions as brief as possible, ignoring the rule about a blank line between declarations and body. So I like your style, but others will disagree: I suppose we should bow to checkpatch there.
Aye, I shall bend the knee.
Peter
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v5] mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
Quoted 13 lines · Peter Feiner
Quoted 11 lines · Hugh Dickins
That sets me wondering: have you placed the
VM_SOFTDIRTYcheck in the right place in this series of tests?I think, once
pgprot_modify()is correct on all architectures, it should be possible to drop thatpgprot_val()check fromvma_wants_writenotify()- which would be a welcome simplification.But what about the
VM_PFNMAPtest below it? If that test was necessary, then having yourVM_SOFTDIRTYcheck before it seems dangerous. But I'm hoping we can persuade ourselves that theVM_PFNMAPtest was unnecessary, and simply delete it.
Quoted 4 lines
If
VM_PFNMAPis necessary, then I definitely put theVM_SOFTDIRTYcheck in the wrong spot :-) I don't know much (i.e., anything) aboutVM_PFNMAP, so I'll have to bone up on a lot of code before I have an informed opinion about the necessity of the check.
AFAICT, the VM_PFNMAP check is unnecessary since I can't find any drivers that set VM_PFNMAP and enable dirty accounting on their mappings. If anything, VM_PFNMAP precludes mapping dirty tracking since set_page_dirty takes a struct_page argument! Perhaps the VM_PFNMAP check was originally put in vma_wants_writenotify as a safeguard against bogus calls to set_page_dirty? In any case, it seems harmless to me to put the VM_SOFTDIRTY check before the VM_PFNMAP check since none of the fault handling code in mm/memory.c calls set_page_dirty on a VM_PFNMAP fault because either vm_normal_page() returns NULL or ->fault() / ->page_mkwrite() return VM_FAULT_NOPAGE. Moreover, for the purpose of softdirty tracking, enabling write notifications on VM_PFNMAP VMAs is OK since do_wp_page does the right thing when vm_normal_page() returns NULL.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v6 1/1 mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
In reply to Peter FeinerFor VMAs that don't want write notifications, PTEs created for read faults have their write bit set. If the read fault happens after VM_SOFTDIRTY is cleared, then the PTE's softdirty bit will remain clear after subsequent writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is cleared. Furthermore, to avoid unnecessary faults, write notifications are disabled when VM_SOFTDIRTY is set.
As a side effect of enabling and disabling write notifications with care, this patch fixes a bug in mprotect where vm_page_prot bits set by drivers were zapped on mprotect. An analogous bug was fixed in mmap by c9d0bf241451a3ab7d02e1652c22b80cd7d93e8f.
---
v1 -> v2: Instead of checking VM_SOFTDIRTY in the fault handler,
enable write notifications on vm_page_prot when we clear VM_SOFTDIRTY.
v2 -> v3: * Grab the mmap_sem in write mode if any VMAs have
VM_SOFTDIRTY set. This involved refactoring clear_refs_write to make it less unwieldy.
* In mprotect, don't inadvertently disable write notifications on VMAs that have had VM_SOFTDIRTY cleared
* The mprotect fix and mmap cleanup that comprised the second and third patches in v2 were swallowed by the main patch because of vm_page_prot corner case handling.
v3 -> v4: Handle !defined(CONFIG_MEM_SOFT_DIRTY): old patch would have
enabled write notifications for all VMAs in this case.
v4 -> v5: IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) instead of #ifdef ...
v5 -> v6:
* Replaced vma_{enable,disable}_writenotify() with vma_set_page_prot() as per Hugh's suggestion.
* Per Hugh's suggestion, added an arch generic pgprot_modify that handles pgprot_noncached and pgprot_writecombine pgprot. This arch generic pgprot_modify fixes the regression introduced in v2 that re-introduced the bug fixed by c9d0bf241451a3ab7d02e1652c22b80cd7d93e8f for arch's without pgprot_modify.
* Made vma_set_page_prot's pgprot check less restrictive by using pgprot_modify. Couldn't remove the pgprot check altogether since pgprot_modify isn't 100% on all arch's, such as powerpc.
* Fixed dirty_accountable bug.
* Fixed non-x86 build (tested build on arm64 and powerpc)
* Per Kirill's suggestion, changed locking so mmap_sem isn't held exclusively during page table traversal.
Kirill & Cyrill: I removed your Reviewed-By: footers since this patch is quite different than v5 and I didn't want to presume your approval. Please re-add as you see fit - this has been a group effort!
--- fs/proc/task_mmu.c | 19 +++++++++++++----- include/asm-generic/pgtable.h | 12 ++++++++++++ include/linux/mm.h | 5 +++++ mm/memory.c | 3 ++- mm/mmap.c | 45 +++++++++++++++++++++++++++---------------- mm/mprotect.c | 20 +++++-------------- 6 files changed, 66 insertions(+), 38 deletions(-)
fs/proc/task_mmu.c +14 -5
@@ -829,8 +829,21 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, .private = &cp, };down_read (&mm->mmap_sem); -if (type ==CLEAR_REFS_SOFT_DIRTY ) +if (type ==CLEAR_REFS_SOFT_DIRTY ) { +for (vma = mm->mmap; vma; vma = vma->vm_next) { +if (!(vma->vm_flags &VM_SOFTDIRTY )) +continue ; +up_read (&mm->mmap_sem); +down_write (&mm->mmap_sem); +for (vma = mm->mmap; vma; vma = vma->vm_next) { + vma->vm_flags &= ~VM_SOFTDIRTY ; +vma_set_page_prot (vma); + } +downgrade_write (&mm->mmap_sem); +break ; + }mmu_notifier_invalidate_range_start (mm,0 ,-1 ); + }for (vma = mm->mmap; vma; vma = vma->vm_next) { cp.vma = vma;if (is_vm_hugetlb_page (vma))
@@ -850,10 +863,6 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,continue ;if (type ==CLEAR_REFS_MAPPED && !vma->vm_file)continue ; -if (type ==CLEAR_REFS_SOFT_DIRTY ) { -if (vma->vm_flags &VM_SOFTDIRTY ) - vma->vm_flags &= ~VM_SOFTDIRTY ; - }walk_page_range (vma->vm_start, vma->vm_end, &clear_refs_walk); }
include/asm-generic/pgtable.h +12 -0
@@ -249,6 +249,18 @@ static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)#define pgprot_writecombine pgprot_noncached#endif +#ifndef pgprot_modify +#define pgprot_modify pgprot_modify +static inline pgprot_t pgprot_modify (pgprot_t oldprot,pgprot_t newprot) +{ +if (pgprot_val (oldprot) ==pgprot_val (pgprot_noncached (oldprot))) + newprot =pgprot_noncached (newprot); +if (pgprot_val (oldprot) ==pgprot_val (pgprot_writecombine (oldprot))) + newprot =pgprot_writecombine (newprot); +return newprot; +} +#endif +/* * When walking page tables, get the address of the next boundary, * or the end address of the range if that comes earlier. Although no
include/linux/mm.h +5 -0
@@ -1939,11 +1939,16 @@ static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,#ifdef CONFIG_MMU pgprot_t vm_get_page_prot (unsigned long vm_flags); +void vma_set_page_prot (struct vm_area_struct *vma);#else static inline pgprot_t vm_get_page_prot (unsigned long vm_flags) {return __pgprot (0 ); } +static inline void vma_set_page_prot (struct vm_area_struct *vma) +{ + vma->vm_page_prot =vm_get_page_prot (vma->vm_flags); +}#endif #ifdef CONFIG_NUMA_BALANCING
mm/memory.c +2 -1
@@ -2051,7 +2051,8 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, old_page =vm_normal_page (vma, address, orig_pte);if (!old_page) {/* -* VM_MIXEDMAP !pfn_valid() case +* VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a +* VM_PFNMAP VMA. * * We should not cow pages in a shared writeable mapping. * Just mark the pages writable as we can't do any dirty
mm/mmap.c +28 -17
72 lines not on this page · raw message
mm/mprotect.c +5 -15
@@ -29,13 +29,6 @@#include <asm/cacheflush.h> #include <asm/tlbflush.h> -#ifndef pgprot_modify -static inline pgprot_t pgprot_modify (pgprot_t oldprot,pgprot_t newprot) -{ -return newprot; -} -#endif -/* * For a prot_numa update we only hold mmap_sem for read so there is a * potential race with faulting where a pmd was temporarily none. This
@@ -93,7 +86,9 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,* Avoid taking write faults for pages we * know to be dirty. */ -if (dirty_accountable &&pte_dirty (ptent)) +if (dirty_accountable &&pte_dirty (ptent) && + (pte_soft_dirty (ptent) || + !(vma->vm_flags &VM_SOFTDIRTY ))) ptent =pte_mkwrite (ptent);ptep_modify_prot_commit (mm, addr, pte, ptent); updated = true;
@@ -320,13 +315,8 @@ success:* held in write mode. */ vma->vm_flags = newflags; - vma->vm_page_prot =pgprot_modify (vma->vm_page_prot, -vm_get_page_prot (newflags)); - -if (vma_wants_writenotify (vma)) { - vma->vm_page_prot =vm_get_page_prot (newflags & ~VM_SHARED ); - dirty_accountable =1 ; - } + dirty_accountable =vma_wants_writenotify (vma); +vma_set_page_prot (vma);change_protection (vma, start, end, vma->vm_page_prot, dirty_accountable,0 );
Signature · 7 lines
2.1.0.rc2.206.gedb03e5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>