tracking dirty pages in shared mappings
Subsystems (5)
RFC PATCH 1/1 tracking dirty pages in shared mappings
People expressed the need to track dirty pages in shared mappings. Linus outlined the general idea of doing that through making clean writable pages write-protected and taking the write fault.
This patch does exactly that, it makes pages in a shared writable mapping write-protected. On write-fault the pages are marked dirty and made writable. When the pages get synced with their backing store, the write-protection is re-instated.
It survives a simple test and shows the dirty pages in /proc/vmstat.
Comments?
---
include/linux/mm.h | 5 +++- include/linux/rmap.h | 6 +++++ mm/filemap.c | 3 +- mm/fremap.c | 9 +++++-- mm/memory.c | 16 +++++++++++++ mm/page-writeback.c | 2 + mm/rmap.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++ mm/shmem.c | 2 - 8 files changed, 98 insertions(+), 6 deletions(-)
include/linux/mm.h +4 -1
@@ -183,6 +183,9 @@ extern unsigned int kobjsize(const void#define VM_SequentialReadHint (v) ((v)->vm_flags & VM_SEQ_READ)#define VM_RandomReadHint (v) ((v)->vm_flags & VM_RAND_READ) +#define VM_SharedWritable (v) (((v)->vm_flags & (VM_SHARED | VM_MAYSHARE)) && \ + ((v)->vm_flags & VM_WRITE)) +/* * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask..
@@ -721,7 +724,7 @@ static inline void unmap_shared_mapping_extern int vmtruncate (struct inode * inode,loff_t offset);extern int vmtruncate_range (struct inode * inode,loff_t offset,loff_t end); -extern int install_page (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,struct page *page,pgprot_t prot); +extern int install_page (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,struct page *page,pgprot_t prot,int wrprotect);extern int install_file_pte (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,unsigned long pgoff,pgprot_t prot);#ifdef CONFIG_MMU
mm/filemap.c +2 -1
@@ -1627,7 +1627,8 @@ repeat:return -ENOMEM ;if (page) { - err =install_page (mm, vma, addr, page, prot); + err =install_page (mm, vma, addr, page, prot, +VM_SharedWritable (vma));if (err) {page_cache_release (page);return err;
mm/fremap.c +6 -3
@@ -49,7 +49,8 @@ static int zap_pte(struct mm_struct *mm,* previously existing mapping. */ int install_page (struct mm_struct *mm,struct vm_area_struct *vma, -unsigned long addr,struct page *page,pgprot_t prot) +unsigned long addr,struct page *page,pgprot_t prot, +int wrprotect) {struct inode *inode;pgoff_t size;
@@ -79,9 +80,11 @@ int install_page(struct mm_struct *mm, sinc_mm_counter (mm, file_rss);flush_icache_page (vma, page); -set_pte_at (mm, addr, pte,mk_pte (page, prot)); + pte_val =mk_pte (page, prot); +if (wrprotect) + pte_val =pte_wrprotect (pte_val); +set_pte_at (mm, addr, pte, pte_val);page_add_file_rmap (page); - pte_val = *pte;update_mmu_cache (vma, addr, pte_val); err =0 ; unlock:
mm/memory.c +16 -0
@@ -1495,6 +1495,18 @@ static int do_wp_page(struct mm_struct * } } +if (VM_SharedWritable (vma)) { +flush_cache_page (vma, address,pte_pfn (orig_pte)); + entry =pte_mkyoung (orig_pte); + entry =pte_mkwrite (pte_mkdirty (entry)); +ptep_set_access_flags (vma, address, page_table, entry,1 ); +update_mmu_cache (vma, address, entry); +lazy_mmu_prot_update (entry); + ret |=VM_FAULT_WRITE ; +set_page_dirty (old_page); + goto unlock; + } +/* * Ok, we need to copy. Oh, well.. */
@@ -2150,6 +2162,8 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma); +else if (VM_SharedWritable (vma)) + entry =pte_wrprotect (entry);set_pte_at (mm, address, page_table, entry);if (anon) {inc_mm_counter (mm, anon_rss);
@@ -2159,6 +2173,8 @@ retry: }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); +if (write_access) +set_page_dirty (new_page); } }else {/* One of our sibling threads was faster, back out. */
mm/shmem.c +1 -1
@@ -1270,7 +1270,7 @@ static int shmem_populate(struct vm_area/* Page may still be null, but only if nonblock was set. */ if (page) {mark_page_accessed (page); - err =install_page (mm, vma, addr, page, prot); + err =install_page (mm, vma, addr, page, prot,0 );if (err) {page_cache_release (page);return err;
mm/page-writeback.c +2 -0
@@ -725,6 +725,7 @@ int test_clear_page_dirty(struct page *ppage_index (page), PAGECACHE_TAG_DIRTY);write_unlock_irqrestore (&mapping->tree_lock, flags); +page_wrprotect (page);if (mapping_cap_account_dirty (mapping))dec_page_state (nr_dirty);return 1 ;
@@ -756,6 +757,7 @@ int clear_page_dirty_for_io(struct pageif (mapping) {if (TestClearPageDirty (page)) { +page_wrprotect (page);if (mapping_cap_account_dirty (mapping))dec_page_state (nr_dirty);return 1 ;
mm/rmap.c +61 -0
67 lines not on this page · raw message
include/linux/rmap.h +6 -0
@@ -105,6 +105,12 @@ pte_t *page_check_address(struct page *,*/ unsigned long page_address_in_vma (struct page *,struct vm_area_struct *); +/* +* Used to writeprotect clean pages, in order to count nr_dirty for shared +* mappings +*/ +int page_wrprotect (struct page *); +#else /* !CONFIG_MMU */ #define anon_vma_init () do {} while (0)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 4 lines · Peter Zijlstra
People expressed the need to track dirty pages in shared mappings. Linus outlined the general idea of doing that through making clean writable pages write-protected and taking the write fault.
Quoted 9 lines
This patch does exactly that, it makes pages in a shared writable mapping write-protected. On write-fault the pages are marked dirty and made writable. When the pages get synced with their backing store, the write-protection is re-instated.
It survives a simple test and shows the dirty pages in
/proc/vmstat.Comments?
Looks pretty good. Christoph and I were looking at ways to improve performance impact of this, and skipping the extra work for particular (eg. shmem) mappings might be a good idea?
Attached is a patch with a couple of things I've currently got.
In the long run, I'd like to be able to set_page_dirty and balance_dirty_pages outside of both ptl and mmap_sem, for performance reasons. That will require a reworking of arch code though :(
mm/memory.c +19 -17
@@ -48,6 +48,7 @@#include <linux/rmap.h> #include <linux/module.h> #include <linux/init.h> +#include <linux/backing-dev.h> #include <asm/pgalloc.h> #include <asm/uaccess.h>
@@ -1466,18 +1467,6 @@ static int do_wp_page(struct mm_struct * } } -if (VM_SharedWritable (vma)) { -flush_cache_page (vma, address,pte_pfn (orig_pte)); - entry =pte_mkyoung (orig_pte); - entry =pte_mkwrite (pte_mkdirty (entry)); -ptep_set_access_flags (vma, address, page_table, entry,1 ); -update_mmu_cache (vma, address, entry); -lazy_mmu_prot_update (entry); - ret |=VM_FAULT_WRITE ; -set_page_dirty (old_page); - goto unlock; - } -/* * Ok, we need to copy. Oh, well.. */
@@ -2131,8 +2120,11 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma); -else if (VM_SharedWritable (vma)) - entry =pte_wrprotect (entry); +else if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (new_page); +if (mapping &&mapping_cap_account_dirty (mapping)) + entry =pte_wrprotect (entry); + }set_pte_at (mm, address, page_table, entry);if (anon) {inc_mm_counter (mm, anon_rss);
@@ -2241,12 +2233,22 @@ static inline int handle_pte_fault(strucif (unlikely (!pte_same (*pte, entry))) goto unlock;if (write_access) { -if (!pte_write (entry)) -return do_wp_page (mm, vma, address, - pte, pmd, ptl, entry); +if (!pte_write (entry)) { +if (!VM_SharedWritable (vma)) { +return do_wp_page (mm, vma, address, + pte, pmd, ptl, entry); + }else { +struct page *page; + entry =pte_mkwrite (entry); + page =vm_normal_page (vma, address, entry); +if (page) +set_page_dirty (page); + } + } entry =pte_mkdirty (entry); } entry =pte_mkyoung (entry); +if (!pte_same (old_entry, entry)) {ptep_set_access_flags (vma, address, pte, entry, write_access);update_mmu_cache (vma, address, entry);
include/linux/mm.h +2 -3
@@ -183,8 +183,7 @@ extern unsigned int kobjsize(const void#define VM_SequentialReadHint (v) ((v)->vm_flags & VM_SEQ_READ)#define VM_RandomReadHint (v) ((v)->vm_flags & VM_RAND_READ) -#define VM_SharedWritable (v) (((v)->vm_flags & (VM_SHARED | VM_MAYSHARE)) && \ - ((v)->vm_flags & VM_WRITE)) +#define VM_SharedWritable (v) ((v)->vm_flags & (VM_SHARED|VM_WRITE))/* * mapping from the currently active vm_flags protection bits (the
@@ -724,7 +723,7 @@ static inline void unmap_shared_mapping_extern int vmtruncate (struct inode * inode,loff_t offset);extern int vmtruncate_range (struct inode * inode,loff_t offset,loff_t end); -extern int install_page (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,struct page *page,pgprot_t prot,int wrprotect); +extern int install_page (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,struct page *page,pgprot_t prot);extern int install_file_pte (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,unsigned long pgoff,pgprot_t prot);#ifdef CONFIG_MMU
mm/filemap.c +1 -2
@@ -1582,8 +1582,7 @@ repeat:return -ENOMEM ;if (page) { - err =install_page (mm, vma, addr, page, prot, -VM_SharedWritable (vma)); + err =install_page (mm, vma, addr, page, prot);if (err) {page_cache_release (page);return err;
mm/fremap.c +7 -4
@@ -15,6 +15,7 @@#include <linux/rmap.h> #include <linux/module.h> #include <linux/syscalls.h> +#include <linux/backing-dev.h> #include <asm/mmu_context.h> #include <asm/cacheflush.h>
@@ -49,8 +50,7 @@ static int zap_pte(struct mm_struct *mm,* previously existing mapping. */ int install_page (struct mm_struct *mm,struct vm_area_struct *vma, -unsigned long addr,struct page *page,pgprot_t prot, -int wrprotect) +unsigned long addr,struct page *page,pgprot_t prot) {struct inode *inode;pgoff_t size;
@@ -81,8 +81,11 @@ int install_page(struct mm_struct *mm, sflush_icache_page (vma, page); pte_val =mk_pte (page, prot); -if (wrprotect) - pte_val =pte_wrprotect (pte_val); +if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (page); +if (mapping &&mapping_cap_account_dirty (mapping)) + pte_val =pte_wrprotect (pte_val); + }set_pte_at (mm, addr, pte, pte_val);page_add_file_rmap (page);update_mmu_cache (vma, addr, pte_val);
mm/shmem.c +1 -1
@@ -1270,7 +1270,7 @@ static int shmem_populate(struct vm_area/* Page may still be null, but only if nonblock was set. */ if (page) {mark_page_accessed (page); - err =install_page (mm, vma, addr, page, prot,0 ); + err =install_page (mm, vma, addr, page, prot);if (err) {page_cache_release (page);return err;
mm/page-writeback.c +7 -4
@@ -29,6 +29,7 @@#include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/syscalls.h> +#include <linux/rmap.h> /* * The maximum number of pages to writeout in a single bdflush/kupdate
@@ -725,9 +726,10 @@ int test_clear_page_dirty(struct page *ppage_index (page), PAGECACHE_TAG_DIRTY);write_unlock_irqrestore (&mapping->tree_lock, flags); -page_wrprotect (page); -if (mapping_cap_account_dirty (mapping)) +if (mapping_cap_account_dirty (mapping)) { +page_wrprotect (page);dec_page_state (nr_dirty); + }return 1 ; }write_unlock_irqrestore (&mapping->tree_lock, flags);
@@ -757,9 +759,10 @@ int clear_page_dirty_for_io(struct pageif (mapping) {if (TestClearPageDirty (page)) { -page_wrprotect (page); -if (mapping_cap_account_dirty (mapping)) +if (mapping_cap_account_dirty (mapping)) { +page_wrprotect (page);dec_page_state (nr_dirty); + }return 1 ; }return 0 ;
Quoted 6 lines · Nick Piggin
Looks pretty good. Christoph and I were looking at ways to improve performance impact of this, and skipping the extra work for particular (eg. shmem) mappings might be a good idea?
Attached is a patch with a couple of things I've currently got.
will merge with mine and post a new version shortly.
Quoted 3 lines
In the long run, I'd like to be able to
set_page_dirtyandbalance_dirty_pagesoutside of both ptl andmmap_sem, for performance reasons. That will require a reworking of arch code though :(
That would indeed be very nice if possible.
Quoted 5 lines
Index:
linux-2.6/include/linux/mm.h
===================================================================
include/linux/mm.h extern unsigned int kobjsize(const void · line 189
Quoted 9 lines
include/linux/mm.h +1
-2@@ -183,8 +183,7 @@ extern unsigned int kobjsize(const void#define VM_SequentialReadHint (v) ((v)->vm_flags & VM_SEQ_READ)#define VM_RandomReadHint (v) ((v)->vm_flags & VM_RAND_READ) -#define VM_SharedWritable (v) (((v)->vm_flags & (VM_SHARED | VM_MAYSHARE)) && \ - ((v)->vm_flags & VM_WRITE)) +#define VM_SharedWritable (v) ((v)->vm_flags & (VM_SHARED|VM_WRITE))/* * mapping from the currently active vm_flags protection bits (the
That doesn't look right, your version is true even for unwritable shared, and writable non-shared VMAs.
PeterZ
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 12 lines · Peter Zijlstra
Quoted 6 lines · Nick Piggin
Looks pretty good. Christoph and I were looking at ways to improve performance impact of this, and skipping the extra work for particular (eg. shmem) mappings might be a good idea?
Attached is a patch with a couple of things I've currently got.
will merge with mine and post a new version shortly.
Thanks.
Quoted 7 lines
Quoted 4 lines
In the long run, I'd like to be able to
set_page_dirtyandbalance_dirty_pagesoutside of both ptl andmmap_sem, for performance reasons. That will require a reworking of arch code though :(That would indeed be very nice if possible.
Yep. Let's not distract from getting the basic mechanism working though. balance_dirty_pages would be patch 2..n ;)
Quoted 12 lines
Quoted 12 lines
Index:
linux-2.6/include/linux/mm.h
===================================================================include/linux/mm.h +1
-2@@ -183,8 +183,7 @@ extern unsigned int kobjsize(const void#define VM_SequentialReadHint (v) ((v)->vm_flags & VM_SEQ_READ)#define VM_RandomReadHint (v) ((v)->vm_flags & VM_RAND_READ) -#define VM_SharedWritable (v) (((v)->vm_flags & (VM_SHARED | VM_MAYSHARE)) && \ - ((v)->vm_flags & VM_WRITE)) +#define VM_SharedWritable (v) ((v)->vm_flags & (VM_SHARED|VM_WRITE))
Quoted 7 lines
include/linux/mm.h extern unsigned int kobjsize(const void · line 189
Quoted 3 lines
include/linux/mm.h +0
-0/* * mapping from the currently active vm_flags protection bits (the That doesn't look right, your version is true even for unwritable shared, and writable non-shared VMAs.
Of course, thanks. I guess that should be
#define VM_SharedWritable(v) ((v)->vm_flags & (VM_SHARED|VM_WRITE) == (VM_SHARED|VM_WRITE))
BTW. It is unconventional (outside the read hints stuff) to use macros like this. I guess real VM hackers have to know what is intended by any given esoteric combination of flags in any given context.
Not that I hate it.
But if we're going to start using it, we should work out a sane convention and stick to it. "StudlyCaps" seem to be out of favour, and using a vma_ prefix would be more sensible.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 2 lines · Nick Piggin
Yep. Let's not distract from getting the basic mechanism working though.
balance_dirty_pageswould be patch 2..n ;)
Attached are both a new version of the shared_mapping_dirty patch, and balance_dirty_pages; to be applied in that order.
It makes my testcase survive and not OOM like it used to.
Quoted 9 lines
BTW. It is unconventional (outside the read hints stuff) to use macros like this. I guess real VM hackers have to know what is intended by any given esoteric combination of flags in any given context.
Not that I hate it.
But if we're going to start using it, we should work out a sane convention and stick to it. "StudlyCaps" seem to be out of favour, and using a
vma_prefix would be more sensible.
Not a real fan of "StudlyCaps" myself either, just adapting to whatever was there.
This macro was born because I find writing the same thing more than twice a nuisance and errorprone. However if ppl feel otherwise I'm fine with either writing it out explicitly or renaming the thing, suggestions?
PeterZ
mm/memory.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-)
mm/memory.c +16 -2
@@ -50,6 +50,7 @@#include <linux/init.h> #include <linux/mm_page_replace.h> #include <linux/backing-dev.h> +#include <linux/writeback.h> #include <asm/pgalloc.h> #include <asm/uaccess.h>
@@ -2078,6 +2079,7 @@ static int do_no_page(struct mm_struct *unsigned int sequence =0 ;int ret =VM_FAULT_MINOR ;int anon =0 ; +int dirty =0 ;pte_unmap (page_table);BUG_ON (vma->vm_flags &VM_PFNMAP );
@@ -2165,8 +2167,10 @@ retry: }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); -if (write_access) +if (write_access) {set_page_dirty (new_page); + dirty++; + } } }else {/* One of our sibling threads was faster, back out. */
@@ -2179,6 +2183,8 @@ retry:lazy_mmu_prot_update (entry); unlock:pte_unmap_unlock (page_table, ptl); +if (dirty) +balance_dirty_pages_ratelimited_nr (mapping, dirty);return ret; oom:page_cache_release (new_page);
@@ -2243,6 +2249,8 @@ static inline int handle_pte_fault(strucpte_t entry;pte_t old_entry;spinlock_t *ptl; +struct address_space *mapping; +int dirty =0 ; old_entry = entry = *pte;if (!pte_present (entry)) {
@@ -2273,8 +2281,11 @@ static inline int handle_pte_fault(strucstruct page *page; entry =pte_mkwrite (entry); page =vm_normal_page (vma, address, entry); -if (page) +if (page) {set_page_dirty (page); + mapping =page_mapping (page); + dirty++; + } } } entry =pte_mkdirty (entry);
@@ -2297,6 +2308,9 @@ static inline int handle_pte_fault(struc } unlock:pte_unmap_unlock (pte, ptl); +if (dirty && mapping) +balance_dirty_pages_ratelimited_nr (mapping, dirty); +return VM_FAULT_MINOR ; }
From: Peter Zijlstra <[email protected]>
People expressed the need to track dirty pages in shared mappings.
Linus outlined the general idea of doing that through making clean writable pages write-protected and taking the write fault.
This patch does exactly that, it makes pages in a shared writable mapping write-protected. On write-fault the pages are marked dirty and made writable. When the pages get synced with their backing store, the write-protection is re-instated.
It survives a simple test and shows the dirty pages in /proc/vmstat.
Changes in -v2:
- only wrprotect pages from dirty capable mappings. (Nick Piggin)
- move the writefault handling from do_wp_page() into handle_pte_fault(). (Nick Piggin)
- revert to the old install_page interface. (Nick Piggin)
- also clear the pte dirty bit when we make pages read-only again. (spotted by Rik van Riel)
- make page_wrprotect() return the number of reprotected ptes.
include/linux/mm.h | 2 + include/linux/rmap.h | 6 ++++ mm/fremap.c | 10 ++++++- mm/memory.c | 24 ++++++++++++++++-- mm/page-writeback.c | 9 +++++- mm/rmap.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 110 insertions(+), 7 deletions(-)
include/linux/mm.h +2 -0
@@ -183,6 +183,8 @@ extern unsigned int kobjsize(const void#define VM_SequentialReadHint (v) ((v)->vm_flags & VM_SEQ_READ)#define VM_RandomReadHint (v) ((v)->vm_flags & VM_RAND_READ) +#define VM_SharedWritable (v) (((v)->vm_flags & (VM_SHARED|VM_WRITE)) == (VM_SHARED|VM_WRITE)) +/* * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask..
mm/fremap.c +8 -2
@@ -15,6 +15,7 @@#include <linux/rmap.h> #include <linux/module.h> #include <linux/syscalls.h> +#include <linux/backing-dev.h> #include <asm/mmu_context.h> #include <asm/cacheflush.h>
@@ -79,9 +80,14 @@ int install_page(struct mm_struct *mm, sinc_mm_counter (mm, file_rss);flush_icache_page (vma, page); -set_pte_at (mm, addr, pte,mk_pte (page, prot)); + pte_val =mk_pte (page, prot); +if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (page); +if (mapping &&mapping_cap_account_dirty (mapping)) + pte_val =pte_wrprotect (pte_val); + } +set_pte_at (mm, addr, pte, pte_val);page_add_file_rmap (page); - pte_val = *pte;update_mmu_cache (vma, addr, pte_val); err =0 ; unlock:
mm/memory.c +21 -3
@@ -49,6 +49,7 @@#include <linux/module.h> #include <linux/init.h> #include <linux/mm_page_replace.h> +#include <linux/backing-dev.h> #include <asm/pgalloc.h> #include <asm/uaccess.h>
@@ -2150,6 +2151,11 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma); +else if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (new_page); +if (mapping &&mapping_cap_account_dirty (mapping)) + entry =pte_wrprotect (entry); + }set_pte_at (mm, address, page_table, entry);if (anon) {inc_mm_counter (mm, anon_rss);
@@ -2159,6 +2165,8 @@ retry: }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); +if (write_access) +set_page_dirty (new_page); } }else {/* One of our sibling threads was faster, back out. */
@@ -2257,12 +2265,22 @@ static inline int handle_pte_fault(strucif (unlikely (!pte_same (*pte, entry))) goto unlock;if (write_access) { -if (!pte_write (entry)) -return do_wp_page (mm, vma, address, - pte, pmd, ptl, entry); +if (!pte_write (entry)) { +if (!VM_SharedWritable (vma)) { +return do_wp_page (mm, vma, address, + pte, pmd, ptl, entry); + }else { +struct page *page; + entry =pte_mkwrite (entry); + page =vm_normal_page (vma, address, entry); +if (page) +set_page_dirty (page); + } + } entry =pte_mkdirty (entry); } entry =pte_mkyoung (entry); +if (!pte_same (old_entry, entry)) {ptep_set_access_flags (vma, address, pte, entry, write_access);update_mmu_cache (vma, address, entry);
mm/page-writeback.c +7 -2
@@ -29,6 +29,7 @@#include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/syscalls.h> +#include <linux/rmap.h> /* * The maximum number of pages to writeout in a single bdflush/kupdate
@@ -725,8 +726,10 @@ int test_clear_page_dirty(struct page *ppage_index (page), PAGECACHE_TAG_DIRTY);write_unlock_irqrestore (&mapping->tree_lock, flags); -if (mapping_cap_account_dirty (mapping)) +if (mapping_cap_account_dirty (mapping)) { +page_wrprotect (page);dec_page_state (nr_dirty); + }return 1 ; }write_unlock_irqrestore (&mapping->tree_lock, flags);
@@ -756,8 +759,10 @@ int clear_page_dirty_for_io(struct pageif (mapping) {if (TestClearPageDirty (page)) { -if (mapping_cap_account_dirty (mapping)) +if (mapping_cap_account_dirty (mapping)) { +page_wrprotect (page);dec_page_state (nr_dirty); + }return 1 ; }return 0 ;
mm/rmap.c +66 -0
72 lines not on this page · raw message
include/linux/rmap.h +6 -0
@@ -105,6 +105,12 @@ pte_t *page_check_address(struct page *,*/ unsigned long page_address_in_vma (struct page *,struct vm_area_struct *); +/* +* Used to writeprotect clean pages, in order to count nr_dirty for shared +* mappings +*/ +int page_wrprotect (struct page *); +#else /* !CONFIG_MMU */ #define anon_vma_init () do {} while (0)
Quoted 12 lines · Peter Zijlstra
Quoted 3 lines · Nick Piggin
Yep. Let's not distract from getting the basic mechanism working though.
balance_dirty_pageswould be patch 2..n ;)Attached are both a new version of the
shared_mapping_dirtypatch, andbalance_dirty_pages; to be applied in that order.It makes my testcase survive and not OOM like it used to.
Looks OK. I wonder if test_clear_page_dirty could skip the page_wrprotect entirely? It would speed up cases like truncate that don't care. OTOH, it looks like several filesystems do not use clear_page_dirty_for_io where they possibly should be...
Perhaps you could consolidate both checks into test_set_page_writeback()?
Andrew, any ideas?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 14 lines · Nick Piggin
Quoted 11 lines · Peter Zijlstra
Quoted 3 lines · Nick Piggin
Yep. Let's not distract from getting the basic mechanism working though.
balance_dirty_pageswould be patch 2..n ;)Attached are both a new version of the
shared_mapping_dirtypatch, andbalance_dirty_pages; to be applied in that order.
It makes my testcase survive and not OOM like it used to.
Quoted 6 lines
Looks OK. I wonder if
test_clear_page_dirtycould skip thepage_wrprotectentirely? It would speed up cases like truncate that don't care. OTOH, it looks like several filesystems do not useclear_page_dirty_for_iowhere they possibly should be...Perhaps you could consolidate both checks into
test_set_page_writeback()?
Actually, you'll skip the slow path of taking the lock most of the times in situations like truncate because it will bail out after the page_mapped check. So doing anything further is probably a premature optimisation at this stage.
Nick
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 2 lines · Peter Zijlstra
Attached are both a new version of the
shared_mapping_dirtypatch, andbalance_dirty_pages; to be applied in that order.
Would you please sent patches inline? It seems that you need to initialize mapping to NULL in handle_pte_fault.
You could defer the page dirtying by taking a reference on the page then dropping the pte lock dirty the page and then drop the reference. That way dirtying is running without the pte lock.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 6 lines · Christoph Lameter
Quoted 2 lines · Peter Zijlstra
Attached are both a new version of the
shared_mapping_dirtypatch, andbalance_dirty_pages; to be applied in that order.Would you please sent patches inline?
Ofcourse, will do next round; in the meantime you can get the patches from:
http://programming.kicks-ass.net/kernel-patches/shared-dirty/
Quoted 2 lines
It seems that you need to initialize
mapping to NULL inhandle_pte_fault.
Almost, the dirty count avoids that. I should just init mapping to NULL and drop the dirty count; there is no way that can ever be more than 1 anyway.
Quoted 3 lines
You could defer the page dirtying by taking a reference on the page then dropping the pte lock dirty the page and then drop the reference. That way dirtying is running without the pte lock.
Right, very good, will do that when I get home.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH 1/2 tracking dirty pages in shared mappings -V3
In reply to Christoph LameterFrom: Peter Zijlstra <[email protected]>
People expressed the need to track dirty pages in shared mappings.
Linus outlined the general idea of doing that through making clean writable pages write-protected and taking the write fault.
This patch does exactly that, it makes pages in a shared writable mapping write-protected. On write-fault the pages are marked dirty and made writable. When the pages get synced with their backing store, the write-protection is re-instated.
It survives a simple test and shows the dirty pages in /proc/vmstat.
Changes in -v3:
- move set_page_dirty() outside pte lock (suggested by Christoph Lameter)
Changes in -v2:
- only wrprotect pages from dirty capable mappings. (Nick Piggin)
- move the writefault handling from do_wp_page() into handle_pte_fault(). (Nick Piggin)
- revert to the old install_page interface. (Nick Piggin)
- also clear the pte dirty bit when we make pages read-only again. (spotted by Rik van Riel)
- make page_wrprotect() return the number of reprotected ptes.
---
include/linux/mm.h | 2 + include/linux/rmap.h | 6 ++++ mm/fremap.c | 10 ++++++- mm/memory.c | 34 +++++++++++++++++++++++--- mm/page-writeback.c | 9 +++++- mm/rmap.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 120 insertions(+), 7 deletions(-)
include/linux/mm.h +2 -0
@@ -183,6 +183,8 @@ extern unsigned int kobjsize(const void#define VM_SequentialReadHint (v) ((v)->vm_flags & VM_SEQ_READ)#define VM_RandomReadHint (v) ((v)->vm_flags & VM_RAND_READ) +#define VM_SharedWritable (v) (((v)->vm_flags & (VM_SHARED|VM_WRITE)) == (VM_SHARED|VM_WRITE)) +/* * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask..
mm/fremap.c +8 -2
@@ -15,6 +15,7 @@#include <linux/rmap.h> #include <linux/module.h> #include <linux/syscalls.h> +#include <linux/backing-dev.h> #include <asm/mmu_context.h> #include <asm/cacheflush.h>
@@ -79,9 +80,14 @@ int install_page(struct mm_struct *mm, sinc_mm_counter (mm, file_rss);flush_icache_page (vma, page); -set_pte_at (mm, addr, pte,mk_pte (page, prot)); + pte_val =mk_pte (page, prot); +if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (page); +if (mapping &&mapping_cap_account_dirty (mapping)) + pte_val =pte_wrprotect (pte_val); + } +set_pte_at (mm, addr, pte, pte_val);page_add_file_rmap (page); - pte_val = *pte;update_mmu_cache (vma, addr, pte_val); err =0 ; unlock:
mm/memory.c +31 -3
@@ -49,6 +49,7 @@#include <linux/module.h> #include <linux/init.h> #include <linux/mm_page_replace.h> +#include <linux/backing-dev.h> #include <asm/pgalloc.h> #include <asm/uaccess.h>
@@ -2077,6 +2078,7 @@ static int do_no_page(struct mm_struct *unsigned int sequence =0 ;int ret =VM_FAULT_MINOR ;int anon =0 ; +int dirty =0 ;pte_unmap (page_table);BUG_ON (vma->vm_flags &VM_PFNMAP );
@@ -2150,6 +2152,11 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma); +else if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (new_page); +if (mapping &&mapping_cap_account_dirty (mapping)) + entry =pte_wrprotect (entry); + }set_pte_at (mm, address, page_table, entry);if (anon) {inc_mm_counter (mm, anon_rss);
@@ -2159,6 +2166,10 @@ retry: }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); +if (write_access) { +get_page (new_page); + dirty++; + } } }else {/* One of our sibling threads was faster, back out. */
@@ -2171,6 +2182,10 @@ retry:lazy_mmu_prot_update (entry); unlock:pte_unmap_unlock (page_table, ptl); +if (dirty) { +set_page_dirty (new_page); +put_page (new_page); + }return ret; oom:page_cache_release (new_page);
@@ -2235,6 +2250,7 @@ static inline int handle_pte_fault(strucpte_t entry;pte_t old_entry;spinlock_t *ptl; +struct page *page =NULL ; old_entry = entry = *pte;if (!pte_present (entry)) {
@@ -2257,12 +2273,20 @@ static inline int handle_pte_fault(strucif (unlikely (!pte_same (*pte, entry))) goto unlock;if (write_access) { -if (!pte_write (entry)) -return do_wp_page (mm, vma, address, - pte, pmd, ptl, entry); +if (!pte_write (entry)) { +if (!VM_SharedWritable (vma)) { +return do_wp_page (mm, vma, address, + pte, pmd, ptl, entry); + }else { + entry =pte_mkwrite (entry); + page =vm_normal_page (vma, address, entry); +get_page (page); + } + } entry =pte_mkdirty (entry); } entry =pte_mkyoung (entry); +if (!pte_same (old_entry, entry)) {ptep_set_access_flags (vma, address, pte, entry, write_access);update_mmu_cache (vma, address, entry);
@@ -2279,6 +2303,10 @@ static inline int handle_pte_fault(struc } unlock:pte_unmap_unlock (pte, ptl); +if (page) { +set_page_dirty (page); +put_page (page); + }return VM_FAULT_MINOR ; }
mm/page-writeback.c +7 -2
@@ -29,6 +29,7 @@#include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/syscalls.h> +#include <linux/rmap.h> /* * The maximum number of pages to writeout in a single bdflush/kupdate
@@ -725,8 +726,10 @@ int test_clear_page_dirty(struct page *ppage_index (page), PAGECACHE_TAG_DIRTY);write_unlock_irqrestore (&mapping->tree_lock, flags); -if (mapping_cap_account_dirty (mapping)) +if (mapping_cap_account_dirty (mapping)) { +page_wrprotect (page);dec_page_state (nr_dirty); + }return 1 ; }write_unlock_irqrestore (&mapping->tree_lock, flags);
@@ -756,8 +759,10 @@ int clear_page_dirty_for_io(struct pageif (mapping) {if (TestClearPageDirty (page)) { -if (mapping_cap_account_dirty (mapping)) +if (mapping_cap_account_dirty (mapping)) { +page_wrprotect (page);dec_page_state (nr_dirty); + }return 1 ; }return 0 ;
mm/rmap.c +66 -0
72 lines not on this page · raw message
include/linux/rmap.h +6 -0
@@ -105,6 +105,12 @@ pte_t *page_check_address(struct page *,*/ unsigned long page_address_in_vma (struct page *,struct vm_area_struct *); +/* +* Used to writeprotect clean pages, in order to count nr_dirty for shared +* mappings +*/ +int page_wrprotect (struct page *); +#else /* !CONFIG_MMU */ #define anon_vma_init () do {} while (0)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/2] tracking dirty pages in shared mappings -V3
mm/memory.c static int do_no_page(struct mm_struct * · line 2081
Quoted 5 lines · Peter Zijlstra
mm/memory.c +1
-0@@ -2077,6 +2078,7 @@ static int do_no_page(struct mm_struct *unsigned int sequence =0 ;int ret =VM_FAULT_MINOR ;int anon =0 ; +int dirty =0 ;
dirtied_page = NULL ?
mm/memory.c retry: · line 2154
Quoted 4 lines
mm/memory.c +0
-0@@ -2150,6 +2152,11 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma);
A write fault to a shared mapping does not make the page dirty, just the pte?
mm/memory.c retry: · line 2171
Quoted 5 lines
mm/memory.c +3
-0inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); +if (write_access) { +get_page (new_page); + dirty++;
dirtied_page = new_page; ? get_page(dirtied_page); ?
mm/memory.c retry: · line 2188
Quoted 4 lines
mm/memory.c +4
-0+if (dirty) { +set_page_dirty (new_page); +put_page (new_page); + }
if (dirtied_page)
set_page_dirty(dirtied_page); put_page(dirtied_page)
?
mm/memory.c static inline int handle_pte_fault(struc · line 2253
Quoted 5 lines
mm/memory.c +1
-0@@ -2235,6 +2250,7 @@ static inline int handle_pte_fault(strucpte_t entry;pte_t old_entry;spinlock_t *ptl; +struct page *page =NULL ;
use dirtied_page instead to make it the same as the other function?
mm/rmap.c int page_referenced(struct page *page, i · line 533
Quoted 1 line
mm/rmap.c +1
-0+int page_wrprotect (struct page *page)
The above and related functions look similar to code in rmap.c and migrate.c. Could those be consolidated?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/2] tracking dirty pages in shared mappings -V3
Quoted 8 lines · Christoph Lameter
mm/memory.c static int do_no_page(struct mm_struct * · line 2081
Quoted 5 lines · Peter Zijlstra
mm/memory.c +1
-0@@ -2077,6 +2078,7 @@ static int do_no_page(struct mm_struct *unsigned int sequence =0 ;int ret =VM_FAULT_MINOR ;int anon =0 ; +int dirty =0 ;
dirtied_page= NULL ?
Much nicer indeed!
Quoted 7 lines
mm/memory.c retry: · line 2154
Quoted 4 lines
mm/memory.c +0
-0@@ -2150,6 +2152,11 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma);A write fault to a shared mapping does not make the page dirty, just the pte?
We do that here:
Quoted 8 lines
Quoted 5 lines
mm/memory.c +3
-0inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); +if (write_access) { +get_page (new_page); + dirty++;
Quoted 4 lines
mm/rmap.c int page_referenced(struct page *page, i · line 533
Quoted 1 line
mm/rmap.c +1
-0+int page_wrprotect (struct page *page)The above and related functions look similar to code in
rmap.candmigrate.c. Could those be consolidated?
I'll have a look.
Peter
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH 1/3 tracking dirty pages in shared mappings -V4
In reply to Christoph LameterFrom: Peter Zijlstra <[email protected]>
People expressed the need to track dirty pages in shared mappings.
Linus outlined the general idea of doing that through making clean writable pages write-protected and taking the write fault.
This patch does exactly that, it makes pages in a shared writable mapping write-protected. On write-fault the pages are marked dirty and made writable. When the pages get synced with their backing store, the write-protection is re-instated.
It survives a simple test and shows the dirty pages in /proc/vmstat.
Changes in -v4:
- small cleanup as suggested by Christoph Lameter.
Changes in -v3:
- move set_page_dirty() outside pte lock (suggested by Christoph Lameter)
Changes in -v2:
- only wrprotect pages from dirty capable mappings. (Nick Piggin)
- move the writefault handling from do_wp_page() into handle_pte_fault(). (Nick Piggin)
- revert to the old install_page interface. (Nick Piggin)
- also clear the pte dirty bit when we make pages read-only again. (spotted by Rik van Riel)
- make page_wrprotect() return the number of reprotected ptes.
---
include/linux/mm.h | 2 + include/linux/rmap.h | 6 ++++ mm/fremap.c | 10 ++++++- mm/memory.c | 34 +++++++++++++++++++++++--- mm/page-writeback.c | 9 +++++- mm/rmap.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 120 insertions(+), 7 deletions(-)
include/linux/mm.h +2 -0
@@ -183,6 +183,8 @@ extern unsigned int kobjsize(const void#define VM_SequentialReadHint (v) ((v)->vm_flags & VM_SEQ_READ)#define VM_RandomReadHint (v) ((v)->vm_flags & VM_RAND_READ) +#define VM_SharedWritable (v) (((v)->vm_flags & (VM_SHARED|VM_WRITE)) == (VM_SHARED|VM_WRITE)) +/* * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask..
mm/fremap.c +8 -2
@@ -15,6 +15,7 @@#include <linux/rmap.h> #include <linux/module.h> #include <linux/syscalls.h> +#include <linux/backing-dev.h> #include <asm/mmu_context.h> #include <asm/cacheflush.h>
@@ -79,9 +80,14 @@ int install_page(struct mm_struct *mm, sinc_mm_counter (mm, file_rss);flush_icache_page (vma, page); -set_pte_at (mm, addr, pte,mk_pte (page, prot)); + pte_val =mk_pte (page, prot); +if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (page); +if (mapping &&mapping_cap_account_dirty (mapping)) + pte_val =pte_wrprotect (pte_val); + } +set_pte_at (mm, addr, pte, pte_val);page_add_file_rmap (page); - pte_val = *pte;update_mmu_cache (vma, addr, pte_val); err =0 ; unlock:
mm/memory.c +31 -3
@@ -49,6 +49,7 @@#include <linux/module.h> #include <linux/init.h> #include <linux/mm_page_replace.h> +#include <linux/backing-dev.h> #include <asm/pgalloc.h> #include <asm/uaccess.h>
@@ -2077,6 +2078,7 @@ static int do_no_page(struct mm_struct *unsigned int sequence =0 ;int ret =VM_FAULT_MINOR ;int anon =0 ; +struct page *dirty_page =NULL ;pte_unmap (page_table);BUG_ON (vma->vm_flags &VM_PFNMAP );
@@ -2150,6 +2152,11 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma); +else if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (new_page); +if (mapping &&mapping_cap_account_dirty (mapping)) + entry =pte_wrprotect (entry); + }set_pte_at (mm, address, page_table, entry);if (anon) {inc_mm_counter (mm, anon_rss);
@@ -2159,6 +2166,10 @@ retry: }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); +if (write_access) { + dirty_page = new_page; +get_page (dirty_page); + } } }else {/* One of our sibling threads was faster, back out. */
@@ -2171,6 +2182,10 @@ retry:lazy_mmu_prot_update (entry); unlock:pte_unmap_unlock (page_table, ptl); +if (dirty_page) { +set_page_dirty (dirty_page); +put_page (dirty_page); + }return ret; oom:page_cache_release (new_page);
@@ -2235,6 +2250,7 @@ static inline int handle_pte_fault(strucpte_t entry;pte_t old_entry;spinlock_t *ptl; +struct page *dirty_page =NULL ; old_entry = entry = *pte;if (!pte_present (entry)) {
@@ -2257,12 +2273,20 @@ static inline int handle_pte_fault(strucif (unlikely (!pte_same (*pte, entry))) goto unlock;if (write_access) { -if (!pte_write (entry)) -return do_wp_page (mm, vma, address, - pte, pmd, ptl, entry); +if (!pte_write (entry)) { +if (!VM_SharedWritable (vma)) { +return do_wp_page (mm, vma, address, + pte, pmd, ptl, entry); + }else { + entry =pte_mkwrite (entry); + dirty_page =vm_normal_page (vma, address, entry); +get_page (dirty_page); + } + } entry =pte_mkdirty (entry); } entry =pte_mkyoung (entry); +if (!pte_same (old_entry, entry)) {ptep_set_access_flags (vma, address, pte, entry, write_access);update_mmu_cache (vma, address, entry);
@@ -2279,6 +2303,10 @@ static inline int handle_pte_fault(struc } unlock:pte_unmap_unlock (pte, ptl); +if (dirty_page) { +set_page_dirty (dirty_page); +put_page (dirty_page); + }return VM_FAULT_MINOR ; }
mm/page-writeback.c +7 -2
@@ -29,6 +29,7 @@#include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/syscalls.h> +#include <linux/rmap.h> /* * The maximum number of pages to writeout in a single bdflush/kupdate
@@ -725,8 +726,10 @@ int test_clear_page_dirty(struct page *ppage_index (page), PAGECACHE_TAG_DIRTY);write_unlock_irqrestore (&mapping->tree_lock, flags); -if (mapping_cap_account_dirty (mapping)) +if (mapping_cap_account_dirty (mapping)) { +page_wrprotect (page);dec_page_state (nr_dirty); + }return 1 ; }write_unlock_irqrestore (&mapping->tree_lock, flags);
@@ -756,8 +759,10 @@ int clear_page_dirty_for_io(struct pageif (mapping) {if (TestClearPageDirty (page)) { -if (mapping_cap_account_dirty (mapping)) +if (mapping_cap_account_dirty (mapping)) { +page_wrprotect (page);dec_page_state (nr_dirty); + }return 1 ; }return 0 ;
mm/rmap.c +66 -0
72 lines not on this page · raw message
include/linux/rmap.h +6 -0
@@ -105,6 +105,12 @@ pte_t *page_check_address(struct page *,*/ unsigned long page_address_in_vma (struct page *,struct vm_area_struct *); +/* +* Used to writeprotect clean pages, in order to count nr_dirty for shared +* mappings +*/ +int page_wrprotect (struct page *); +#else /* !CONFIG_MMU */ #define anon_vma_init () do {} while (0)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Peter" == Peter Zijlstra <[email protected]> writes:
Peter> From: Peter Zijlstra <[email protected]> People expressed Peter> the need to track dirty pages in shared mappings.
Peter> Linus outlined the general idea of doing that through making Peter> clean writable pages write-protected and taking the write Peter> fault.
What does this do to performance on TPC workloads? How many extra faults are there likely to be?
--
Dr Peter Chubb http://www.gelato.unsw.edu.au peterc AT gelato.unsw.edu.au
http://www.ertos.nicta.com.au ERTOS within National ICT Australia
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 6 lines · Peter Chubb
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
"Peter" == Peter Zijlstra <[email protected]> writes:
Peter> From: Peter Zijlstra <[email protected]> People expressed Peter> the need to track dirty pages in shared mappings.
Quoted 6 lines
Peter> Linus outlined the general idea of doing that through making Peter> clean writable pages write-protected and taking the write Peter> fault.
What does this do to performance on TPC workloads? How many extra faults are there likely to be?
They all use large pages anyway ...
M.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 11 lines · Martin Bligh
Quoted 3 lines · Peter Chubb
What does this do to performance on TPC workloads? How many extra faults are there likely to be?
They all use large pages anyway ...
M.
That much is true, but let me make sure I understand how this will impact customer transaction-processing workloads, where hugepages are infrequently used.
Under the proposed scheme, when the database starts up, it will take a write fault when it initializes a shared memory page, and the page will be marked as dirty. Then, as long as the page is never swapped out, the write protection will never be "re-armed".
Since customers usually tune their databases to never swap, then the only potential additional cost I would envision would be the small cost of initially marking the page as write-protected, and marking the page as dirty during the first fault.
Does this sound reasonable ?
Cheers,
Brian
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Martin BlighQuoted 6 lines · Martin Bligh
Quoted 3 lines · Peter Chubb
What does this do to performance on TPC workloads? How many extra faults are there likely to be?
They all use large pages anyway ...
Yes, but we should really answer that question anyway.
I don't think anybody has really objected to this patch series, and it clearly seems to fix something (if nothing else, then Peter's test-program ;), and for fairly obvious reasons _I_ approve of it.
But I've been asking people to benchmark it anyway. Becuase it _will_ hurt users of shared writable mappings, even if it's hopefully less of an issue today thanks to large pages.
Now, the fact that it will hurt them is not a total disaster, since we already know that there are ways to mitigate it. However, it's still true that we should have the numbers, if only to perhaps be able to say that we don't care.
Linus
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 10 lines · Linus Torvalds
Quoted 6 lines · Martin Bligh
Quoted 2 lines · Peter Chubb
What does this do to performance on TPC workloads? How many extra faults are there likely to be?
They all use large pages anyway ...
Quoted 7 lines
Yes, but we should really answer that question anyway.
I don't think anybody has really objected to this patch series, and it clearly seems to fix something (if nothing else, then Peter's test-program ;), and for fairly obvious reasons _I_ approve of it.
But I've been asking people to benchmark it anyway. Becuase it _will_ hurt users of shared writable mappings, even if it's hopefully less of an issue today thanks to large pages.
Now, the fact that it will hurt them is not a total disaster, since we already know that there are ways to mitigate it. However, it's still true that we should have the numbers, if only to perhaps be able to say that we don't care.
Those filesystems (shmem, presumably hugetlbfs) that are !mapping_cap_account_dirty should not notice any difference, except perhaps for the several extra branches and bytes of icahce.
Or do these databases use regular filesystem backed shared files?
Signature · 8 lines
SUSE Labs, Novell Inc. Send instant messages to your online friends http://au.messenger.yahoo.com -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 2 lines · Nick Piggin
Or do these databases use regular filesystem backed shared files?
Some definitely do.
I don't work much with them, but I'm pretty sure, that's exactly what Berkeley DB does. Probably SQLite too (and maybe MySQL?).
And I know that nntpd used to map its "newsgroup list" (which is a database, although not a relational one, but a really stupid line-based plain-text setup).
So yes, we'll impact performance. Whether it will be really noticeable or not, who knows.
The only way to find out may be to just apply the patches (which I'm currently planning on doing early after 2.6.17 unless we can get data otherwise) and see who screams, if anybody.
Linus
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
From: Peter Zijlstra <[email protected]>
People expressed the need to track dirty pages in shared mappings.
Linus outlined the general idea of doing that through making clean writable pages write-protected and taking the write fault.
Quoted 6 lines
This patch does exactly that, it makes pages in a shared writable mapping write-protected. On write-fault the pages are marked dirty and made writable. When the pages get synced with their backing store, the write-protection is re-instated.
It survives a simple test and shows the dirty pages in
/proc/vmstat.
It'd be nice to have more that a "simple test" done. Bugs in this area will be subtle and will manifest in unpleasant ways. That goes for both correctness and performance bugs.
Quoted 33 lines
Index:
linux-2.6/mm/memory.c
===================================================================mm/memory.c +7
-0@@ -49,6 +49,7 @@#include <linux/module.h> #include <linux/init.h> #include <linux/mm_page_replace.h> +#include <linux/backing-dev.h> #include <asm/pgalloc.h> #include <asm/uaccess.h> @@ -2077,6 +2078,7 @@ static int do_no_page(struct mm_struct *unsigned int sequence =0 ;int ret =VM_FAULT_MINOR ;int anon =0 ; +struct page *dirty_page =NULL ;pte_unmap (page_table);BUG_ON (vma->vm_flags &VM_PFNMAP );@@ -2150,6 +2152,11 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma); +else if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (new_page); +if (mapping &&mapping_cap_account_dirty (mapping)) + entry =pte_wrprotect (entry); + }set_pte_at (mm, address, page_table, entry);if (anon) {inc_mm_counter (mm, anon_rss);
mm/memory.c retry: · line 2172
Quoted 7 lines
mm/memory.c +4
-0@@ -2159,6 +2166,10 @@ retry: }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); +if (write_access) { + dirty_page = new_page; +get_page (dirty_page); + }
So let's see. We take a write fault, we mark the page dirty then we return to userspace which will proceed with the write and will mark the pte dirty.
Later, the VM will write the page out.
Later still, the pte will get cleaned by reclaim or by munmap or whatever and the page will be marked dirty and the page will again be written out. Potentially needlessly.
How much extra IO will we be doing because of this change?
Quoted 56 lines
mm/memory.c +0
-0return 0 ;Index:
linux-2.6/mm/rmap.c
===================================================================mm/rmap.c +47
-0@@ -478,6 +478,72 @@ int page_referenced(struct page *page, ireturn referenced; } +static int page_wrprotect_one (struct page *page,struct vm_area_struct *vma) +{ +struct mm_struct *mm = vma->vm_mm; +unsigned long address; +pte_t *pte, entry; +spinlock_t *ptl; +int ret =0 ; + + address =vma_address (page, vma); +if (address == -EFAULT ) + goto out; + + pte =page_check_address (page, mm, address, &ptl); +if (!pte) + goto out; + +if (!pte_write (*pte)) + goto unlock; + + entry =pte_mkclean (pte_wrprotect (*pte)); +ptep_establish (vma, address, pte, entry); +update_mmu_cache (vma, address, entry); +lazy_mmu_prot_update (entry); + ret =1 ; + +unlock: +pte_unmap_unlock (pte, ptl); +out: +return ret; +} + +static int page_wrprotect_file (struct page *page) +{ +struct address_space *mapping = page->mapping; +pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT -PAGE_SHIFT ); +struct vm_area_struct *vma; +struct prio_tree_iter iter; +int ret =0 ; + +BUG_ON (PageAnon (page)); + +spin_lock (&mapping->i_mmap_lock); + + vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { +if (VM_SharedWritable (vma)) + ret +=page_wrprotect_one (page, vma); + }
mm/rmap.c int page_referenced(struct page *page, i · line 537
Quoted 10 lines
mm/rmap.c +10
-0+ +spin_unlock (&mapping->i_mmap_lock); +return ret; +} + +int page_wrprotect (struct page *page) +{ +int ret =0 ; + +BUG_ON (!PageLocked (page));
hm. So clear_page_dirty() and clear_page_dirty_for_io() are only ever called against a locked page? I guess that makes sense, but it's not a guarantee which we had in the past. It really _has_ to be true, because lock_page() is the only thing which can protect the address_space from memory reclaim in those two functions.
Oh well. We'll find out if people's machines start to go BUG.
mm/rmap.c int page_referenced(struct page *page, i · line 539
Quoted 1 line
mm/rmap.c +1
-0+if (page_mapped (page) && page->mapping) {
umm, afaict this function can be called for swapcache pages and Bad Things will happen. I think we need page_mapping(page) here?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 15 lines · Andrew Morton
Quoted 13 lines · Peter Zijlstra
From: Peter Zijlstra <[email protected]>
People expressed the need to track dirty pages in shared mappings.
Linus outlined the general idea of doing that through making clean writable pages write-protected and taking the write fault.
This patch does exactly that, it makes pages in a shared writable mapping write-protected. On write-fault the pages are marked dirty and made writable. When the pages get synced with their backing store, the write-protection is re-instated.
Quoted 6 lines
Quoted 1 line
It survives a simple test and shows the dirty pages in
/proc/vmstat.It'd be nice to have more that a "simple test" done. Bugs in this area will be subtle and will manifest in unpleasant ways. That goes for both correctness and performance bugs.
I'll kick off some testing of this stack and see what occurs. Should appear on t.k.o in due time.
Cheers.
-apw
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Andrew MortonQuoted 5 lines · Andrew Morton
Quoted 1 line
It survives a simple test and shows the dirty pages in
/proc/vmstat.It'd be nice to have more that a "simple test" done. Bugs in this area will be subtle and will manifest in unpleasant ways. That goes for both correctness and performance bugs.
Standard tests such as AIM7 will not trigger these paths. It is rather unusual for small unix processes to have a shared writable mapping and therefore I doubt that the typical benchmarks may show much of a difference. These types of mappings are more typical for large or specialized apps. Be sure that the tests actually do dirty pages in shared writeable mappings.
Quoted 11 lines
mm/rmap.c int page_referenced(struct page *page, i · line 537
Quoted 5 lines
mm/rmap.c +5
-0+int page_wrprotect (struct page *page) +{ +int ret =0 ; + +BUG_ON (!PageLocked (page));hm. So
clear_page_dirty()andclear_page_dirty_for_io()are only ever called against a locked page? I guess that makes sense, but it's not a guarantee which we had in the past. It really _has_ to be true, becauselock_page()is the only thing which can protect theaddress_spacefrom memory reclaim in those two functions.
If that is true then we can get rid of atomic ops in both functions.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 6 lines · Christoph Lameter
Quoted 4 lines · Andrew Morton
It'd be nice to have more that a "simple test" done. Bugs in this area will be subtle and will manifest in unpleasant ways. That goes for both correctness and performance bugs.
Quoted 6 lines
Standard tests such as AIM7 will not trigger these paths. It is rather unusual for small unix processes to have a shared writable mapping and therefore I doubt that the typical benchmarks may show much of a difference. These types of mappings are more typical for large or specialized apps. Be sure that the tests actually do dirty pages in shared writeable mappings.
What happened to the VM stress-test programs that we used to test the page-out with? I forget who kept a collection of them around, but they did things like trying to cause MM problems on purpose. And I'm pretty sure some of the nastiest ones used shared mappings, exactly because we've had problems with the virtual scanning.
I have a very distinct memory of somebody (I'd like to say Con, but that's probably bogus) collecting a few programs that were known to cause nasty problems (like the system just becoming totally unresponsive). For checking that things degraded reasonably before getting killed by OOM.
I'm talking the 2.4.x timeframe, so it's a few years ago. It might not be a real _benchmark_ per se, but I think it would be an interesting data-point whether the system acts "better" with some of those tests..
Linus
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 4 lines · Linus Torvalds
What happened to the VM stress-test programs that we used to test the page-out with? I forget who kept a collection of them around, but they did things like trying to cause MM problems on purpose.
I think that was me, back in my programming days.
Quoted 3 lines
And I'm pretty sure
some of the nastiest ones used shared mappings, exactly because we've had problems with the virtual scanning.
http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
run-bash-shared-mapping.sh is a good stress-tester and deadlock-finder.
Running fsx-linux (in mmap-read and mmap-write and read and write mode) in combination with memory pressure is a good correctness-tester. Needs to be run on various filesystems too.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 2 lines · Andrew Morton
I think that was me, back in my programming days.
How times flies ;)
usemem was one of the tools I was thinking of, so this may well be it.
Linus
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Andrew MortonQuoted 15 lines · Andrew Morton
Quoted 3 lines · Linus Torvalds
What happened to the VM stress-test programs that we used to test the page-out with? I forget who kept a collection of them around, but they did things like trying to cause MM problems on purpose.
I think that was me, back in my programming days.
Quoted 3 lines
And I'm pretty sure
some of the nastiest ones used shared mappings, exactly because we've had problems with the virtual scanning.
Quoted 7 lines
http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
run-bash-shared-mapping.shis a good stress-tester and deadlock-finder.Running fsx-linux (in mmap-read and mmap-write and read and write mode) in combination with memory pressure is a good correctness-tester. Needs to be run on various filesystems too.
Well for what its worth (and from this thread it may not be that much) the testing I did over night shows green across all the test boxes I have. The tests do include fsx-linux across a limited range of filesystems.
I'll see if I can get the other one done.
-apw
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 3 lines
Well for what its worth (and from this thread it may not be that much) the testing I did over night shows green across all the test boxes I have. The tests do include fsx-linux across a limited range of filesystems.
There's no perf regressions anywhere in there either (across dbench, reaim, kernbench, tbench, at least) on a multitude of machines ...
M.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Andrew MortonQuoted 10 lines · Andrew Morton
Quoted 3 lines · Linus Torvalds
What happened to the VM stress-test programs that we used to test the page-out with? I forget who kept a collection of them around, but they did things like trying to cause MM problems on purpose.
I think that was me, back in my programming days.
Quoted 8 lines
Quoted 3 lines
And I'm pretty sure
some of the nastiest ones used shared mappings, exactly because we've had problems with the virtual scanning.http://www.zip.com.au/~akpm/linux/patches/stuff/ext3-tools.tar.gz
run-bash-shared-mapping.shis a good stress-tester and deadlock-finder.
Well my amd64 box ran run-bash-shared-mapping.sh for 6 hours without a peep or oops. Machine seemed ok at the end.
-apw
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Andrew MortonQuoted 68 lines · Andrew Morton
Quoted 8 lines · Peter Zijlstra
From: Peter Zijlstra <[email protected]>
People expressed the need to track dirty pages in shared mappings.
Linus outlined the general idea of doing that through making clean writable pages write-protected and taking the write fault.
Quoted 7 lines
This patch does exactly that, it makes pages in a shared writable mapping write-protected. On write-fault the pages are marked dirty and made writable. When the pages get synced with their backing store, the write-protection is re-instated.
It survives a simple test and shows the dirty pages in
/proc/vmstat.It'd be nice to have more that a "simple test" done. Bugs in this area will be subtle and will manifest in unpleasant ways. That goes for both correctness and performance bugs.
Quoted 33 lines
Index:
linux-2.6/mm/memory.c
===================================================================mm/memory.c +7
-0@@ -49,6 +49,7 @@#include <linux/module.h> #include <linux/init.h> #include <linux/mm_page_replace.h> +#include <linux/backing-dev.h> #include <asm/pgalloc.h> #include <asm/uaccess.h> @@ -2077,6 +2078,7 @@ static int do_no_page(struct mm_struct *unsigned int sequence =0 ;int ret =VM_FAULT_MINOR ;int anon =0 ; +struct page *dirty_page =NULL ;pte_unmap (page_table);BUG_ON (vma->vm_flags &VM_PFNMAP );@@ -2150,6 +2152,11 @@ retry: entry =mk_pte (new_page, vma->vm_page_prot);if (write_access) entry =maybe_mkwrite (pte_mkdirty (entry), vma); +else if (VM_SharedWritable (vma)) { +struct address_space *mapping =page_mapping (new_page); +if (mapping &&mapping_cap_account_dirty (mapping)) + entry =pte_wrprotect (entry); + }set_pte_at (mm, address, page_table, entry);if (anon) {inc_mm_counter (mm, anon_rss);mm/memory.c retry: · line 2173
Quoted 8 lines
mm/memory.c +4
-0@@ -2159,6 +2166,10 @@ retry: }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (new_page); +if (write_access) { + dirty_page = new_page; +get_page (dirty_page); + }So let's see. We take a write fault, we mark the page dirty then we return to userspace which will proceed with the write and will mark the pte dirty.
Quoted 5 lines
Later, the VM will write the page out.
Later still, the pte will get cleaned by reclaim or by munmap or whatever
and the page will be marked dirty and the page will again be written out. Potentially needlessly.
page_wrprotect also marks the page clean, so this window is very small. The window is that the fault path might set_page_dirty, then throttle on writeout, and the page gets written out before it really gets dirtied by the application (which then has to fault again).
Quoted 3 lines
How much extra IO will we be doing because of this change?
Of course it can do potentially quite a lot more IO in some cases, if an application likes to dirty a working set larger than the writeout thresholds... the same scenario as write(2) has now.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 4 lines · Nick Piggin
Quoted 4 lines
So let's see. We take a write fault, we mark the page dirty then we return to userspace which will proceed with the write and will mark the pte dirty.
Quoted 8 lines
Quoted 6 lines
Later, the VM will write the page out.
Later still, the pte will get cleaned by reclaim or by munmap or whatever and the page will be marked dirty and the page will again be written out. Potentially needlessly.
page_wrprotectalso marks the page clean,
Oh. I missed that when reading the comment which describes page_wrprotect() (I do go on).
Quoted 4 lines
so this window is very small.
The window is that the fault path mightset_page_dirty, then throttle on writeout, and the page gets written out before it really gets dirtied by the application (which then has to fault again).
: int test_clear_page_dirty(struct page *page)
: {
: struct address_space *mapping = page_mapping(page);
: unsigned long flags;
:
: if (mapping) {
: write_lock_irqsave(&mapping->tree_lock, flags);
: if (TestClearPageDirty(page)) {
: radix_tree_tag_clear(&mapping->page_tree,
: page_index(page),
: PAGECACHE_TAG_DIRTY);
: write_unlock_irqrestore(&mapping->tree_lock, flags);
: /*
: * We can continue to use `mapping' here because the
: * page is locked, which pins the address_space
: */
So if userspace modifies the page right here, and marks the pte dirty.
: if (mapping_cap_account_dirty(mapping)) {
: page_wrprotect(page);
We just lost that pte dirty bit, and hence the user's data.
: dec_page_state(nr_dirty);
: }
: return 1;
: }
: write_unlock_irqrestore(&mapping->tree_lock, flags);
: return 0;
: }
: return TestClearPageDirty(page);
: }
:
Which is just the sort of subtle and nasty problem I was referring to...
If that's correct then I guess we need the
if (ptep_clear_flush_dirty(vma, addr, pte) ||
page_test_and_clear_dirty(page))
ret += set_page_dirty(page);treatment in page_wrprotect().
Now I suppose it's not really a dataloss race, because in practice the kernel is about to write this page to backing store anwyay. I guess. I cannot immediately think of any clear_page_dirty() callers for whom that won't be true.
Someone please convince me that this has all been thought about and is solid as a rock.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 11 lines · Andrew Morton
Quoted 9 lines · Nick Piggin
Quoted 8 lines
So let's see. We take a write fault, we mark the page dirty then we return to userspace which will proceed with the write and will mark the pte dirty.
Later, the VM will write the page out.
Later still, the pte will get cleaned by reclaim or by munmap or whatever
and the page will be marked dirty and the page will again be written out. Potentially needlessly.
Quoted 6 lines
Quoted 2 lines
page_wrprotectalso marks the page clean,Oh. I missed that when reading the comment which describes
page_wrprotect()(I do go on).
I guess page_wrprotect isn't a good name, because it would suggest it can be used in situations where it would cause data loss.
page_mappings_mkclean or page_mkclean might be better (the wrprotect is just a side effect of the fact that clean,shared mappings are readonly).
Quoted 49 lines
Quoted 6 lines
so this window is very small.
The window is that the fault path mightset_page_dirty, then throttle on writeout, and the page gets written out before it really gets dirtied by the application (which then has to fault again).: int
test_clear_page_dirty(struct page *page)
: {
: structaddress_space*mapping =page_mapping(page);
: unsigned long flags;
: : if (mapping) {
:write_lock_irqsave(&mapping->tree_lock, flags);
: if (TestClearPageDirty(page)) {
: radix_tree_tag_clear(&mapping->page_tree,
:page_index(page),
:PAGECACHE_TAG_DIRTY);
:write_unlock_irqrestore(&mapping->tree_lock, flags);
: /*
: * We can continue to use `mapping' here because the
: * page is locked, which pins theaddress_space
: */So if userspace modifies the page right here, and marks the pte dirty.
: if (
mapping_cap_account_dirty(mapping)) {
:page_wrprotect(page);We just lost that pte dirty bit, and hence the user's data.
:
dec_page_state(nr_dirty);
: }
: return 1;
: }
:write_unlock_irqrestore(&mapping->tree_lock, flags);
: return 0;
: }
: return TestClearPageDirty(page);
: }
:Which is just the sort of subtle and nasty problem I was referring to...
If that's correct then I guess we need the
if (ptep_clear_flush_dirty(vma, addr, pte) || page_test_and_clear_dirty(page)) ret += set_page_dirty(page);
Quoted 7 lines
treatment in
page_wrprotect().Now I suppose it's not really a dataloss race, because in practice the kernel is about to write this page to backing store anwyay. I guess. I cannot immediately think of any
clear_page_dirty()callers for whom that won't be true.
If they do a clear_page_dirty, then fail to clean the page, then fail to subsequently set_page_dirty again, it is a data-loss situation anyway.
If they do set_page_dirty (which they have to, for correctness), then the page has PG_dirty set again; true dirty bits are moved out of the ptes, but that's no problem.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Andrew MortonQuoted 12 lines · Andrew Morton
Quoted 11 lines · Nick Piggin
Quoted 10 lines
So let's see. We take a write fault, we mark the page dirty then we return to userspace which will proceed with the write and will mark the pte dirty.
Later, the VM will write the page out.
Later still, the pte will get cleaned by reclaim or by munmap or whatever and the page will be marked dirty and the page will again be written out. Potentially needlessly.
Quoted 4 lines
Quoted 1 line
page_wrprotectalso marks the page clean,Oh. I missed that when reading the comment which describes
page_wrprotect()(I do go on).
Yes, this name is not the best of names :-(
I was aware of this, but since in my mind the counting through protection
faults was the prime idea, I stuck to page_wrprotect().
But I'm hard pressed to come up with a better one. Nick proposes:
page_mkclean()
But that also doesn't cover the whole of it from my perspective.
Quoted 22 lines
Quoted 4 lines
so this window is very small.
The window is that the fault path mightset_page_dirty, then throttle on writeout, and the page gets written out before it really gets dirtied by the application (which then has to fault again).: int
test_clear_page_dirty(struct page *page)
: {
: structaddress_space*mapping =page_mapping(page);
: unsigned long flags;
:
: if (mapping) {
:write_lock_irqsave(&mapping->tree_lock, flags);
: if (TestClearPageDirty(page)) {
: radix_tree_tag_clear(&mapping->page_tree,
:page_index(page),
:PAGECACHE_TAG_DIRTY);
:write_unlock_irqrestore(&mapping->tree_lock, flags);
: /*
: * We can continue to use `mapping' here because the
: * page is locked, which pins theaddress_space
: */
Quoted 6 lines
So if userspace modifies the page right here, and marks the pte dirty.
: if (
mapping_cap_account_dirty(mapping)) {
:page_wrprotect(page);We just lost that pte dirty bit, and hence the user's data.
I thought that at the time we clean PAGECACHE_TAG_DIRTY the page is in flight to disk.
Now that I look at it again, perhaps the page_wrprotect() call in clear_page_dirty_for_io()
should be in test_set_page_writeback().
Quoted 13 lines
:
dec_page_state(nr_dirty);
: }
: return 1;
: }
:write_unlock_irqrestore(&mapping->tree_lock, flags);
: return 0;
: }
: return TestClearPageDirty(page);
: }
:Which is just the sort of subtle and nasty problem I was referring to...
Quoted 7 lines
If that's correct then I guess we need the
if (ptep_clear_flush_dirty(vma, addr, pte) || page_test_and_clear_dirty(page)) ret += set_page_dirty(page);treatment in
page_wrprotect().
I would make that a BUG_ON(); the only way for a pte of a shared mapping to become
dirty is through the fault handler, and that should already call set_page_dirty() on it.
Quoted 7 lines
Now I suppose it's not really a dataloss race, because in practice the kernel is about to write this page to backing store anwyay. I guess. I cannot immediately think of any
clear_page_dirty()callers for whom that won't be true.Someone please convince me that this has all been thought about and is solid as a rock.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 19 lines · Peter Zijlstra
Quoted 7 lines · Andrew Morton
Quoted 5 lines · Nick Piggin
Quoted 5 lines
So let's see. We take a write fault, we mark the page dirty then we return to userspace which will proceed with the write and will mark the pte dirty.
Later, the VM will write the page out.
Quoted 9 lines
Quoted 6 lines
Quoted 3 lines
Later still, the pte will get cleaned by reclaim or by munmap or whatever
and the page will be marked dirty and the page will again be written out. Potentially needlessly.
page_wrprotectalso marks the page clean,Oh. I missed that when reading the comment which describes
page_wrprotect()(I do go on).
Quoted 9 lines
Yes, this name is not the best of names :-(
I was aware of this, but since in my mind the counting through protection faults was the prime idea, I stuck to
page_wrprotect().But I'm hard pressed to come up with a better one. Nick proposes:
page_mkclean()
But that also doesn't cover the whole of it from my perspective.
What's your perspective?
With mmap shared accounting, the _whole VM's_ perspective is that clean MAP_SHARED ptes are marked readonly.
The logical operation is marking the page's ptes clean. The VM mechanism also marks the ptes readonly as a side effect of that. Think about it:
writeback does not want to make the page write protected, it wants to make it clean.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
Quoted 35 lines · Nick Piggin
Quoted 22 lines · Peter Zijlstra
Quoted 12 lines · Andrew Morton
Quoted 10 lines · Nick Piggin
Quoted 9 lines
So let's see. We take a write fault, we mark the page dirty then we return to userspace which will proceed with the write and will mark the pte dirty.
Later, the VM will write the page out.
Later still, the pte will get cleaned by reclaim or by munmap or whatever and the page will be marked dirty and the page will again be written out. Potentially needlessly.
Quoted 5 lines
Quoted 2 lines
page_wrprotectalso marks the page clean,Oh. I missed that when reading the comment which describes
page_wrprotect()(I do go on).Yes, this name is not the best of names :-(
Quoted 8 lines
I was aware of this, but since in my mind the counting through protection
faults was the prime idea, I stuck topage_wrprotect().But I'm hard pressed to come up with a better one. Nick proposes:
page_mkclean()
But that also doesn't cover the whole of it from my perspective.What's your perspective?
Quoted 7 lines
With mmap shared accounting, the
_wholeVM's_ perspective is that cleanMAP_SHAREDptes are marked readonly.The logical operation is marking the page's ptes clean. The VM mechanism also marks the ptes readonly as a side effect of that. Think about it:
writeback does not want to make the page write protected, it wants to make it clean.
As said, I was looking at the added functionality on its own; that is, counting dirty pages by trapping write faults.
However your view; the big picture; does make more sense. I shall rename.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
Quoted 3 lines · Andrew Morton
We just lost that pte dirty bit, and hence the user's data.
I thought that at the time we clean
PAGECACHE_TAG_DIRTYthe page is in flight to disk.
No.
Quoted 4 lines
Now that I look at it again, perhaps the
page_wrprotect()call inclear_page_dirty_for_io()
should be intest_set_page_writeback().
No. The logical operation is clearing the dirty bits from the ptes. Such an operation would be valid even if we didn't set the ptes readonly.
And clearing dirty belongs in clear_page_dirty.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 1/3] tracking dirty pages in shared mappings -V4
In reply to Andrew MortonQuoted 2 lines · Andrew Morton
So let's see. We take a write fault, we mark the page dirty then we return to userspace which will proceed with the write and will mark the pte dirty.
The pte is marked dirty when the page is marked dirty.
Quoted 3 lines
Later still, the pte will get cleaned by reclaim or by munmap or whatever and the page will be marked dirty and the page will again be written out. Potentially needlessly.
But consistent with the way write() works on page buffers. It is rather surprising that one can dirty lots of mmapped pages and they are only written out when the process terminates.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH 2/3 throttle writers of shared mappings
In reply to Christoph LameterFrom: Peter Zijlstra <[email protected]>
Now that we can detect writers of shared mappings, throttle them. Avoids OOM by surprise.
---
mm/memory.c | 7 +++++++ 1 file changed, 7 insertions(+)
mm/memory.c +7 -0
@@ -50,6 +50,7 @@#include <linux/init.h> #include <linux/mm_page_replace.h> #include <linux/backing-dev.h> +#include <linux/writeback.h> #include <asm/pgalloc.h> #include <asm/uaccess.h>
@@ -2183,8 +2184,11 @@ retry: unlock:pte_unmap_unlock (page_table, ptl);if (dirty_page) { +struct address_space *mapping =page_mapping (dirty_page);set_page_dirty (dirty_page);put_page (dirty_page); +if (mapping) +balance_dirty_pages_ratelimited_nr (mapping,1 ); }return ret; oom:
@@ -2304,8 +2308,11 @@ static inline int handle_pte_fault(struc unlock:pte_unmap_unlock (pte, ptl);if (dirty_page) { +struct address_space *mapping =page_mapping (dirty_page);set_page_dirty (dirty_page);put_page (dirty_page); +if (mapping) +balance_dirty_pages_ratelimited_nr (mapping,1 ); }return VM_FAULT_MINOR ; }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 2/3] throttle writers of shared mappings
Review
Quoted 11 lines · Peter Zijlstra
(quoted diff) +0
-0@@ -2304,8 +2308,11 @@ static inline int handle_pte_fault(strucunlock:
pte_unmap_unlock(pte, ptl); if (dirty_page) {(quoted diff) +1
-0+ struct address_space *mapping = page_mapping(dirty_page);set_page_dirty(dirty_page); put_page(dirty_page);(quoted diff) +2
-0+ if (mapping) + balance_dirty_pages_ratelimited_nr(mapping, 1);
Just use balance_dirty_pages()
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 2/3] throttle writers of shared mappings
Quoted 15 lines · Nick Piggin
mm/memory.c retry: · line 2191
Quoted 11 lines · Peter Zijlstra
(quoted diff) +0
-0@@ -2304,8 +2308,11 @@ static inline int handle_pte_fault(strucunlock:
pte_unmap_unlock(pte, ptl); if (dirty_page) {mm/memory.c +3
-0+struct address_space *mapping =page_mapping (dirty_page);set_page_dirty (dirty_page);put_page (dirty_page); +if (mapping) +balance_dirty_pages_ratelimited_nr (mapping,1 );Just use
balance_dirty_pages()
Err.. balance_dirty_pages_ratelimited();
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC][PATCH 2/3] throttle writers of shared mappings
Quoted 4 lines · Nick Piggin
Quoted 1 line · Nick Piggin
Just use
balance_dirty_pages()Err..
balance_dirty_pages_ratelimited();
Ah, tucked away in the header :-)
From: Peter Zijlstra <[email protected]>
Now that we can detect writers of shared mappings, throttle them. Avoids OOM by surprise.
mm/memory.c | 7 +++++++ 1 file changed, 7 insertions(+)
mm/memory.c +7 -0
@@ -50,6 +50,7 @@#include <linux/init.h> #include <linux/mm_page_replace.h> #include <linux/backing-dev.h> +#include <linux/writeback.h> #include <asm/pgalloc.h> #include <asm/uaccess.h>
@@ -2183,8 +2184,11 @@ retry: unlock:pte_unmap_unlock (page_table, ptl);if (dirty_page) { +struct address_space *mapping =page_mapping (dirty_page);set_page_dirty (dirty_page);put_page (dirty_page); +if (mapping) +balance_dirty_pages_ratelimited (mapping); }return ret; oom:
@@ -2304,8 +2308,11 @@ static inline int handle_pte_fault(struc unlock:pte_unmap_unlock (pte, ptl);if (dirty_page) { +struct address_space *mapping =page_mapping (dirty_page);set_page_dirty (dirty_page);put_page (dirty_page); +if (mapping) +balance_dirty_pages_ratelimited (mapping); }return VM_FAULT_MINOR ; }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH 3/3 optimize follow_pages()
In reply to Christoph LameterFrom: Peter Zijlstra <[email protected]>
Christoph Lameter suggested I pull set_page_dirty() out from under the pte lock.
I reviewed the current calls and found the one in follow_page() a candidate for the same treatment.
---
include/linux/mm.h | 1 + mm/memory.c | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-)
include/linux/mm.h +1 -0
@@ -1015,6 +1015,7 @@ struct page *follow_page(struct vm_area_#define FOLL_TOUCH 0x02/* mark page accessed */ #define FOLL_GET 0x04/* do get_page on page */ #define FOLL_ANON 0x08/* give ZERO_PAGE if no pgtable */ +#define FOLL_DIRTY 0x10/* the page was dirtied */ #ifdef CONFIG_PROC_FS void vm_stat_account (struct mm_struct *,unsigned long ,struct file *,long );
mm/memory.c +9 -3
@@ -962,16 +962,22 @@ struct page *follow_page(struct vm_area_if (unlikely (!page)) goto unlock; -if (flags &FOLL_GET ) +if (flags & (FOLL_GET |FOLL_TOUCH ))get_page (page);if (flags &FOLL_TOUCH ) {if ((flags &FOLL_WRITE ) && !pte_dirty (pte) && !PageDirty (page)) -set_page_dirty (page); -mark_page_accessed (page); + flags |=FOLL_DIRTY ; } unlock:pte_unmap_unlock (ptep, ptl); +if (flags &FOLL_TOUCH ) { +if (flags &FOLL_DIRTY ) +set_page_dirty (page); +mark_page_accessed (page); + } +if (!(flags &FOLL_GET )) +put_page (page); out:return page;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH 3/3 optimize follow_pages()
Ofcourse I got the return paths tangled :-(
---
From: Peter Zijlstra <[email protected]>
Christoph Lameter suggested I pull set_page_dirty() out from under the pte lock.
I reviewed the current calls and found the one in follow_page() a candidate for the same treatment.
---
include/linux/mm.h | 1 + mm/memory.c | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-)
include/linux/mm.h +1 -0
@@ -1015,6 +1015,7 @@ struct page *follow_page(struct vm_area_#define FOLL_TOUCH 0x02/* mark page accessed */ #define FOLL_GET 0x04/* do get_page on page */ #define FOLL_ANON 0x08/* give ZERO_PAGE if no pgtable */ +#define FOLL_DIRTY 0x10/* the page was dirtied */ #ifdef CONFIG_PROC_FS void vm_stat_account (struct mm_struct *,unsigned long ,struct file *,long );
mm/memory.c +14 -3
@@ -962,18 +962,28 @@ struct page *follow_page(struct vm_area_if (unlikely (!page)) goto unlock; -if (flags &FOLL_GET ) +if (flags & (FOLL_GET |FOLL_TOUCH ))get_page (page);if (flags &FOLL_TOUCH ) {if ((flags &FOLL_WRITE ) && !pte_dirty (pte) && !PageDirty (page)) + flags |=FOLL_DIRTY ; + } + +pte_unmap_unlock (ptep, ptl); + +if (flags &FOLL_TOUCH ) { +if (flags &FOLL_DIRTY )set_page_dirty (page);mark_page_accessed (page); } +if (!(flags &FOLL_GET )) +put_page (page); + goto out; + unlock:pte_unmap_unlock (ptep, ptl); -out: -return page; + goto out; no_page_table:/*
@@ -986,6 +996,7 @@ no_page_table:get_page (page);BUG_ON (flags &FOLL_WRITE ); } +out:return page; }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH 2/2 throttle writers of shared mappings
In reply to Christoph LameterFrom: Peter Zijlstra <[email protected]>
Now that we can detect writers of shared mappings, throttle them. Avoids OOM by surprise.
---
mm/memory.c | 7 +++++++ 1 file changed, 7 insertions(+)
mm/memory.c +7 -0
@@ -50,6 +50,7 @@#include <linux/init.h> #include <linux/mm_page_replace.h> #include <linux/backing-dev.h> +#include <linux/writeback.h> #include <asm/pgalloc.h> #include <asm/uaccess.h>
@@ -2183,8 +2184,11 @@ retry: unlock:pte_unmap_unlock (page_table, ptl);if (dirty) { +struct address_space *mapping =page_mapping (new_page);set_page_dirty (new_page);put_page (new_page); +if (mapping) +balance_dirty_pages_ratelimited_nr (mapping,1 ); }return ret; oom:
@@ -2304,8 +2308,11 @@ static inline int handle_pte_fault(struc unlock:pte_unmap_unlock (pte, ptl);if (page) { +struct address_space *mapping =page_mapping (page);set_page_dirty (page);put_page (page); +if (mapping) +balance_dirty_pages_ratelimited_nr (mapping,1 ); }return VM_FAULT_MINOR ; }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>