Memory folios: Pagecache edition
Subsystems (24)
Matthew Wilcox (Oracle)
maintainercover Memory folios: Pagecache edition
This patch series is for human review. The bots are going to complain because it depends on the previous 50 folio patches which are now quite well reviewed (thanks!) It's almost identical to patches 51-89 of folio v14 (added a few R-b tags and a couple of minor build fixes)
This set of patches are everything that needs to be done before I can convert iomap to use folios. It's probably also all or most of what needs to be done before other filesystems can be converted to use folios. So getting this into v5.15 will enable a lot of other work to start.
Matthew Wilcox (Oracle) (39):
mm: Add folio_pfn()
mm: Add folio_raw_mapping()
mm: Add flush_dcache_folio()
mm: Add kmap_local_folio()
mm: Add arch_make_folio_accessible()
mm: Add folio_young and folio_idle
mm/swap: Add folio_activate()
mm/swap: Add folio_mark_accessed()
mm/rmap: Add folio_mkclean()
mm/migrate: Add folio_migrate_mapping()
mm/migrate: Add folio_migrate_flags()
mm/migrate: Add folio_migrate_copy()
mm/writeback: Rename __add_wb_stat() to wb_stat_mod()
flex_proportions: Allow N events instead of 1 mm/writeback: Change __wb_writeout_inc() to __wb_writeout_add() mm/writeback: Add __folio_end_writeback() mm/writeback: Add folio_start_writeback() mm/writeback: Add folio_mark_dirty()
mm/writeback: Add __folio_mark_dirty()
mm/writeback: Convert tracing writeback_page_template to folios mm/writeback: Add filemap_dirty_folio()
mm/writeback: Add folio_account_cleaned() mm/writeback: Add folio_cancel_dirty()
mm/writeback: Add folio_clear_dirty_for_io() mm/writeback: Add folio_account_redirty() mm/writeback: Add folio_redirty_for_writepage() mm/filemap: Add i_blocks_per_folio()
mm/filemap: Add folio_mkwrite_check_truncate() mm/filemap: Add readahead_folio()
mm/workingset: Convert workingset_refault() to take a folio
mm: Add folio_evictable()
mm/lru: Convert __pagevec_lru_add_fn to take a folio mm/lru: Add folio_add_lru()
mm/page_alloc: Add folio allocation functions mm/filemap: Add filemap_alloc_folio
mm/filemap: Add filemap_add_folio()
mm/filemap: Convert mapping_get_entry to return a folio mm/filemap: Add filemap_get_folio
mm/filemap: Add FGP_STABLE
Documentation/core-api/cachetlb.rst | 6 + arch/arm/include/asm/cacheflush.h | 1 + arch/nds32/include/asm/cacheflush.h | 1 + fs/jfs/jfs_metapage.c | 1 + include/asm-generic/cacheflush.h | 6 + include/linux/backing-dev.h | 6 +- include/linux/flex_proportions.h | 9 +- include/linux/gfp.h | 22 +- include/linux/highmem-internal.h | 11 + include/linux/highmem.h | 38 +++ include/linux/ksm.h | 4 +- include/linux/memcontrol.h | 5 +- include/linux/migrate.h | 4 + include/linux/mm.h | 62 +++-- include/linux/page-flags.h | 20 +- include/linux/page_idle.h | 99 ++++--- include/linux/page_owner.h | 8 +- include/linux/pagemap.h | 189 ++++++++++---- include/linux/rmap.h | 10 +- include/linux/swap.h | 8 +- include/linux/writeback.h | 9 +- include/trace/events/pagemap.h | 46 ++-- include/trace/events/writeback.h | 20 +- kernel/bpf/verifier.c | 2 +- lib/flex_proportions.c | 28 +- mm/filemap.c | 238 ++++++++--------- mm/folio-compat.c | 99 +++++++ mm/hugetlb.c | 2 +- mm/internal.h | 35 ++- mm/ksm.c | 31 ++- mm/memory.c | 3 +- mm/mempolicy.c | 10 + mm/migrate.c | 187 +++++++------- mm/page-writeback.c | 383 +++++++++++++++------------- mm/page_alloc.c | 12 + mm/page_owner.c | 10 +- mm/rmap.c | 12 +- mm/swap.c | 136 +++++----- mm/swap_state.c | 2 +- mm/util.c | 39 +-- mm/workingset.c | 34 +-- 41 files changed, 1095 insertions(+), 753 deletions(-)
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 1/39 mm: Add folio_pfn()
This is the folio equivalent of page_to_pfn().
--- include/linux/mm.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
include/linux/mm.h +14 -0
@@ -1623,6 +1623,20 @@ static inline unsigned long page_to_section(const struct page *page) }#endif +/** +* folio_pfn - Return the Page Frame Number of a folio. +* @folio: The folio. +* +* A folio may contain multiple pages. The pages have consecutive +* Page Frame Numbers. +* +* Return: The Page Frame Number of the first page in the folio. +*/ +static inline unsigned long folio_pfn (struct folio *folio) +{ +return page_to_pfn (&folio->page); +} +/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */ #ifdef CONFIG_MIGRATION static inline bool is_pinnable_page (struct page *page)
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 2/39 mm: Add folio_raw_mapping()
In reply to Matthew Wilcox (Oracle)Convert __page_rmapping to folio_raw_mapping and move it to mm/internal.h. It's only a couple of instructions (load and mask), so it's definitely going to be cheaper to inline it than call it. Leave page_rmapping out of line.
--- mm/internal.h | 7 +++++++ mm/util.c | 20 ++++---------------- 2 files changed, 11 insertions(+), 16 deletions(-)
mm/internal.h +7 -0
@@ -34,6 +34,13 @@void page_writeback_init (void ); +static inline void *folio_raw_mapping (struct folio *folio) +{ +unsigned long mapping = (unsigned long )folio->mapping; + +return (void *)(mapping & ~PAGE_MAPPING_FLAGS ); +} +vm_fault_t do_swap_page (struct vm_fault *vmf);void folio_rotate_reclaimable (struct folio *folio);
mm/util.c +4 -16
@@ -635,21 +635,10 @@ void kvfree_sensitive(const void *addr, size_t len) }EXPORT_SYMBOL (kvfree_sensitive); -static inline void *__page_rmapping (struct page *page) -{ -unsigned long mapping; - - mapping = (unsigned long )page->mapping; - mapping &= ~PAGE_MAPPING_FLAGS ; - -return (void *)mapping; -} -/* Neutral page->mapping pointer to address_space or anon_vma or other */ void *page_rmapping (struct page *page) { - page =compound_head (page); -return __page_rmapping (page); +return folio_raw_mapping (page_folio (page)); }/**
@@ -680,13 +669,12 @@ EXPORT_SYMBOL(folio_mapped);struct anon_vma *page_anon_vma (struct page *page) { -unsigned long mapping; +struct folio *folio =page_folio (page); +unsigned long mapping = (unsigned long )folio->mapping; - page =compound_head (page); - mapping = (unsigned long )page->mapping;if ((mapping &PAGE_MAPPING_FLAGS ) !=PAGE_MAPPING_ANON )return NULL ; -return __page_rmapping (page); +return (void *)(mapping -PAGE_MAPPING_ANON ); }/**
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 3/39 mm: Add flush_dcache_folio()
In reply to Matthew Wilcox (Oracle)This is a default implementation which calls flush_dcache_page() on each page in the folio. If architectures can do better, they should implement their own version of it.
--- Documentation/core-api/cachetlb.rst | 6 ++++++ arch/arm/include/asm/cacheflush.h | 1 + arch/nds32/include/asm/cacheflush.h | 1 + include/asm-generic/cacheflush.h | 6 ++++++ mm/util.c | 13 +++++++++++++ 5 files changed, 27 insertions(+)
Documentation/core-api/cachetlb.rst +6 -0
@@ -325,6 +325,12 @@ maps this page at its virtual address. dirty. Again, see sparc64 for examples of how to deal with this. + ``void flush_dcache_folio(struct folio *folio)`` + This function is called under the same circumstances as + flush_dcache_page(). It allows the architecture to + optimise for flushing the entire folio of pages instead + of flushing one page at a time. + ``void copy_to_user_page(struct vm_area_struct *vma, struct page *page, unsigned long user_vaddr, void *dst, void *src, int len)`` ``void copy_from_user_page(struct vm_area_struct *vma, struct page *page,
arch/arm/include/asm/cacheflush.h +1 -0
@@ -290,6 +290,7 @@ extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr*/ #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1extern void flush_dcache_page (struct page *); +void flush_dcache_folio (struct folio *folio);static inline void flush_kernel_vmap_range (void *addr,int size) {
arch/nds32/include/asm/cacheflush.h +1 -0
@@ -38,6 +38,7 @@ void flush_anon_page(struct vm_area_struct *vma,#define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE void flush_kernel_dcache_page (struct page *page); +void flush_dcache_folio (struct folio *folio);void flush_kernel_vmap_range (void *addr,int size);void invalidate_kernel_vmap_range (void *addr,int size);#define flush_dcache_mmap_lock (mapping) xa_lock_irq(&(mapping)->i_pages)
include/asm-generic/cacheflush.h +6 -0
@@ -49,9 +49,15 @@ static inline void flush_cache_page(struct vm_area_struct *vma,static inline void flush_dcache_page (struct page *page) { } + +static inline void flush_dcache_folio (struct folio *folio) { }#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_FOLIO #endif +#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_FOLIO +void flush_dcache_folio (struct folio *folio); +#endif #ifndef flush_dcache_mmap_lockstatic inline void flush_dcache_mmap_lock (struct address_space *mapping)
mm/util.c +13 -0
@@ -1057,3 +1057,16 @@ void page_offline_end(void)up_write (&page_offline_rwsem); }EXPORT_SYMBOL (page_offline_end); + +#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_FOLIO +void flush_dcache_folio (struct folio *folio) +{ +unsigned int n =folio_nr_pages (folio); + +do { + n--; +flush_dcache_page (folio_page (folio, n)); + }while (n); +} +EXPORT_SYMBOL (flush_dcache_folio); +#endif
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 4/39 mm: Add kmap_local_folio()
In reply to Matthew Wilcox (Oracle)This allows us to map a portion of a folio. Callers can only expect to access up to the next page boundary.
--- include/linux/highmem-internal.h | 11 +++++++++ include/linux/highmem.h | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+)
include/linux/highmem-internal.h +11 -0
@@ -73,6 +73,12 @@ static inline void *kmap_local_page(struct page *page)return __kmap_local_page_prot (page, kmap_prot); } +static inline void *kmap_local_folio (struct folio *folio,size_t offset) +{ +struct page *page =folio_page (folio, offset /PAGE_SIZE ); +return __kmap_local_page_prot (page, kmap_prot) + offset %PAGE_SIZE ; +} +static inline void *kmap_local_page_prot (struct page *page,pgprot_t prot) {return __kmap_local_page_prot (page, prot);
@@ -160,6 +166,11 @@ static inline void *kmap_local_page(struct page *page)return page_address (page); } +static inline void *kmap_local_folio (struct folio *folio,size_t offset) +{ +return page_address (&folio->page) + offset; +} +static inline void *kmap_local_page_prot (struct page *page,pgprot_t prot) {return kmap_local_page (page);
include/linux/highmem.h +38 -0
@@ -96,6 +96,44 @@ static inline void kmap_flush_unused(void);*/ static inline void *kmap_local_page (struct page *page); +/** +* kmap_local_folio - Map a page in this folio for temporary usage +* @folio: The folio to be mapped. +* @offset: The byte offset within the folio. +* +* Returns: The virtual address of the mapping +* +* Can be invoked from any context. +* +* Requires careful handling when nesting multiple mappings because the map +* management is stack based. The unmap has to be in the reverse order of +* the map operation: +* +* addr1 = kmap_local_folio(page1, offset1); +* addr2 = kmap_local_folio(page2, offset2); +* ... +* kunmap_local(addr2); +* kunmap_local(addr1); +* +* Unmapping addr1 before addr2 is invalid and causes malfunction. +* +* Contrary to kmap() mappings the mapping is only valid in the context of +* the caller and cannot be handed to other contexts. +* +* On CONFIG_HIGHMEM=n kernels and for low memory pages this returns the +* virtual address of the direct mapping. Only real highmem pages are +* temporarily mapped. +* +* While it is significantly faster than kmap() for the higmem case it +* comes with restrictions about the pointer validity. Only use when really +* necessary. +* +* On HIGHMEM enabled systems mapping a highmem page has the side effect of +* disabling migration in order to keep the virtual address stable across +* preemption. No caller of kmap_local_folio() can rely on this side effect. +*/ +static inline void *kmap_local_folio (struct folio *folio,size_t offset); +/** * kmap_atomic - Atomically map a page for temporary usage - Deprecated! * @page: Pointer to the page to be mapped
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 5/39 mm: Add arch_make_folio_accessible()
In reply to Matthew Wilcox (Oracle)As a default implementation, call arch_make_page_accessible n times. If an architecture can do better, it can override this.
Also move the default implementation of arch_make_page_accessible() from gfp.h to mm.h.
--- include/linux/gfp.h | 6 ------ include/linux/mm.h | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-)
include/linux/gfp.h +0 -6
@@ -520,12 +520,6 @@ static inline void arch_free_page(struct page *page, int order) { }#ifndef HAVE_ARCH_ALLOC_PAGE static inline void arch_alloc_page (struct page *page,int order) { }#endif -#ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE -static inline int arch_make_page_accessible (struct page *page) -{ -return 0 ; -} -#endif struct page *__alloc_pages (gfp_t gfp,unsigned int order,int preferred_nid,nodemask_t *nodemask);
include/linux/mm.h +21 -0
@@ -1732,6 +1732,27 @@ static inline size_t folio_size(struct folio *folio)return PAGE_SIZE <<folio_order (folio); } +#ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE +static inline int arch_make_page_accessible (struct page *page) +{ +return 0 ; +} +#endif + +#ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE +static inline int arch_make_folio_accessible (struct folio *folio) +{ +int ret, i; +for (i =0 ; i <folio_nr_pages (folio); i++) { + ret =arch_make_page_accessible (folio_page (folio, i)); +if (ret) +break ; + } + +return ret; +} +#endif +/* * Some inline functions in vmstat.h depend on page_zone() */
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 6/39 mm: Add folio_young and folio_idle
In reply to Matthew Wilcox (Oracle)Idle page tracking is handled through page_ext on 32-bit architectures. Add folio equivalents for 32-bit and move all the page compatibility parts to common code.
--- include/linux/page_idle.h | 99 +++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 50 deletions(-)
include/linux/page_idle.h +49 -50
@@ -8,46 +8,16 @@ #ifdef CONFIG_IDLE_PAGE_TRACKING -#ifdef CONFIG_64BIT -static inline bool page_is_young(struct page *page) -{ - return PageYoung(page); -} - -static inline void set_page_young(struct page *page) -{ - SetPageYoung(page); -} - -static inline bool test_and_clear_page_young(struct page *page) -{ - return TestClearPageYoung(page); -} - -static inline bool page_is_idle(struct page *page) -{ - return PageIdle(page); -} - -static inline void set_page_idle(struct page *page) -{ - SetPageIdle(page); -} - -static inline void clear_page_idle(struct page *page) -{ - ClearPageIdle(page); -} -#else /* !CONFIG_64BIT */ +#ifndef CONFIG_64BIT /* * If there is not enough space to store Idle and Young bits in page flags, use * page ext flags instead. */ extern struct page_ext_operations page_idle_ops; -static inline bool page_is_young(struct page *page) +static inline bool folio_test_young(struct folio *folio) { - struct page_ext *page_ext = lookup_page_ext(page); + struct page_ext *page_ext = lookup_page_ext(&folio->page); if (unlikely(!page_ext)) return false;
@@ -55,9 +25,9 @@ static inline bool page_is_young(struct page *page)return test_bit (PAGE_EXT_YOUNG , &page_ext->flags); } -static inline void set_page_young (struct page *page) +static inline void folio_set_young (struct folio *folio) { -struct page_ext *page_ext =lookup_page_ext (page); +struct page_ext *page_ext =lookup_page_ext (&folio->page);if (unlikely (!page_ext))return ;
@@ -65,9 +35,9 @@ static inline void set_page_young(struct page *page)set_bit (PAGE_EXT_YOUNG , &page_ext->flags); } -static inline bool test_and_clear_page_young (struct page *page) +static inline bool folio_test_clear_young (struct folio *folio) { -struct page_ext *page_ext =lookup_page_ext (page); +struct page_ext *page_ext =lookup_page_ext (&folio->page);if (unlikely (!page_ext))return false;
@@ -75,9 +45,9 @@ static inline bool test_and_clear_page_young(struct page *page)return test_and_clear_bit (PAGE_EXT_YOUNG , &page_ext->flags); } -static inline bool page_is_idle (struct page *page) +static inline bool folio_test_idle (struct folio *folio) { -struct page_ext *page_ext =lookup_page_ext (page); +struct page_ext *page_ext =lookup_page_ext (&folio->page);if (unlikely (!page_ext))return false;
@@ -85,9 +55,9 @@ static inline bool page_is_idle(struct page *page)return test_bit (PAGE_EXT_IDLE , &page_ext->flags); } -static inline void set_page_idle (struct page *page) +static inline void folio_set_idle (struct folio *folio) { -struct page_ext *page_ext =lookup_page_ext (page); +struct page_ext *page_ext =lookup_page_ext (&folio->page);if (unlikely (!page_ext))return ;
@@ -95,46 +65,75 @@ static inline void set_page_idle(struct page *page)set_bit (PAGE_EXT_IDLE , &page_ext->flags); } -static inline void clear_page_idle (struct page *page) +static inline void folio_clear_idle (struct folio *folio) { -struct page_ext *page_ext =lookup_page_ext (page); +struct page_ext *page_ext =lookup_page_ext (&folio->page);if (unlikely (!page_ext))return ;clear_bit (PAGE_EXT_IDLE , &page_ext->flags); } -#endif /* CONFIG_64BIT */ +#endif /* !CONFIG_64BIT */ #else /* !CONFIG_IDLE_PAGE_TRACKING */ -static inline bool page_is_young (struct page *page) +static inline bool folio_test_young (struct folio *folio) {return false; } -static inline void set_page_young (struct page *page) +static inline void folio_set_young (struct folio *folio) { } -static inline bool test_and_clear_page_young (struct page *page) +static inline bool folio_test_clear_young (struct folio *folio) {return false; } -static inline bool page_is_idle (struct page *page) +static inline bool folio_test_idle (struct folio *folio) {return false; } -static inline void set_page_idle (struct page *page) +static inline void folio_set_idle (struct folio *folio) { } -static inline void clear_page_idle (struct page *page) +static inline void folio_clear_idle (struct folio *folio) { }#endif /* CONFIG_IDLE_PAGE_TRACKING */ +static inline bool page_is_young (struct page *page) +{ +return folio_test_young (page_folio (page)); +} + +static inline void set_page_young (struct page *page) +{ +folio_set_young (page_folio (page)); +} + +static inline bool test_and_clear_page_young (struct page *page) +{ +return folio_test_clear_young (page_folio (page)); +} + +static inline bool page_is_idle (struct page *page) +{ +return folio_test_idle (page_folio (page)); +} + +static inline void set_page_idle (struct page *page) +{ +folio_set_idle (page_folio (page)); +} + +static inline void clear_page_idle (struct page *page) +{ +folio_clear_idle (page_folio (page)); +}#endif /* _LINUX_MM_PAGE_IDLE_H */
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 7/39 mm/swap: Add folio_activate()
In reply to Matthew Wilcox (Oracle)This replaces activate_page() and eliminates lots of calls to compound_head(). Saves net 118 bytes of kernel text. There are still some redundant calls to page_folio() here which will be removed when pagevec_lru_move_fn() is converted to use folios.
--- include/trace/events/pagemap.h | 14 +++++------- mm/swap.c | 41 ++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 27 deletions(-)
include/trace/events/pagemap.h +6 -8
@@ -60,23 +60,21 @@ TRACE_EVENT(mm_lru_insertion,TRACE_EVENT (mm_lru_activate, - TP_PROTO(struct page *page), + TP_PROTO(struct folio *folio), -TP_ARGS (page), +TP_ARGS (folio),TP_STRUCT__entry ( - __field(struct page *, page ) + __field(struct folio *, folio ) __field(unsigned long, pfn ) ),TP_fast_assign ( - __entry->page = page; - __entry->pfn = page_to_pfn(page); + __entry->folio = folio; + __entry->pfn = folio_pfn(folio); ), -/* Flag format is based on page-types.c formatting for pagemap */ -TP_printk ("page=%p pfn=0x%lx" , __entry->page, __entry->pfn) - +TP_printk ("folio=%p pfn=0x%lx" , __entry->folio, __entry->pfn) );#endif /* _TRACE_PAGEMAP_H */
mm/swap.c +22 -19
83 lines not on this page · raw message
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 8/39 mm/swap: Add folio_mark_accessed()
In reply to Matthew Wilcox (Oracle)Convert mark_page_accessed() to folio_mark_accessed(). It already operated on the entire compound page, but now we can avoid calling compound_head quite so many times. Shrinks the function from 424 bytes to 295 bytes (shrinking by 129 bytes). The compatibility wrapper is 30 bytes, plus the 8 bytes for the exported symbol means the kernel shrinks by 91 bytes.
--- include/linux/swap.h | 3 ++- mm/folio-compat.c | 7 +++++++ mm/swap.c | 34 ++++++++++++++++------------------ 3 files changed, 25 insertions(+), 19 deletions(-)
include/linux/swap.h +2 -1
@@ -352,7 +352,8 @@ extern void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages);extern void lru_note_cost_page (struct page *);extern void lru_cache_add (struct page *); -extern void mark_page_accessed (struct page *); +void mark_page_accessed (struct page *); +void folio_mark_accessed (struct folio *);extern atomic_t lru_disable_count;
mm/folio-compat.c +7 -0
@@ -5,6 +5,7 @@*/ #include <linux/pagemap.h> +#include <linux/swap.h> struct address_space *page_mapping (struct page *page) {
@@ -41,3 +42,9 @@ bool page_mapped(struct page *page)return folio_mapped (page_folio (page)); }EXPORT_SYMBOL (page_mapped); + +void mark_page_accessed (struct page *page) +{ +folio_mark_accessed (page_folio (page)); +} +EXPORT_SYMBOL (mark_page_accessed);
mm/swap.c +16 -18
@@ -390,7 +390,7 @@ static void folio_activate(struct folio *folio) }#endif -static void __lru_cache_activate_page (struct page *page) +static void __lru_cache_activate_folio (struct folio *folio) {struct pagevec *pvec;int i;
@@ -411,8 +411,8 @@ static void __lru_cache_activate_page(struct page *page)for (i =pagevec_count (pvec) -1 ; i >=0 ; i--) {struct page *pagevec_page = pvec->pages[i]; -if (pagevec_page == page) { -SetPageActive (page); +if (pagevec_page == &folio->page) { +folio_set_active (folio);break ; } }
@@ -430,36 +430,34 @@ static void __lru_cache_activate_page(struct page *page)* When a newly allocated page is not yet visible, so safe for non-atomic ops, * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). */ -void mark_page_accessed (struct page *page) +void folio_mark_accessed (struct folio *folio) { - page =compound_head (page); - -if (!PageReferenced (page)) { -SetPageReferenced (page); - }else if (PageUnevictable (page)) { +if (!folio_test_referenced (folio)) { +folio_set_referenced (folio); + }else if (folio_test_unevictable (folio)) {/* * Unevictable pages are on the "LRU_UNEVICTABLE" list. But, * this list is never rotated or maintained, so marking an * evictable page accessed has no effect. */ - }else if (!PageActive (page)) { + }else if (!folio_test_active (folio)) {/* * If the page is on the LRU, queue it for activation via * lru_pvecs.activate_page. Otherwise, assume the page is on a * pagevec, mark it active and it'll be moved to the active * LRU on the next drain. */ -if (PageLRU (page)) -folio_activate (page_folio (page)); +if (folio_test_lru (folio)) +folio_activate (folio);else -__lru_cache_activate_page (page); -ClearPageReferenced (page); -workingset_activation (page_folio (page)); +__lru_cache_activate_folio (folio); +folio_clear_referenced (folio); +workingset_activation (folio); } -if (page_is_idle (page)) -clear_page_idle (page); +if (folio_test_idle (folio)) +folio_clear_idle (folio); } -EXPORT_SYMBOL (mark_page_accessed); +EXPORT_SYMBOL (folio_mark_accessed);/** * lru_cache_add - add a page to a page list
Signature · 1 line
2.30.2
Re: [PATCH v14 08/39] mm/swap: Add folio_mark_accessed()
On Thu, 15 Jul 2021 20:59:59 +0100, Matthew Wilcox wrote
Quoted 36 lines
include/linux/swap.h +2
-1@@ -352,7 +352,8 @@ extern void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages);extern void lru_note_cost_page (struct page *);extern void lru_cache_add (struct page *); -extern void mark_page_accessed (struct page *); +void mark_page_accessed (struct page *); +void folio_mark_accessed (struct folio *);extern atomic_t lru_disable_count;mm/folio-compat.c +7
-0@@ -5,6 +5,7 @@*/ #include <linux/pagemap.h> +#include <linux/swap.h> struct address_space *page_mapping (struct page *page) {@@ -41,3 +42,9 @@ bool page_mapped(struct page *page)return folio_mapped (page_folio (page)); }EXPORT_SYMBOL (page_mapped); + +void mark_page_accessed (struct page *page) +{ +folio_mark_accessed (page_folio (page)); +} +EXPORT_SYMBOL (mark_page_accessed);
... snip ...
Quoted 6 lines
mm/folio-compat.c +0
-1@@ -430,36 +430,34 @@ static void __lru_cache_activate_page(struct page *page)* When a newly allocated page is not yet visible, so safe for non-atomic ops, * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). */ -void mark_page_accessed (struct page *page)
mm/folio-compat.c static void __lru_cache_activate_page(struct page *page) · line 437
Quoted 10 lines
mm/folio-compat.c +4
-5+void folio_mark_accessed (struct folio *folio) { - page =compound_head (page); - -if (!PageReferenced (page)) { -SetPageReferenced (page); - }else if (PageUnevictable (page)) { +if (!folio_test_referenced (folio)) { +folio_set_referenced (folio); + }else if (folio_test_unevictable (folio)) {
Hi Matthew,
My colleagues and I have been testing some page-tracking algorithms and we tried using the reference bit by using /proc/pid/clear_clears, /proc/pid/pagemap, and /proc/kpageflags.
I'm not 100% of the issue, but we're finding some inconsistencies when tracking page reference bits, and this seems to be at least one of the patches we saw that might be in the mix.
From reading up on folios, it seems like this change prevents each individual page in a folio from being marked referenced, and instead prefers to simply mark the entire folio containing the page as accessed.
When looking at the proc/ interface, it seems like it is still referencing the page and not using the folio for a page (see below).
Our current suspicion is that since only the folio head gets marked referenced, any pages inside the folio that aren't the head will basically never be marked referenced, leading to an inconsistent view.
I'm curious your thoughts on whether (or neither):
a) Should we update kpageflags_read to use page_folio() and get the folio flags for the head page
or
b) the above change to mark_page_accessed() should mark both the individual page flags to accessed as well as the folio head accessed.
Thanks for your time.
Gregory Price
(relevant fs/proc/page.c code:)
static ssize_t kpageflags_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
... snip ...
ppage = pfn_to_online_page(pfn);
if (put_user(stable_page_flags(ppage), out)) {
ret = -EFAULT;
break;
}... snip ...
}
u64 stable_page_flags(struct page *page)
{
... snip ...
k = page->flags;
... snip ...
}
Matthew Wilcox
maintainerRe: [PATCH v14 08/39] mm/swap: Add folio_mark_accessed()
Quoted 47 lines · Gregory Price
On Thu, 15 Jul 2021 20:59:59 +0100, Matthew Wilcox wrote
Quoted 5 lines
mm/folio-compat.c +5
-0+void mark_page_accessed (struct page *page) +{ +folio_mark_accessed (page_folio (page)); +} +EXPORT_SYMBOL (mark_page_accessed);... snip ...
Quoted 6 lines
mm/swap.c +0
-1@@ -430,36 +430,34 @@ static void __lru_cache_activate_page(struct page *page)* When a newly allocated page is not yet visible, so safe for non-atomic ops, * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). */ -void mark_page_accessed (struct page *page)mm/swap.c static void __lru_cache_activate_page(struct page *page) · line 437
Quoted 10 lines
mm/swap.c +4
-5+void folio_mark_accessed (struct folio *folio) { - page =compound_head (page); - -if (!PageReferenced (page)) { -SetPageReferenced (page); - }else if (PageUnevictable (page)) { +if (!folio_test_referenced (folio)) { +folio_set_referenced (folio); + }else if (folio_test_unevictable (folio)) {Hi Matthew,
My colleagues and I have been testing some page-tracking algorithms and we tried using the reference bit by using
/proc/pid/clear_clears,/proc/pid/pagemap, and/proc/kpageflags.I'm not 100% of the issue, but we're finding some inconsistencies when tracking page reference bits, and this seems to be at least one of the patches we saw that might be in the mix.
Quoted 1 line
From reading up on folios, it seems like this change prevents each
individual page in a folio from being marked referenced, and instead prefers to simply mark the entire folio containing the page as accessed.
When looking at the proc/ interface, it seems like it is still referencing the page and not using the folio for a page (see below).
Our current suspicion is that since only the folio head gets marked referenced, any pages inside the folio that aren't the head will basically never be marked referenced, leading to an inconsistent view.
I'm curious your thoughts on whether (or neither):
Quoted 7 lines
a) Should we update
kpageflags_readto usepage_folio()and get the folio flags for the head pageor
b) the above change to
mark_page_accessed()should mark both the individual page flags to accessed as well as the folio head accessed.
Hi Greg, thanks for reaching out.
The referenced bit has been tracked on what is now known as a per-folio basis since commit 8cb38fabb6bc in 2016. That is, if you tried to SetPageReferenced / ClearPageReferenced / test PageReferenced on a tail page, it would redirect to the head page in order to set/clear/test the bit in the head page's flags field.
That's not to say that all the code which sets/checks/tests this really understands that! We've definitely found bugs during the folio work where code has mistakenly assumed that operations on a tail page actually affect that page rather than the whole allocation.
There's also code which is now being exposed to compound pages for the first time, and is not holding up well.
I hope that's helpful in finding the bug you're chasing.
Quoted 27 lines
Thanks for your time.
Gregory Price(relevant
fs/proc/page.ccode:)static
ssize_tkpageflags_read(struct file *file, char__user*buf,
size_tcount,loff_t*ppos)
{
... snip ...ppage = pfn_to_online_page(pfn); if (put_user(stable_page_flags(ppage), out)) { ret = -EFAULT; break; }... snip ...
}u64
stable_page_flags(struct page *page)
{
... snip ...k = page->flags;... snip ...
}
Re: [PATCH v14 08/39] mm/swap: Add folio_mark_accessed()
Quoted 22 lines · Matthew Wilcox
Quoted 7 lines · Gregory Price
a) Should we update
kpageflags_readto usepage_folio()and get the folio flags for the head pageor
b) the above change to
mark_page_accessed()should mark both the individual page flags to accessed as well as the folio head accessed.Hi Greg, thanks for reaching out.
The referenced bit has been tracked on what is now known as a per-folio basis since commit
8cb38fabb6bcin 2016. That is, if you tried to SetPageReferenced / ClearPageReferenced / test PageReferenced on a tail page, it would redirect to the head page in order to set/clear/test the bit in the head page's flags field.That's not to say that all the code which sets/checks/tests this really understands that! We've definitely found bugs during the folio work where code has mistakenly assumed that operations on a tail page actually affect that page rather than the whole allocation.
Quoted 5 lines
There's also code which is now being exposed to compound pages for the first time, and is not holding up well.
I hope that's helpful in finding the bug you're chasing.
ah, very interesting. So the kpageflags code doesn't appear to respect flag locations defined in 8cb38fabb6bc
We tested this hack of a patch which swaps the page flags for the folio head flags according to the flag locations defined in page-flags.h.
The result was that the referenced bit was almost *never* set on an anonymous pages, no matter where we look, even when we have a program constantly touching every page in a large chunk of memory.
Head scratcher.
Thank you for the response. We're still very much confused, but it looks like there are potentially many problems (unless we're misunderstanding the reference bit).
(cc Naoya since I saw you were actively working on kpageflags for folio
support not sure what else you're seeing).
~Gregory
fs/proc/page.c +7 -6
@@ -110,6 +110,7 @@ static inline u64 kpf_copy_bit(u64 kflags, int ubit, int kbit)u64 stable_page_flags (struct page *page) {u64 k; +u64 fh;u64 u;/*
@@ -119,7 +120,7 @@ u64 stable_page_flags(struct page *page)if (!page)return 1 <<KPF_NOPAGE ; - k = page->flags; + fh = *folio_flags (page_folio (folio),0 ); u =0 ;/*
@@ -188,20 +189,20 @@ u64 stable_page_flags(struct page *page) u |=1 <<KPF_SLAB ; u |=kpf_copy_bit (k,KPF_ERROR , PG_error); - u |=kpf_copy_bit (k,KPF_DIRTY , PG_dirty); + u |=kpf_copy_bit (fh,KPF_DIRTY , PG_dirty); u |=kpf_copy_bit (k,KPF_UPTODATE , PG_uptodate); u |=kpf_copy_bit (k,KPF_WRITEBACK , PG_writeback); - u |=kpf_copy_bit (k,KPF_LRU , PG_lru); - u |=kpf_copy_bit (k,KPF_REFERENCED , PG_referenced); - u |=kpf_copy_bit (k,KPF_ACTIVE , PG_active); + u |=kpf_copy_bit (fh,KPF_LRU , PG_lru); + u |=kpf_copy_bit (fh,KPF_REFERENCED , PG_referenced); + u |=kpf_copy_bit (fh,KPF_ACTIVE , PG_active); u |=kpf_copy_bit (k,KPF_RECLAIM , PG_reclaim);if (PageSwapCache (page)) u |=1 <<KPF_SWAPCACHE ; u |=kpf_copy_bit (k,KPF_SWAPBACKED , PG_swapbacked); - u |=kpf_copy_bit (k,KPF_UNEVICTABLE , PG_unevictable); + u |=kpf_copy_bit (fh,KPF_UNEVICTABLE , PG_unevictable); u |=kpf_copy_bit (k,KPF_MLOCKED , PG_mlocked);#ifdef CONFIG_MEMORY_FAILURE
Matthew Wilcox (Oracle)
maintainerPATCH v14 9/39 mm/rmap: Add folio_mkclean()
In reply to Matthew Wilcox (Oracle)Transform page_mkclean() into folio_mkclean() and add a page_mkclean() wrapper around folio_mkclean().
folio_mkclean is 15 bytes smaller than page_mkclean, but the kernel is enlarged by 33 bytes due to inlining page_folio() into each caller. This will go away once the callers are converted to use folio_mkclean().
--- include/linux/rmap.h | 10 ++++++---- mm/rmap.c | 12 ++++++------ 2 files changed, 12 insertions(+), 10 deletions(-)
include/linux/rmap.h +6 -4
@@ -235,7 +235,7 @@ unsigned long page_address_in_vma(struct page *, struct vm_area_struct *);* * returns the number of cleaned PTEs. */ -int page_mkclean (struct page *); +int folio_mkclean (struct folio *);/* * called in munlock()/munmap() path to check for other vmas holding
@@ -293,12 +293,14 @@ static inline int page_referenced(struct page *page, int is_locked,#define try_to_unmap (page, refs) false -static inline int page_mkclean (struct page *page) +static inline int folio_mkclean (struct folio *folio) {return 0 ; } - -#endif /* CONFIG_MMU */ +static inline int page_mkclean (struct page *page) +{ +return folio_mkclean (page_folio (page)); +}#endif /* _LINUX_RMAP_H */
mm/rmap.c +6 -6
@@ -980,7 +980,7 @@ static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)return true; } -int page_mkclean (struct page *page) +int folio_mkclean (struct folio *folio) {int cleaned =0 ;struct address_space *mapping;
@@ -990,20 +990,20 @@ int page_mkclean(struct page *page) .invalid_vma = invalid_mkclean_vma, }; -BUG_ON (!PageLocked (page)); +BUG_ON (!folio_test_locked (folio)); -if (!page_mapped (page)) +if (!folio_mapped (folio))return 0 ; - mapping =page_mapping (page); + mapping =folio_mapping (folio);if (!mapping)return 0 ; -rmap_walk (page, &rwc); +rmap_walk (&folio->page, &rwc);return cleaned; } -EXPORT_SYMBOL_GPL (page_mkclean); +EXPORT_SYMBOL_GPL (folio_mkclean);/** * page_move_anon_rmap - move a page to our anon_vma
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 10/39 mm/migrate: Add folio_migrate_mapping()
In reply to Matthew Wilcox (Oracle)Reimplement migrate_page_move_mapping() as a wrapper around folio_migrate_mapping(). Saves 193 bytes of kernel text.
--- include/linux/migrate.h | 2 + mm/folio-compat.c | 11 ++++++ mm/migrate.c | 85 +++++++++++++++++++++-------------------- 3 files changed, 57 insertions(+), 41 deletions(-)
include/linux/migrate.h +2 -0
@@ -51,6 +51,8 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page);extern int migrate_page_move_mapping (struct address_space *mapping,struct page *newpage,struct page *page,int extra_count); +int folio_migrate_mapping (struct address_space *mapping, +struct folio *newfolio,struct folio *folio,int extra_count);#else static inline void putback_movable_pages (struct list_head *l) {}
mm/folio-compat.c +11 -0
@@ -4,6 +4,7 @@* eventually. */ +#include <linux/migrate.h> #include <linux/pagemap.h> #include <linux/swap.h>
@@ -48,3 +49,13 @@ void mark_page_accessed(struct page *page)folio_mark_accessed (page_folio (page)); }EXPORT_SYMBOL (mark_page_accessed); + +#ifdef CONFIG_MIGRATION +int migrate_page_move_mapping (struct address_space *mapping, +struct page *newpage,struct page *page,int extra_count) +{ +return folio_migrate_mapping (mapping,page_folio (newpage), +page_folio (page), extra_count); +} +EXPORT_SYMBOL (migrate_page_move_mapping); +#endif
mm/migrate.c +44 -41
186 lines not on this page · raw message
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 11/39 mm/migrate: Add folio_migrate_flags()
In reply to Matthew Wilcox (Oracle)Turn migrate_page_states() into a wrapper around folio_migrate_flags(). Also convert two functions only called from folio_migrate_flags() to be folio-based. ksm_migrate_page() becomes folio_migrate_ksm() and copy_page_owner() becomes folio_copy_owner(). folio_migrate_flags() alone shrinks by two thirds -- 1967 bytes down to 642 bytes.
--- include/linux/ksm.h | 4 +- include/linux/migrate.h | 1 + include/linux/page_owner.h | 8 ++-- mm/folio-compat.c | 6 +++ mm/ksm.c | 31 ++++++++------ mm/migrate.c | 84 +++++++++++++++++++------------------- mm/page_owner.c | 10 ++--- 7 files changed, 77 insertions(+), 67 deletions(-)
include/linux/ksm.h +2 -2
@@ -52,7 +52,7 @@ struct page *ksm_might_need_to_copy(struct page *page, struct vm_area_struct *vma, unsigned long address);void rmap_walk_ksm (struct page *page,struct rmap_walk_control *rwc); -void ksm_migrate_page (struct page *newpage,struct page *oldpage); +void folio_migrate_ksm (struct folio *newfolio,struct folio *folio);#else /* !CONFIG_KSM */
@@ -83,7 +83,7 @@ static inline void rmap_walk_ksm(struct page *page, { } -static inline void ksm_migrate_page (struct page *newpage,struct page *oldpage) +static inline void folio_migrate_ksm (struct folio *newfolio,struct folio *old) { }#endif /* CONFIG_MMU */
include/linux/migrate.h +1 -0
@@ -51,6 +51,7 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page);extern int migrate_page_move_mapping (struct address_space *mapping,struct page *newpage,struct page *page,int extra_count); +void folio_migrate_flags (struct folio *newfolio,struct folio *folio);int folio_migrate_mapping (struct address_space *mapping,struct folio *newfolio,struct folio *folio,int extra_count);#else
include/linux/page_owner.h +4 -4
@@ -12,7 +12,7 @@ extern void __reset_page_owner(struct page *page, unsigned int order);extern void __set_page_owner (struct page *page,unsigned int order,gfp_t gfp_mask);extern void __split_page_owner (struct page *page,unsigned int nr); -extern void __copy_page_owner (struct page *oldpage,struct page *newpage); +extern void __folio_copy_owner (struct folio *newfolio,struct folio *old);extern void __set_page_owner_migrate_reason (struct page *page,int reason);extern void __dump_page_owner (const struct page *page); extern void pagetypeinfo_showmixedcount_print(struct seq_file *m,
@@ -36,10 +36,10 @@ static inline void split_page_owner(struct page *page, unsigned int nr)if (static_branch_unlikely (&page_owner_inited))__split_page_owner (page, nr); } -static inline void copy_page_owner (struct page *oldpage,struct page *newpage) +static inline void folio_copy_owner (struct folio *newfolio,struct folio *old) {if (static_branch_unlikely (&page_owner_inited)) -__copy_page_owner (oldpage, newpage); +__folio_copy_owner (newfolio, old); }static inline void set_page_owner_migrate_reason (struct page *page,int reason) {
@@ -63,7 +63,7 @@ static inline void split_page_owner(struct page *page, unsigned int order) { } -static inline void copy_page_owner (struct page *oldpage,struct page *newpage) +static inline void folio_copy_owner (struct folio *newfolio,struct folio *folio) { }static inline void set_page_owner_migrate_reason (struct page *page,int reason)
mm/folio-compat.c +6 -0
@@ -58,4 +58,10 @@ int migrate_page_move_mapping(struct address_space *mapping, page_folio(page), extra_count); }EXPORT_SYMBOL (migrate_page_move_mapping); + +void migrate_page_states (struct page *newpage,struct page *page) +{ +folio_migrate_flags (page_folio (newpage),page_folio (page)); +} +EXPORT_SYMBOL (migrate_page_states);#endif
mm/ksm.c +18 -13
@@ -753,7 +753,7 @@ static struct page *get_ksm_page(struct stable_node *stable_node,/* * We come here from above when page->mapping or !PageSwapCache * suggests that the node is stale; but it might be under migration. -* We need smp_rmb(), matching the smp_wmb() in ksm_migrate_page(), +* We need smp_rmb(), matching the smp_wmb() in folio_migrate_ksm(), * before checking whether node->kpfn has been changed. */ smp_rmb ();
@@ -854,9 +854,14 @@ static int unmerge_ksm_pages(struct vm_area_struct *vma,return err; } +static inline struct stable_node *folio_stable_node (struct folio *folio) +{ +return folio_test_ksm (folio) ?folio_raw_mapping (folio) :NULL ; +} +static inline struct stable_node *page_stable_node (struct page *page) { -return PageKsm (page) ?page_rmapping (page) :NULL ; +return folio_stable_node (page_folio (page)); } static inline void set_page_stable_node(struct page *page,
@@ -2661,26 +2666,26 @@ void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc) }#ifdef CONFIG_MIGRATION -void ksm_migrate_page (struct page *newpage,struct page *oldpage) +void folio_migrate_ksm (struct folio *newfolio,struct folio *folio) {struct stable_node *stable_node; -VM_BUG_ON_PAGE (!PageLocked (oldpage), oldpage); -VM_BUG_ON_PAGE (!PageLocked (newpage), newpage); -VM_BUG_ON_PAGE (newpage->mapping != oldpage->mapping, newpage); +VM_BUG_ON_FOLIO (!folio_test_locked (folio), folio); +VM_BUG_ON_FOLIO (!folio_test_locked (newfolio), newfolio); +VM_BUG_ON_FOLIO (newfolio->mapping != folio->mapping, newfolio); - stable_node =page_stable_node (newpage); + stable_node =folio_stable_node (folio);if (stable_node) { -VM_BUG_ON_PAGE (stable_node->kpfn !=page_to_pfn (oldpage), oldpage); - stable_node->kpfn =page_to_pfn (newpage); +VM_BUG_ON_FOLIO (stable_node->kpfn !=folio_pfn (folio), folio); + stable_node->kpfn =folio_pfn (newfolio);/* -* newpage->mapping was set in advance; now we need smp_wmb() +* newfolio->mapping was set in advance; now we need smp_wmb() * to make sure that the new stable_node->kpfn is visible -* to get_ksm_page() before it can see that oldpage->mapping -* has gone stale (or that PageSwapCache has been cleared). +* to get_ksm_page() before it can see that folio->mapping +* has gone stale (or that folio_test_swapcache has been cleared). */ smp_wmb (); -set_page_stable_node (oldpage,NULL ); +set_page_stable_node (&folio->page,NULL ); } }#endif /* CONFIG_MIGRATION */
mm/migrate.c +41 -43
130 lines not on this page · raw message
mm/page_owner.c +5 -5
@@ -210,10 +210,10 @@ void __split_page_owner(struct page *page, unsigned int nr) } } -void __copy_page_owner (struct page *oldpage,struct page *newpage) +void __folio_copy_owner (struct folio *newfolio,struct folio *old) { -struct page_ext *old_ext =lookup_page_ext (oldpage); -struct page_ext *new_ext =lookup_page_ext (newpage); +struct page_ext *old_ext =lookup_page_ext (&old->page); +struct page_ext *new_ext =lookup_page_ext (&newfolio->page);struct page_owner *old_page_owner, *new_page_owner;if (unlikely (!old_ext || !new_ext))
@@ -231,11 +231,11 @@ void __copy_page_owner(struct page *oldpage, struct page *newpage) new_page_owner->free_ts_nsec = old_page_owner->ts_nsec;/* -* We don't clear the bit on the oldpage as it's going to be freed +* We don't clear the bit on the old folio as it's going to be freed * after migration. Until then, the info can be useful in case of * a bug, and the overall stats will be off a bit only temporarily. * Also, migrate_misplaced_transhuge_page() can still fail the -* migration and then we want the oldpage to retain the info. But +* migration and then we want the old folio to retain the info. But * in that case we also don't need to explicitly clear the info from * the new page, which will be freed. */
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 12/39 mm/migrate: Add folio_migrate_copy()
In reply to Matthew Wilcox (Oracle)This is the folio equivalent of migrate_page_copy(), which is retained as a wrapper for filesystems which are not yet converted to folios. Also convert copy_huge_page() to folio_copy().
--- include/linux/migrate.h | 1 + include/linux/mm.h | 2 +- mm/folio-compat.c | 6 ++++++ mm/hugetlb.c | 2 +- mm/migrate.c | 14 +++++--------- mm/util.c | 6 +++--- 6 files changed, 17 insertions(+), 14 deletions(-)
include/linux/migrate.h +1 -0
@@ -52,6 +52,7 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping,extern int migrate_page_move_mapping (struct address_space *mapping,struct page *newpage,struct page *page,int extra_count);void folio_migrate_flags (struct folio *newfolio,struct folio *folio); +void folio_migrate_copy (struct folio *newfolio,struct folio *folio);int folio_migrate_mapping (struct address_space *mapping,struct folio *newfolio,struct folio *folio,int extra_count);#else
include/linux/mm.h +1 -1
@@ -911,7 +911,7 @@ void __put_page(struct page *page);void put_pages_list (struct list_head *pages);void split_page (struct page *page,unsigned int order); -void copy_huge_page (struct page *dst,struct page *src); +void folio_copy (struct folio *dst,struct folio *src);/* * Compound pages have a destructor function. Provide a
mm/folio-compat.c +6 -0
@@ -64,4 +64,10 @@ void migrate_page_states(struct page *newpage, struct page *page)folio_migrate_flags (page_folio (newpage),page_folio (page)); }EXPORT_SYMBOL (migrate_page_states); + +void migrate_page_copy (struct page *newpage,struct page *page) +{ +folio_migrate_copy (page_folio (newpage),page_folio (page)); +} +EXPORT_SYMBOL (migrate_page_copy);#endif
mm/hugetlb.c +1 -1
@@ -5200,7 +5200,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, *pagep =NULL ; goto out; } -copy_huge_page (page, *pagep); +folio_copy (page_folio (page),page_folio (*pagep));put_page (*pagep); *pagep =NULL ; }
mm/migrate.c +5 -9
@@ -613,16 +613,12 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) }EXPORT_SYMBOL (folio_migrate_flags); -void migrate_page_copy (struct page *newpage,struct page *page) +void folio_migrate_copy (struct folio *newfolio,struct folio *folio) { -if (PageHuge (page) ||PageTransHuge (page)) -copy_huge_page (newpage, page); -else -copy_highpage (newpage, page); - -migrate_page_states (newpage, page); +folio_copy (newfolio, folio); +folio_migrate_flags (newfolio, folio); } -EXPORT_SYMBOL (migrate_page_copy); +EXPORT_SYMBOL (folio_migrate_copy);/************************************************************ * Migration functions
@@ -650,7 +646,7 @@ int migrate_page(struct address_space *mapping,return rc;if (mode !=MIGRATE_SYNC_NO_COPY ) -migrate_page_copy (newpage, page); +folio_migrate_copy (newfolio, folio);else folio_migrate_flags (newfolio, folio);return MIGRATEPAGE_SUCCESS ;
mm/util.c +3 -3
@@ -728,13 +728,13 @@ int __page_mapcount(struct page *page) }EXPORT_SYMBOL_GPL (__page_mapcount); -void copy_huge_page (struct page *dst,struct page *src) +void folio_copy (struct folio *dst,struct folio *src) { -unsigned i, nr =compound_nr (src); +unsigned i, nr =folio_nr_pages (src);for (i =0 ; i < nr; i++) {cond_resched (); -copy_highpage (nth_page (dst, i),nth_page (src, i)); +copy_highpage (folio_page (dst, i),folio_page (src, i)); } }
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 13/39 mm/writeback: Rename __add_wb_stat() to wb_stat_mod()
In reply to Matthew Wilcox (Oracle)Make this look like the newly renamed vmstat functions.
--- include/linux/backing-dev.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
include/linux/backing-dev.h +3 -3
@@ -64,7 +64,7 @@ static inline bool bdi_has_dirty_io(struct backing_dev_info *bdi)return atomic_long_read (&bdi->tot_write_bandwidth); } -static inline void __add_wb_stat (struct bdi_writeback *wb, +static inline void wb_stat_mod (struct bdi_writeback *wb,enum wb_stat_item item,s64 amount) {percpu_counter_add_batch (&wb->stat[item], amount,WB_STAT_BATCH );
@@ -72,12 +72,12 @@ static inline void __add_wb_stat(struct bdi_writeback *wb,static inline void inc_wb_stat (struct bdi_writeback *wb,enum wb_stat_item item) { -__add_wb_stat (wb, item,1 ); +wb_stat_mod (wb, item,1 ); }static inline void dec_wb_stat (struct bdi_writeback *wb,enum wb_stat_item item) { -__add_wb_stat (wb, item,-1 ); +wb_stat_mod (wb, item,-1 ); }static inline s64 wb_stat (struct bdi_writeback *wb,enum wb_stat_item item)
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 14/39 flex_proportions: Allow N events instead of 1
In reply to Matthew Wilcox (Oracle)When batching events (such as writing back N pages in a single I/O), it is better to do one flex_proportion operation instead of N. There is only one caller of __fprop_inc_percpu_max(), and it's the one we're going to change in the next patch, so rename it instead of adding a compatibility wrapper.
--- include/linux/flex_proportions.h | 9 +++++---- lib/flex_proportions.c | 28 +++++++++++++++++++--------- mm/page-writeback.c | 4 ++-- 3 files changed, 26 insertions(+), 15 deletions(-)
include/linux/flex_proportions.h +5 -4
@@ -83,9 +83,10 @@ struct fprop_local_percpu {int fprop_local_init_percpu (struct fprop_local_percpu *pl,gfp_t gfp);void fprop_local_destroy_percpu (struct fprop_local_percpu *pl); -void __fprop_inc_percpu (struct fprop_global *p,struct fprop_local_percpu *pl); -void __fprop_inc_percpu_max (struct fprop_global *p,struct fprop_local_percpu *pl, -int max_frac); +void __fprop_add_percpu (struct fprop_global *p,struct fprop_local_percpu *pl, +long nr); +void __fprop_add_percpu_max (struct fprop_global *p, +struct fprop_local_percpu *pl,int max_frac,long nr);void fprop_fraction_percpu (struct fprop_global *p,struct fprop_local_percpu *pl,unsigned long *numerator,unsigned long *denominator);
@@ -96,7 +97,7 @@ void fprop_inc_percpu(struct fprop_global *p, struct fprop_local_percpu *pl)unsigned long flags;local_irq_save (flags); -__fprop_inc_percpu (p, pl); +__fprop_add_percpu (p, pl,1 );local_irq_restore (flags); }
lib/flex_proportions.c +19 -9
@@ -217,11 +217,12 @@ static void fprop_reflect_period_percpu(struct fprop_global *p, }/* Event of type pl happened */ -void __fprop_inc_percpu (struct fprop_global *p,struct fprop_local_percpu *pl) +void __fprop_add_percpu (struct fprop_global *p,struct fprop_local_percpu *pl, +long nr) {fprop_reflect_period_percpu (p, pl); -percpu_counter_add_batch (&pl->events,1 ,PROP_BATCH ); -percpu_counter_add (&p->events,1 ); +percpu_counter_add_batch (&pl->events, nr,PROP_BATCH ); +percpu_counter_add (&p->events, nr); } void fprop_fraction_percpu(struct fprop_global *p,
@@ -253,20 +254,29 @@ void fprop_fraction_percpu(struct fprop_global *p, }/* -* Like __fprop_inc_percpu() except that event is counted only if the given +* Like __fprop_add_percpu() except that event is counted only if the given * type has fraction smaller than @max_frac/FPROP_FRAC_BASE */ -void __fprop_inc_percpu_max (struct fprop_global *p, -struct fprop_local_percpu *pl,int max_frac) +void __fprop_add_percpu_max (struct fprop_global *p, +struct fprop_local_percpu *pl,int max_frac,long nr) {if (unlikely (max_frac <FPROP_FRAC_BASE )) {unsigned long numerator, denominator; +s64 tmp;fprop_fraction_percpu (p, pl, &numerator, &denominator); -if (numerator > - (((u64 )denominator) * max_frac) >>FPROP_FRAC_SHIFT ) +/* Adding 'nr' to fraction exceeds max_frac/FPROP_FRAC_BASE? */ + tmp = (u64 )denominator * max_frac - + ((u64 )numerator <<FPROP_FRAC_SHIFT ); +if (tmp <0 ) { +/* Maximum fraction already exceeded? */ return ; + }else if (tmp < nr * (FPROP_FRAC_BASE - max_frac)) { +/* Add just enough for the fraction to saturate */ + nr =div_u64 (tmp +FPROP_FRAC_BASE - max_frac -1 , +FPROP_FRAC_BASE - max_frac); + } } -__fprop_inc_percpu (p, pl); +__fprop_add_percpu (p, pl, nr); }
mm/page-writeback.c +2 -2
@@ -566,8 +566,8 @@ static void wb_domain_writeout_inc(struct wb_domain *dom,struct fprop_local_percpu *completions, unsigned int max_prop_frac) { - __fprop_inc_percpu_max(&dom->completions, completions, -max_prop_frac ); + __fprop_add_percpu_max(&dom->completions, completions, + max_prop_frac, 1);/* First event after period switching was turned off? */ if (unlikely (!dom->period_time)) {/*
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 15/39 mm/writeback: Change __wb_writeout_inc() to __wb_writeout_add()
In reply to Matthew Wilcox (Oracle)Allow for accounting N pages at once instead of one page at a time.
--- mm/page-writeback.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-)
mm/page-writeback.c +11 -11
@@ -562,12 +562,12 @@ static unsigned long wp_next_time(unsigned long cur_time)return cur_time; } -static void wb_domain_writeout_inc (struct wb_domain *dom, +static void wb_domain_writeout_add (struct wb_domain *dom,struct fprop_local_percpu *completions, -unsigned int max_prop_frac) +unsigned int max_prop_frac,long nr) {__fprop_add_percpu_max (&dom->completions, completions, - max_prop_frac,1 ); + max_prop_frac, nr);/* First event after period switching was turned off? */ if (unlikely (!dom->period_time)) {/*
@@ -585,18 +585,18 @@ static void wb_domain_writeout_inc(struct wb_domain *dom,* Increment @wb's writeout completion count and the global writeout * completion count. Called from test_clear_page_writeback(). */ -static inline void __wb_writeout_inc (struct bdi_writeback *wb) +static inline void __wb_writeout_add (struct bdi_writeback *wb,long nr) {struct wb_domain *cgdom; -inc_wb_stat (wb,WB_WRITTEN ); -wb_domain_writeout_inc (&global_wb_domain, &wb->completions, - wb->bdi->max_prop_frac); +wb_stat_mod (wb,WB_WRITTEN , nr); +wb_domain_writeout_add (&global_wb_domain, &wb->completions, + wb->bdi->max_prop_frac, nr); cgdom =mem_cgroup_wb_domain (wb);if (cgdom) -wb_domain_writeout_inc (cgdom,wb_memcg_completions (wb), - wb->bdi->max_prop_frac); +wb_domain_writeout_add (cgdom,wb_memcg_completions (wb), + wb->bdi->max_prop_frac, nr); }void wb_writeout_inc (struct bdi_writeback *wb)
@@ -604,7 +604,7 @@ void wb_writeout_inc(struct bdi_writeback *wb)unsigned long flags;local_irq_save (flags); -__wb_writeout_inc (wb); +__wb_writeout_add (wb,1 );local_irq_restore (flags); }EXPORT_SYMBOL_GPL (wb_writeout_inc);
@@ -2751,7 +2751,7 @@ int test_clear_page_writeback(struct page *page)struct bdi_writeback *wb =inode_to_wb (inode);dec_wb_stat (wb,WB_WRITEBACK ); -__wb_writeout_inc (wb); +__wb_writeout_add (wb,1 ); } }
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 16/39 mm/writeback: Add __folio_end_writeback()
In reply to Matthew Wilcox (Oracle)test_clear_page_writeback() is actually an mm-internal function, although it's named as if it's a pagecache function. Move it to mm/internal.h, rename it to __folio_end_writeback() and change the return type to bool.
The conversion from page to folio is mostly about accounting the number of pages being written back, although it does eliminate a couple of calls to compound_head().
--- include/linux/page-flags.h | 1 - mm/filemap.c | 2 +- mm/internal.h | 1 + mm/page-writeback.c | 29 +++++++++++++++-------------- 4 files changed, 17 insertions(+), 16 deletions(-)
include/linux/page-flags.h +0 -1
@@ -655,7 +655,6 @@ static __always_inline void SetPageUptodate(struct page *page) CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL) -int test_clear_page_writeback (struct page *page);int __test_set_page_writeback (struct page *page,bool keep_write);#define test_set_page_writeback (page) \
mm/filemap.c +1 -1
@@ -1535,7 +1535,7 @@ void folio_end_writeback(struct folio *folio)* reused before the folio_wake(). */ folio_get (folio); -if (!test_clear_page_writeback (&folio->page)) +if (!__folio_end_writeback (folio))BUG ();smp_mb__after_atomic ();
mm/internal.h +1 -0
@@ -43,6 +43,7 @@ static inline void *folio_raw_mapping(struct folio *folio)vm_fault_t do_swap_page (struct vm_fault *vmf);void folio_rotate_reclaimable (struct folio *folio); +bool __folio_end_writeback (struct folio *folio);void free_pgtables (struct mmu_gather *tlb,struct vm_area_struct *start_vma,unsigned long floor,unsigned long ceiling);
mm/page-writeback.c +15 -14
@@ -583,7 +583,7 @@ static void wb_domain_writeout_add(struct wb_domain *dom,/* * Increment @wb's writeout completion count and the global writeout -* completion count. Called from test_clear_page_writeback(). +* completion count. Called from __folio_end_writeback(). */ static inline void __wb_writeout_add (struct bdi_writeback *wb,long nr) {
@@ -2731,27 +2731,28 @@ int clear_page_dirty_for_io(struct page *page) }EXPORT_SYMBOL (clear_page_dirty_for_io); -int test_clear_page_writeback (struct page *page) +bool __folio_end_writeback (struct folio *folio) { -struct address_space *mapping =page_mapping (page); -int ret; +long nr =folio_nr_pages (folio); +struct address_space *mapping =folio_mapping (folio); +bool ret; -lock_page_memcg (page); +folio_memcg_lock (folio);if (mapping &&mapping_use_writeback_tags (mapping)) {struct inode *inode = mapping->host;struct backing_dev_info *bdi =inode_to_bdi (inode);unsigned long flags;xa_lock_irqsave (&mapping->i_pages, flags); - ret =TestClearPageWriteback (page); + ret =folio_test_clear_writeback (folio);if (ret) { -__xa_clear_mark (&mapping->i_pages,page_index (page), +__xa_clear_mark (&mapping->i_pages,folio_index (folio),PAGECACHE_TAG_WRITEBACK );if (bdi->capabilities &BDI_CAP_WRITEBACK_ACCT ) {struct bdi_writeback *wb =inode_to_wb (inode); -dec_wb_stat (wb,WB_WRITEBACK ); -__wb_writeout_add (wb,1 ); +wb_stat_mod (wb,WB_WRITEBACK , -nr); +__wb_writeout_add (wb, nr); } }
@@ -2761,14 +2762,14 @@ int test_clear_page_writeback(struct page *page)xa_unlock_irqrestore (&mapping->i_pages, flags); }else { - ret =TestClearPageWriteback (page); + ret =folio_test_clear_writeback (folio); }if (ret) { -dec_lruvec_page_state (page,NR_WRITEBACK ); -dec_zone_page_state (page,NR_ZONE_WRITE_PENDING ); -inc_node_page_state (page,NR_WRITTEN ); +lruvec_stat_mod_folio (folio,NR_WRITEBACK , -nr); +zone_stat_mod_folio (folio,NR_ZONE_WRITE_PENDING , -nr); +node_stat_mod_folio (folio,NR_WRITTEN , nr); } -unlock_page_memcg (page); +folio_memcg_unlock (folio);return ret; }
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 17/39 mm/writeback: Add folio_start_writeback()
In reply to Matthew Wilcox (Oracle)Rename set_page_writeback() to folio_start_writeback() to match folio_end_writeback(). Do not bother with wrappers that return void; callers are perfectly capable of ignoring return values.
Add wrappers for set_page_writeback(), set_page_writeback_keepwrite() and test_set_page_writeback() for compatibililty with existing filesystems. The main advantage of this patch is getting the statistics right, although it does eliminate a couple of calls to compound_head().
--- include/linux/page-flags.h | 19 +++++++++--------- mm/folio-compat.c | 6 ++++++ mm/page-writeback.c | 40 ++++++++++++++++++++------------------ 3 files changed, 37 insertions(+), 28 deletions(-)
include/linux/page-flags.h +10 -9
@@ -655,21 +655,22 @@ static __always_inline void SetPageUptodate(struct page *page) CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL) -int __test_set_page_writeback (struct page *page,bool keep_write); +bool __folio_start_writeback (struct folio *folio,bool keep_write); +bool set_page_writeback (struct page *page); -#define test_set_page_writeback (page) \ - __test_set_page_writeback(page, false) -#define test_set_page_writeback_keepwrite (page) \ - __test_set_page_writeback(page, true) +#define folio_start_writeback (folio) \ + __folio_start_writeback(folio, false) +#define folio_start_writeback_keepwrite (folio) \ + __folio_start_writeback(folio, true) -static inline void set_page_writeback (struct page *page) +static inline void set_page_writeback_keepwrite (struct page *page) { -test_set_page_writeback (page); +folio_start_writeback_keepwrite (page_folio (page)); } -static inline void set_page_writeback_keepwrite (struct page *page) +static inline bool test_set_page_writeback (struct page *page) { -test_set_page_writeback_keepwrite (page); +return set_page_writeback (page); } __PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
mm/folio-compat.c +6 -0
@@ -71,3 +71,9 @@ void migrate_page_copy(struct page *newpage, struct page *page) }EXPORT_SYMBOL (migrate_page_copy);#endif + +bool set_page_writeback (struct page *page) +{ +return folio_start_writeback (page_folio (page)); +} +EXPORT_SYMBOL (set_page_writeback);
mm/page-writeback.c +21 -19
@@ -2773,21 +2773,23 @@ bool __folio_end_writeback(struct folio *folio)return ret; } -int __test_set_page_writeback (struct page *page,bool keep_write) +bool __folio_start_writeback (struct folio *folio,bool keep_write) { -struct address_space *mapping =page_mapping (page); -int ret, access_ret; +long nr =folio_nr_pages (folio); +struct address_space *mapping =folio_mapping (folio); +bool ret; +int access_ret; -lock_page_memcg (page); +folio_memcg_lock (folio);if (mapping &&mapping_use_writeback_tags (mapping)) { -XA_STATE (xas, &mapping->i_pages,page_index (page)); +XA_STATE (xas, &mapping->i_pages,folio_index (folio));struct inode *inode = mapping->host;struct backing_dev_info *bdi =inode_to_bdi (inode);unsigned long flags;xas_lock_irqsave (&xas, flags);xas_load (&xas); - ret =TestSetPageWriteback (page); + ret =folio_test_set_writeback (folio);if (!ret) {bool on_wblist;
@@ -2796,40 +2798,40 @@ int __test_set_page_writeback(struct page *page, bool keep_write)xas_set_mark (&xas,PAGECACHE_TAG_WRITEBACK );if (bdi->capabilities &BDI_CAP_WRITEBACK_ACCT ) -inc_wb_stat (inode_to_wb (inode),WB_WRITEBACK ); +wb_stat_mod (inode_to_wb (inode),WB_WRITEBACK , + nr);/* -* We can come through here when swapping anonymous -* pages, so we don't necessarily have an inode to track -* for sync. +* We can come through here when swapping +* anonymous folios, so we don't necessarily +* have an inode to track for sync. */ if (mapping->host && !on_wblist)sb_mark_inode_writeback (mapping->host); } -if (!PageDirty (page)) +if (!folio_test_dirty (folio))xas_clear_mark (&xas,PAGECACHE_TAG_DIRTY );if (!keep_write)xas_clear_mark (&xas,PAGECACHE_TAG_TOWRITE );xas_unlock_irqrestore (&xas, flags); }else { - ret =TestSetPageWriteback (page); + ret =folio_test_set_writeback (folio); }if (!ret) { -inc_lruvec_page_state (page,NR_WRITEBACK ); -inc_zone_page_state (page,NR_ZONE_WRITE_PENDING ); +lruvec_stat_mod_folio (folio,NR_WRITEBACK , nr); +zone_stat_mod_folio (folio,NR_ZONE_WRITE_PENDING , nr); } -unlock_page_memcg (page); - access_ret =arch_make_page_accessible (page); +folio_memcg_unlock (folio); + access_ret =arch_make_folio_accessible (folio);/* * If writeback has been triggered on a page that cannot be made * accessible, it is too late to recover here. */ -VM_BUG_ON_PAGE (access_ret !=0 , page); +VM_BUG_ON_FOLIO (access_ret !=0 , folio);return ret; - } -EXPORT_SYMBOL (__test_set_page_writeback); +EXPORT_SYMBOL (__folio_start_writeback);/** * folio_wait_writeback - Wait for a folio to finish writeback.
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 18/39 mm/writeback: Add folio_mark_dirty()
In reply to Matthew Wilcox (Oracle)Reimplement set_page_dirty() as a wrapper around folio_mark_dirty(). There is no change to filesystems as they were already being called with the compound_head of the page being marked dirty. We avoid several calls to compound_head(), both statically (through using folio_test_dirty() instead of PageDirty() and dynamically by calling folio_mapping() instead of page_mapping().
Also return bool instead of int to show the range of values actually returned, and add kernel-doc.
--- include/linux/mm.h | 3 ++- mm/folio-compat.c | 6 ++++++ mm/page-writeback.c | 35 +++++++++++++++++++---------------- 3 files changed, 27 insertions(+), 17 deletions(-)
include/linux/mm.h +2 -1
@@ -2005,7 +2005,8 @@ int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page);void account_page_cleaned (struct page *page,struct address_space *mapping,struct bdi_writeback *wb); -int set_page_dirty (struct page *page); +bool folio_mark_dirty (struct folio *folio); +bool set_page_dirty (struct page *page);int set_page_dirty_lock (struct page *page);void __cancel_dirty_page (struct page *page);static inline void cancel_dirty_page (struct page *page)
mm/folio-compat.c +6 -0
@@ -77,3 +77,9 @@ bool set_page_writeback(struct page *page)return folio_start_writeback (page_folio (page)); }EXPORT_SYMBOL (set_page_writeback); + +bool set_page_dirty (struct page *page) +{ +return folio_mark_dirty (page_folio (page)); +} +EXPORT_SYMBOL (set_page_dirty);
mm/page-writeback.c +19 -16
@@ -2564,18 +2564,21 @@ int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page) }EXPORT_SYMBOL (redirty_page_for_writepage); -/* -* Dirty a page. +/** +* folio_mark_dirty - Mark a folio as being modified. +* @folio: The folio. +* +* For folios with a mapping this should be done under the page lock +* for the benefit of asynchronous memory errors who prefer a consistent +* dirty state. This rule can be broken in some special cases, +* but should be better not to. * -* For pages with a mapping this should be done under the page lock for the -* benefit of asynchronous memory errors who prefer a consistent dirty state. -* This rule can be broken in some special cases, but should be better not to. +* Return: True if the folio was newly dirtied, false if it was already dirty. */ -int set_page_dirty (struct page *page) +bool folio_mark_dirty (struct folio *folio) { -struct address_space *mapping =page_mapping (page); +struct address_space *mapping =folio_mapping (folio); - page =compound_head (page);if (likely (mapping)) {/* * readahead/lru_deactivate_page could remain
@@ -2587,17 +2590,17 @@ int set_page_dirty(struct page *page)* it will confuse readahead and make it restart the size rampup * process. But it's a trivial problem. */ -if (PageReclaim (page)) -ClearPageReclaim (page); -return mapping->a_ops->set_page_dirty (page); +if (folio_test_reclaim (folio)) +folio_clear_reclaim (folio); +return mapping->a_ops->set_page_dirty (&folio->page); } -if (!PageDirty (page)) { -if (!TestSetPageDirty (page)) -return 1 ; +if (!folio_test_dirty (folio)) { +if (!folio_test_set_dirty (folio)) +return true; } -return 0 ; +return false; } -EXPORT_SYMBOL (set_page_dirty); +EXPORT_SYMBOL (folio_mark_dirty);/* * set_page_dirty() is racy if the caller has no reference against
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 19/39 mm/writeback: Add __folio_mark_dirty()
In reply to Matthew Wilcox (Oracle)Turn __set_page_dirty() into a wrapper around __folio_mark_dirty(). Convert account_page_dirtied() into folio_account_dirtied() and account the number of pages in the folio to support multi-page folios.
--- include/linux/memcontrol.h | 5 ++--- include/linux/pagemap.h | 7 ++++++- mm/page-writeback.c | 41 +++++++++++++++++++------------------- 3 files changed, 29 insertions(+), 24 deletions(-)
include/linux/memcontrol.h +2 -3
@@ -1574,10 +1574,9 @@ void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,void mem_cgroup_track_foreign_dirty_slowpath (struct folio *folio,struct bdi_writeback *wb); -static inline void mem_cgroup_track_foreign_dirty (struct page *page, +static inline void mem_cgroup_track_foreign_dirty (struct folio *folio,struct bdi_writeback *wb) { -struct folio *folio =page_folio (page);if (mem_cgroup_disabled ())return ;
@@ -1602,7 +1601,7 @@ static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb, { } -static inline void mem_cgroup_track_foreign_dirty (struct page *page, +static inline void mem_cgroup_track_foreign_dirty (struct folio *folio,struct bdi_writeback *wb) { }
include/linux/pagemap.h +6 -1
@@ -773,8 +773,13 @@ void end_page_writeback(struct page *page);void folio_end_writeback (struct folio *folio);void wait_for_stable_page (struct page *page);void folio_wait_stable (struct folio *folio); +void __folio_mark_dirty (struct folio *folio,struct address_space *,int warn); +static inline void __set_page_dirty (struct page *page, +struct address_space *mapping,int warn) +{ +__folio_mark_dirty (page_folio (page), mapping, warn); +} -void __set_page_dirty (struct page *,struct address_space *,int warn);int __set_page_dirty_nobuffers (struct page *page);int __set_page_dirty_no_writeback (struct page *page);
mm/page-writeback.c +21 -20
74 lines not on this page · raw message
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 20/39 mm/writeback: Convert tracing writeback_page_template to folios
In reply to Matthew Wilcox (Oracle)Rename writeback_dirty_page() to writeback_dirty_folio() and wait_on_page_writeback() to folio_wait_writeback().
--- include/trace/events/writeback.h | 20 ++++++++++---------- mm/page-writeback.c | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-)
include/trace/events/writeback.h +10 -10
@@ -52,11 +52,11 @@ WB_WORK_REASONstruct wb_writeback_work ; -DECLARE_EVENT_CLASS(writeback_page_template, +DECLARE_EVENT_CLASS(writeback_folio_template, - TP_PROTO(struct page *page, struct address_space *mapping), + TP_PROTO(struct folio *folio, struct address_space *mapping), - TP_ARGS(page, mapping), + TP_ARGS(folio, mapping), TP_STRUCT__entry ( __array(char, name, 32)
@@ -69,7 +69,7 @@ DECLARE_EVENT_CLASS(writeback_page_template,bdi_dev_name (mapping ?inode_to_bdi (mapping->host) : NULL), 32); __entry->ino = mapping ? mapping->host->i_ino :0 ; - __entry->index = page->index; + __entry->index = folio->index; ), TP_printk("bdi %s: ino=%lu index=%lu",
@@ -79,18 +79,18 @@ DECLARE_EVENT_CLASS(writeback_page_template, ) ); -DEFINE_EVENT (writeback_page_template, writeback_dirty_page, +DEFINE_EVENT (writeback_folio_template, writeback_dirty_folio, - TP_PROTO(struct page *page, struct address_space *mapping), + TP_PROTO(struct folio *folio, struct address_space *mapping), -TP_ARGS (page, mapping) +TP_ARGS (folio, mapping) ); -DEFINE_EVENT (writeback_page_template, wait_on_page_writeback, +DEFINE_EVENT (writeback_folio_template, folio_wait_writeback, - TP_PROTO(struct page *page, struct address_space *mapping), + TP_PROTO(struct folio *folio, struct address_space *mapping), -TP_ARGS (page, mapping) +TP_ARGS (folio, mapping) ); DECLARE_EVENT_CLASS(writeback_dirty_inode_template,
mm/page-writeback.c +3 -3
@@ -2426,7 +2426,7 @@ static void folio_account_dirtied(struct folio *folio, {struct inode *inode = mapping->host; -trace_writeback_dirty_page (&folio->page, mapping); +trace_writeback_dirty_folio (folio, mapping);if (mapping_can_writeback (mapping)) {struct bdi_writeback *wb;
@@ -2852,7 +2852,7 @@ EXPORT_SYMBOL(__folio_start_writeback);void folio_wait_writeback (struct folio *folio) {while (folio_test_writeback (folio)) { -trace_wait_on_page_writeback (&folio->page,folio_mapping (folio)); +trace_folio_wait_writeback (folio,folio_mapping (folio));folio_wait_bit (folio, PG_writeback); } }
@@ -2874,7 +2874,7 @@ EXPORT_SYMBOL_GPL(folio_wait_writeback);int folio_wait_writeback_killable (struct folio *folio) {while (folio_test_writeback (folio)) { -trace_wait_on_page_writeback (&folio->page,folio_mapping (folio)); +trace_folio_wait_writeback (folio,folio_mapping (folio));if (folio_wait_bit_killable (folio, PG_writeback))return -EINTR ; }
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 21/39 mm/writeback: Add filemap_dirty_folio()
In reply to Matthew Wilcox (Oracle)Reimplement __set_page_dirty_nobuffers() as a wrapper around filemap_dirty_folio().
--- include/linux/writeback.h | 1 + mm/folio-compat.c | 6 ++++ mm/page-writeback.c | 60 ++++++++++++++++++++------------------- 3 files changed, 38 insertions(+), 29 deletions(-)
include/linux/writeback.h +1 -0
@@ -398,6 +398,7 @@ void writeback_set_ratelimit(void);void tag_pages_for_writeback (struct address_space *mapping,pgoff_t start,pgoff_t end); +bool filemap_dirty_folio (struct address_space *mapping,struct folio *folio);void account_page_redirty (struct page *page);void sb_mark_inode_writeback (struct inode *inode);
mm/folio-compat.c +6 -0
@@ -83,3 +83,9 @@ bool set_page_dirty(struct page *page)return folio_mark_dirty (page_folio (page)); }EXPORT_SYMBOL (set_page_dirty); + +int __set_page_dirty_nobuffers (struct page *page) +{ +return filemap_dirty_folio (page_mapping (page),page_folio (page)); +} +EXPORT_SYMBOL (__set_page_dirty_nobuffers);
mm/page-writeback.c +31 -29
72 lines not on this page · raw message
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 22/39 mm/writeback: Add folio_account_cleaned()
In reply to Matthew Wilcox (Oracle)Get the statistics right; compound pages were being accounted as a single page. This didn't matter before now as no filesystem which supported compound pages did writeback. Also move the declaration to filemap.h since this is part of the page cache. Add a wrapper for account_page_cleaned().
--- include/linux/mm.h | 3 --- include/linux/pagemap.h | 7 +++++++ mm/page-writeback.c | 11 ++++++----- 3 files changed, 13 insertions(+), 8 deletions(-)
include/linux/mm.h +0 -3
@@ -39,7 +39,6 @@ struct anon_vma_chain;struct file_ra_state ;struct user_struct ;struct writeback_control ; -struct bdi_writeback ;struct pt_regs ;extern int sysctl_page_lock_unfairness;
@@ -2003,8 +2002,6 @@ extern void do_invalidatepage(struct page *page, unsigned int offset,int redirty_page_for_writepage (struct writeback_control *wbc,struct page *page); -void account_page_cleaned (struct page *page,struct address_space *mapping, -struct bdi_writeback *wb);bool folio_mark_dirty (struct folio *folio);bool set_page_dirty (struct page *page);int set_page_dirty_lock (struct page *page);
include/linux/pagemap.h +7 -0
@@ -779,6 +779,13 @@ static inline void __set_page_dirty(struct page *page, {__folio_mark_dirty (page_folio (page), mapping, warn); } +void folio_account_cleaned (struct folio *folio,struct address_space *mapping, +struct bdi_writeback *wb); +static inline void account_page_cleaned (struct page *page, +struct address_space *mapping,struct bdi_writeback *wb) +{ +return folio_account_cleaned (page_folio (page), mapping, wb); +}int __set_page_dirty_nobuffers (struct page *page);int __set_page_dirty_no_writeback (struct page *page);
mm/page-writeback.c +6 -5
@@ -2453,14 +2453,15 @@ static void folio_account_dirtied(struct folio *folio,* * Caller must hold lock_page_memcg(). */ -void account_page_cleaned (struct page *page,struct address_space *mapping, +void folio_account_cleaned (struct folio *folio,struct address_space *mapping,struct bdi_writeback *wb) {if (mapping_can_writeback (mapping)) { -dec_lruvec_page_state (page,NR_FILE_DIRTY ); -dec_zone_page_state (page,NR_ZONE_WRITE_PENDING ); -dec_wb_stat (wb,WB_RECLAIMABLE ); -task_io_account_cancelled_write (PAGE_SIZE ); +long nr =folio_nr_pages (folio); +lruvec_stat_mod_folio (folio,NR_FILE_DIRTY , -nr); +zone_stat_mod_folio (folio,NR_ZONE_WRITE_PENDING , -nr); +wb_stat_mod (wb,WB_RECLAIMABLE , -nr); +task_io_account_cancelled_write (folio_size (folio)); } }
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 23/39 mm/writeback: Add folio_cancel_dirty()
In reply to Matthew Wilcox (Oracle)Turn __cancel_dirty_page() into __folio_cancel_dirty() and add wrappers. Move the prototypes into pagemap.h since this is page cache functionality. Saves 44 bytes of kernel text in total; 33 bytes from __folio_cancel_dirty and 11 from two callers of cancel_dirty_page().
--- include/linux/mm.h | 7 ------- include/linux/pagemap.h | 11 +++++++++++ mm/page-writeback.c | 16 ++++++++-------- 3 files changed, 19 insertions(+), 15 deletions(-)
include/linux/mm.h +0 -7
@@ -2005,13 +2005,6 @@ int redirty_page_for_writepage(struct writeback_control *wbc,bool folio_mark_dirty (struct folio *folio);bool set_page_dirty (struct page *page);int set_page_dirty_lock (struct page *page); -void __cancel_dirty_page (struct page *page); -static inline void cancel_dirty_page (struct page *page) -{ -/* Avoid atomic ops, locking, etc. when not actually needed. */ -if (PageDirty (page)) -__cancel_dirty_page (page); -}int clear_page_dirty_for_io (struct page *page);int get_cmdline (struct task_struct *task,char *buffer,int buflen);
include/linux/pagemap.h +11 -0
@@ -786,6 +786,17 @@ static inline void account_page_cleaned(struct page *page, {return folio_account_cleaned (page_folio (page), mapping, wb); } +void __folio_cancel_dirty (struct folio *folio); +static inline void folio_cancel_dirty (struct folio *folio) +{ +/* Avoid atomic ops, locking, etc. when not actually needed. */ +if (folio_test_dirty (folio)) +__folio_cancel_dirty (folio); +} +static inline void cancel_dirty_page (struct page *page) +{ +folio_cancel_dirty (page_folio (page)); +}int __set_page_dirty_nobuffers (struct page *page);int __set_page_dirty_no_writeback (struct page *page);
mm/page-writeback.c +8 -8
@@ -2640,28 +2640,28 @@ EXPORT_SYMBOL(set_page_dirty_lock);* page without actually doing it through the VM. Can you say "ext3 is * horribly ugly"? Thought you could. */ -void __cancel_dirty_page (struct page *page) +void __folio_cancel_dirty (struct folio *folio) { -struct address_space *mapping =page_mapping (page); +struct address_space *mapping =folio_mapping (folio);if (mapping_can_writeback (mapping)) {struct inode *inode = mapping->host;struct bdi_writeback *wb;struct wb_lock_cookie cookie = {}; -lock_page_memcg (page); +folio_memcg_lock (folio); wb =unlocked_inode_to_wb_begin (inode, &cookie); -if (TestClearPageDirty (page)) -account_page_cleaned (page, mapping, wb); +if (folio_test_clear_dirty (folio)) +folio_account_cleaned (folio, mapping, wb);unlocked_inode_to_wb_end (inode, &cookie); -unlock_page_memcg (page); +folio_memcg_unlock (folio); }else { -ClearPageDirty (page); +folio_clear_dirty (folio); } } -EXPORT_SYMBOL (__cancel_dirty_page); +EXPORT_SYMBOL (__folio_cancel_dirty);/* * Clear a page's dirty flag, while caring for dirty memory accounting.
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 24/39 mm/writeback: Add folio_clear_dirty_for_io()
In reply to Matthew Wilcox (Oracle)Transform clear_page_dirty_for_io() into folio_clear_dirty_for_io() and add a compatibility wrapper. Also move the declaration to pagemap.h as this is page cache functionality that doesn't need to be used by the rest of the kernel.
Increases the size of the kernel by 79 bytes. While we remove a few calls to compound_head(), we add a call to folio_nr_pages() to get the stats correct for the eventual support of multi-page folios.
--- include/linux/mm.h | 1 - include/linux/pagemap.h | 2 ++ mm/folio-compat.c | 6 ++++ mm/page-writeback.c | 63 +++++++++++++++++++++-------------------- 4 files changed, 40 insertions(+), 32 deletions(-)
include/linux/mm.h +0 -1
@@ -2005,7 +2005,6 @@ int redirty_page_for_writepage(struct writeback_control *wbc,bool folio_mark_dirty (struct folio *folio);bool set_page_dirty (struct page *page);int set_page_dirty_lock (struct page *page); -int clear_page_dirty_for_io (struct page *page);int get_cmdline (struct task_struct *task,char *buffer,int buflen);
include/linux/pagemap.h +2 -0
@@ -797,6 +797,8 @@ static inline void cancel_dirty_page(struct page *page) {folio_cancel_dirty (page_folio (page)); } +bool folio_clear_dirty_for_io (struct folio *folio); +bool clear_page_dirty_for_io (struct page *page);int __set_page_dirty_nobuffers (struct page *page);int __set_page_dirty_no_writeback (struct page *page);
mm/folio-compat.c +6 -0
@@ -89,3 +89,9 @@ int __set_page_dirty_nobuffers(struct page *page)return filemap_dirty_folio (page_mapping (page),page_folio (page)); }EXPORT_SYMBOL (__set_page_dirty_nobuffers); + +bool clear_page_dirty_for_io (struct page *page) +{ +return folio_clear_dirty_for_io (page_folio (page)); +} +EXPORT_SYMBOL (clear_page_dirty_for_io);
mm/page-writeback.c +32 -31
105 lines not on this page · raw message
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 25/39 mm/writeback: Add folio_account_redirty()
In reply to Matthew Wilcox (Oracle)Account the number of pages in the folio that we're redirtying. Turn account_page_dirty() into a wrapper around it. Also turn the comment on folio_account_redirty() into kernel-doc and edit it slightly so it makes sense to its potential callers.
--- include/linux/writeback.h | 6 +++++- mm/page-writeback.c | 32 +++++++++++++++++++------------- 2 files changed, 24 insertions(+), 14 deletions(-)
include/linux/writeback.h +5 -1
@@ -399,7 +399,11 @@ void tag_pages_for_writeback(struct address_space *mapping, pgoff_t start, pgoff_t end);bool filemap_dirty_folio (struct address_space *mapping,struct folio *folio); -void account_page_redirty (struct page *page); +void folio_account_redirty (struct folio *folio); +static inline void account_page_redirty (struct page *page) +{ +folio_account_redirty (page_folio (page)); +}void sb_mark_inode_writeback (struct inode *inode);void sb_clear_inode_writeback (struct inode *inode);
mm/page-writeback.c +19 -13
@@ -1084,7 +1084,7 @@ static void wb_update_write_bandwidth(struct bdi_writeback *wb,* write_bandwidth = --------------------------------------------------- * period * -* @written may have decreased due to account_page_redirty(). +* @written may have decreased due to folio_account_redirty(). * Avoid underflowing @bw calculation. */ bw = written -min (written, wb->written_stamp);
@@ -2527,30 +2527,36 @@ bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio) }EXPORT_SYMBOL (filemap_dirty_folio); -/* -* Call this whenever redirtying a page, to de-account the dirty counters -* (NR_DIRTIED, WB_DIRTIED, tsk->nr_dirtied), so that they match the written -* counters (NR_WRITTEN, WB_WRITTEN) in long term. The mismatches will lead to -* systematic errors in balanced_dirty_ratelimit and the dirty pages position -* control. +/** +* folio_account_redirty - Manually account for redirtying a page. +* @folio: The folio which is being redirtied. +* +* Most filesystems should call folio_redirty_for_writepage() instead +* of this fuction. If your filesystem is doing writeback outside the +* context of a writeback_control(), it can call this when redirtying +* a folio, to de-account the dirty counters (NR_DIRTIED, WB_DIRTIED, +* tsk->nr_dirtied), so that they match the written counters (NR_WRITTEN, +* WB_WRITTEN) in long term. The mismatches will lead to systematic errors +* in balanced_dirty_ratelimit and the dirty pages position control. */ -void account_page_redirty (struct page *page) +void folio_account_redirty (struct folio *folio) { -struct address_space *mapping = page->mapping; +struct address_space *mapping = folio->mapping;if (mapping &&mapping_can_writeback (mapping)) {struct inode *inode = mapping->host;struct bdi_writeback *wb;struct wb_lock_cookie cookie = {}; +unsigned nr =folio_nr_pages (folio); wb =unlocked_inode_to_wb_begin (inode, &cookie); - current->nr_dirtied--; -dec_node_page_state (page,NR_DIRTIED ); -dec_wb_stat (wb,WB_DIRTIED ); + current->nr_dirtied -= nr; +node_stat_mod_folio (folio,NR_DIRTIED , -nr); +wb_stat_mod (wb,WB_DIRTIED , -nr);unlocked_inode_to_wb_end (inode, &cookie); } } -EXPORT_SYMBOL (account_page_redirty); +EXPORT_SYMBOL (folio_account_redirty);/* * When a writepage implementation decides that it doesn't want to write this
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 26/39 mm/writeback: Add folio_redirty_for_writepage()
In reply to Matthew Wilcox (Oracle)Reimplement redirty_page_for_writepage() as a wrapper around folio_redirty_for_writepage(). Account the number of pages in the folio, add kernel-doc and move the prototype to writeback.h.
--- fs/jfs/jfs_metapage.c | 1 + include/linux/mm.h | 4 ---- include/linux/writeback.h | 2 ++ mm/folio-compat.c | 7 +++++++ mm/page-writeback.c | 30 ++++++++++++++++++++---------- 5 files changed, 30 insertions(+), 14 deletions(-)
fs/jfs/jfs_metapage.c +1 -0
@@ -13,6 +13,7 @@#include <linux/buffer_head.h> #include <linux/mempool.h> #include <linux/seq_file.h> +#include <linux/writeback.h> #include "jfs_incore.h" #include "jfs_superblock.h" #include "jfs_filsys.h"
include/linux/mm.h +0 -4
@@ -36,9 +36,7 @@struct mempolicy ;struct anon_vma ;struct anon_vma_chain ; -struct file_ra_state ;struct user_struct ; -struct writeback_control ;struct pt_regs ;extern int sysctl_page_lock_unfairness;
@@ -2000,8 +1998,6 @@ extern int try_to_release_page(struct page * page, gfp_t gfp_mask);extern void do_invalidatepage (struct page *page,unsigned int offset,unsigned int length); -int redirty_page_for_writepage (struct writeback_control *wbc, -struct page *page);bool folio_mark_dirty (struct folio *folio);bool set_page_dirty (struct page *page);int set_page_dirty_lock (struct page *page);
include/linux/writeback.h +2 -0
@@ -404,6 +404,8 @@ static inline void account_page_redirty(struct page *page) {folio_account_redirty (page_folio (page)); } +bool folio_redirty_for_writepage (struct writeback_control *,struct folio *); +bool redirty_page_for_writepage (struct writeback_control *,struct page *);void sb_mark_inode_writeback (struct inode *inode);void sb_clear_inode_writeback (struct inode *inode);
mm/folio-compat.c +7 -0
@@ -95,3 +95,10 @@ bool clear_page_dirty_for_io(struct page *page)return folio_clear_dirty_for_io (page_folio (page)); }EXPORT_SYMBOL (clear_page_dirty_for_io); + +bool redirty_page_for_writepage (struct writeback_control *wbc, +struct page *page) +{ +return folio_redirty_for_writepage (wbc,page_folio (page)); +} +EXPORT_SYMBOL (redirty_page_for_writepage);
mm/page-writeback.c +20 -10
@@ -2558,21 +2558,31 @@ void folio_account_redirty(struct folio *folio) }EXPORT_SYMBOL (folio_account_redirty); -/* -* When a writepage implementation decides that it doesn't want to write this -* page for some reason, it should redirty the locked page via -* redirty_page_for_writepage() and it should then unlock the page and return 0 +/** +* folio_redirty_for_writepage - Decline to write a dirty folio. +* @wbc: The writeback control. +* @folio: The folio. +* +* When a writepage implementation decides that it doesn't want to write +* @folio for some reason, it should call this function, unlock @folio and +* return 0. +* +* Return: True if we redirtied the folio. False if someone else dirtied +* it first. */ -int redirty_page_for_writepage (struct writeback_control *wbc,struct page *page) +bool folio_redirty_for_writepage (struct writeback_control *wbc, +struct folio *folio) { -int ret; +bool ret; +unsigned nr =folio_nr_pages (folio); + + wbc->pages_skipped += nr; + ret =filemap_dirty_folio (folio->mapping, folio); +folio_account_redirty (folio); - wbc->pages_skipped++; - ret =__set_page_dirty_nobuffers (page); -account_page_redirty (page);return ret; } -EXPORT_SYMBOL (redirty_page_for_writepage); +EXPORT_SYMBOL (folio_redirty_for_writepage);/** * folio_mark_dirty - Mark a folio as being modified.
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 27/39 mm/filemap: Add i_blocks_per_folio()
In reply to Matthew Wilcox (Oracle)Reimplement i_blocks_per_page() as a wrapper around i_blocks_per_folio().
--- include/linux/pagemap.h | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-)
include/linux/pagemap.h +12 -6
@@ -1150,19 +1150,25 @@ static inline int page_mkwrite_check_truncate(struct page *page, }/** -* i_blocks_per_page - How many blocks fit in this page. +* i_blocks_per_folio - How many blocks fit in this folio. * @inode: The inode which contains the blocks. -* @page: The page (head page if the page is a THP). +* @folio: The folio. * -* If the block size is larger than the size of this page, return zero. +* If the block size is larger than the size of this folio, return zero. * -* Context: The caller should hold a refcount on the page to prevent it +* Context: The caller should hold a refcount on the folio to prevent it * from being split. -* Return: The number of filesystem blocks covered by this page. +* Return: The number of filesystem blocks covered by this folio. */ +static inline +unsigned int i_blocks_per_folio (struct inode *inode,struct folio *folio) +{ +return folio_size (folio) >> inode->i_blkbits; +} +static inline unsigned int i_blocks_per_page (struct inode *inode,struct page *page) { -return thp_size (page) >> inode->i_blkbits; +return i_blocks_per_folio (inode,page_folio (page)); }#endif /* _LINUX_PAGEMAP_H */
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 28/39 mm/filemap: Add folio_mkwrite_check_truncate()
In reply to Matthew Wilcox (Oracle)This is the folio equivalent of page_mkwrite_check_truncate().
--- include/linux/pagemap.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)
include/linux/pagemap.h +28 -0
@@ -1121,6 +1121,34 @@ static inline unsigned long dir_pages(struct inode *inode)PAGE_SHIFT ; } +/** +* folio_mkwrite_check_truncate - check if folio was truncated +* @folio: the folio to check +* @inode: the inode to check the folio against +* +* Return: the number of bytes in the folio up to EOF, +* or -EFAULT if the folio was truncated. +*/ +static inline ssize_t folio_mkwrite_check_truncate (struct folio *folio, +struct inode *inode) +{ +loff_t size =i_size_read (inode); +pgoff_t index = size >>PAGE_SHIFT ; +size_t offset =offset_in_folio (folio, size); + +if (!folio->mapping) +return -EFAULT ; + +/* folio is wholly inside EOF */ +if (folio_next_index (folio) -1 < index) +return folio_size (folio); +/* folio is wholly past EOF */ +if (folio->index > index || !offset) +return -EFAULT ; +/* folio is partially inside EOF */ +return offset; +} +/** * page_mkwrite_check_truncate - check if page was truncated * @page: the page to check
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 29/39 mm/filemap: Add readahead_folio()
In reply to Matthew Wilcox (Oracle)The pointers stored in the page cache are folios, by definition. This change comes with a behaviour change -- callers of readahead_folio() are no longer required to put the page reference themselves. This matches how readpage works, rather than matching how readpages used to work.
--- include/linux/pagemap.h | 53 +++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 15 deletions(-)
include/linux/pagemap.h +38 -15
@@ -988,33 +988,56 @@ void page_cache_async_readahead(struct address_space *mapping,page_cache_async_ra (&ractl, page, req_count); } +static inline struct folio *__readahead_folio (struct readahead_control *ractl) +{ +struct folio *folio; + +BUG_ON (ractl->_batch_count > ractl->_nr_pages); + ractl->_nr_pages -= ractl->_batch_count; + ractl->_index += ractl->_batch_count; + +if (!ractl->_nr_pages) { + ractl->_batch_count =0 ; +return NULL ; + } + + folio =xa_load (&ractl->mapping->i_pages, ractl->_index); +VM_BUG_ON_FOLIO (!folio_test_locked (folio), folio); + ractl->_batch_count =folio_nr_pages (folio); + +return folio; +} +/** * readahead_page - Get the next page to read. -* @rac: The current readahead request. +* @ractl: The current readahead request. * * Context: The page is locked and has an elevated refcount. The caller * should decreases the refcount once the page has been submitted for I/O * and unlock the page once all I/O to that page has completed. * Return: A pointer to the next page, or %NULL if we are done. */ -static inline struct page *readahead_page (struct readahead_control *rac) +static inline struct page *readahead_page (struct readahead_control *ractl) { -struct page *page; +struct folio *folio =__readahead_folio (ractl); -BUG_ON (rac->_batch_count > rac->_nr_pages); - rac->_nr_pages -= rac->_batch_count; - rac->_index += rac->_batch_count; - -if (!rac->_nr_pages) { - rac->_batch_count =0 ; -return NULL ; - } +return &folio->page; +} - page =xa_load (&rac->mapping->i_pages, rac->_index); -VM_BUG_ON_PAGE (!PageLocked (page), page); - rac->_batch_count =thp_nr_pages (page); +/** +* readahead_folio - Get the next folio to read. +* @ractl: The current readahead request. +* +* Context: The folio is locked. The caller should unlock the folio once +* all I/O to that folio has completed. +* Return: A pointer to the next folio, or %NULL if we are done. +*/ +static inline struct folio *readahead_folio (struct readahead_control *ractl) +{ +struct folio *folio =__readahead_folio (ractl); -return page; +folio_put (folio); +return folio; } static inline unsigned int __readahead_batch(struct readahead_control *rac,
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 30/39 mm/workingset: Convert workingset_refault() to take a folio
In reply to Matthew Wilcox (Oracle)This nets us 178 bytes of savings from removing calls to compound_head. The three callers all grow a little, but each of them will be converted to use folios soon, so that's fine.
--- include/linux/swap.h | 4 ++-- mm/filemap.c | 2 +- mm/memory.c | 3 ++- mm/swap.c | 7 +++---- mm/swap_state.c | 2 +- mm/workingset.c | 34 +++++++++++++++++----------------- 6 files changed, 26 insertions(+), 26 deletions(-)
include/linux/swap.h +2 -2
@@ -329,7 +329,7 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio)/* linux/mm/workingset.c */ void workingset_age_nonresident (struct lruvec *lruvec,unsigned long nr_pages);void *workingset_eviction (struct page *page,struct mem_cgroup *target_memcg); -void workingset_refault (struct page *page,void *shadow); +void workingset_refault (struct folio *folio,void *shadow);void workingset_activation (struct folio *folio);/* Only track the nodes of mappings with shadow entries */
@@ -350,7 +350,7 @@ extern unsigned long nr_free_buffer_pages(void);/* linux/mm/swap.c */ extern void lru_note_cost (struct lruvec *lruvec,bool file,unsigned int nr_pages); -extern void lru_note_cost_page (struct page *); +extern void lru_note_cost_folio (struct folio *);extern void lru_cache_add (struct page *);void mark_page_accessed (struct page *);void folio_mark_accessed (struct folio *);
mm/filemap.c +1 -1
@@ -981,7 +981,7 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,*/ WARN_ON_ONCE (PageActive (page));if (!(gfp_mask & __GFP_WRITE) && shadow) -workingset_refault (page, shadow); +workingset_refault (page_folio (page), shadow);lru_cache_add (page); }return ret;
mm/memory.c +2 -1
@@ -3538,7 +3538,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) shadow =get_shadow_from_swap_cache (entry);if (shadow) -workingset_refault (page, shadow); +workingset_refault (page_folio (page), + shadow);lru_cache_add (page);
mm/swap.c +3 -4
@@ -315,11 +315,10 @@ void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages) }while ((lruvec =parent_lruvec (lruvec))); } -void lru_note_cost_page (struct page *page) +void lru_note_cost_folio (struct folio *folio) { -struct folio *folio =page_folio (page); -lru_note_cost (folio_lruvec (folio), -page_is_file_lru (page),thp_nr_pages (page)); +lru_note_cost (folio_lruvec (folio),folio_is_file_lru (folio), +folio_nr_pages (folio)); }static void __folio_activate (struct folio *folio,struct lruvec *lruvec)
mm/swap_state.c +1 -1
@@ -498,7 +498,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,mem_cgroup_swapin_uncharge_swap (entry);if (shadow) -workingset_refault (page, shadow); +workingset_refault (page_folio (page), shadow);/* Caller will initiate read into locked page */ lru_cache_add (page);
mm/workingset.c +17 -17
@@ -273,17 +273,17 @@ void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg) }/** -* workingset_refault - evaluate the refault of a previously evicted page -* @page: the freshly allocated replacement page -* @shadow: shadow entry of the evicted page +* workingset_refault - evaluate the refault of a previously evicted folio +* @page: the freshly allocated replacement folio +* @shadow: shadow entry of the evicted folio * * Calculates and evaluates the refault distance of the previously -* evicted page in the context of the node and the memcg whose memory +* evicted folio in the context of the node and the memcg whose memory * pressure caused the eviction. */ -void workingset_refault (struct page *page,void *shadow) +void workingset_refault (struct folio *folio,void *shadow) { -bool file =page_is_file_lru (page); +bool file =folio_is_file_lru (folio);struct mem_cgroup *eviction_memcg;struct lruvec *eviction_lruvec;unsigned long refault_distance;
@@ -301,10 +301,10 @@ void workingset_refault(struct page *page, void *shadow)rcu_read_lock ();/* * Look up the memcg associated with the stored ID. It might -* have been deleted since the page's eviction. +* have been deleted since the folio's eviction. * * Note that in rare events the ID could have been recycled -* for a new cgroup that refaults a shared page. This is +* for a new cgroup that refaults a shared folio. This is * impossible to tell from the available data. However, this * should be a rare and limited disturbance, and activations * are always speculative anyway. Ultimately, it's the aging
@@ -340,14 +340,14 @@ void workingset_refault(struct page *page, void *shadow) refault_distance = (refault - eviction) &EVICTION_MASK ;/* -* The activation decision for this page is made at the level +* The activation decision for this folio is made at the level * where the eviction occurred, as that is where the LRU order -* during page reclaim is being determined. +* during folio reclaim is being determined. * -* However, the cgroup that will own the page is the one that +* However, the cgroup that will own the folio is the one that * is actually experiencing the refault event. */ - memcg =page_memcg (page); + memcg =folio_memcg (folio); lruvec =mem_cgroup_lruvec (memcg, pgdat);inc_lruvec_state (lruvec,WORKINGSET_REFAULT_BASE + file);
@@ -375,15 +375,15 @@ void workingset_refault(struct page *page, void *shadow)if (refault_distance > workingset_size) goto out; -SetPageActive (page); -workingset_age_nonresident (lruvec,thp_nr_pages (page)); +folio_set_active (folio); +workingset_age_nonresident (lruvec,folio_nr_pages (folio));inc_lruvec_state (lruvec,WORKINGSET_ACTIVATE_BASE + file); -/* Page was active prior to eviction */ +/* Folio was active prior to eviction */ if (workingset) { -SetPageWorkingset (page); +folio_set_workingset (folio);/* XXX: Move to lru_cache_add() when it supports new vs putback */ -lru_note_cost_page (page); +lru_note_cost_folio (folio);inc_lruvec_state (lruvec,WORKINGSET_RESTORE_BASE + file); } out:
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 31/39 mm: Add folio_evictable()
In reply to Matthew Wilcox (Oracle)This is the folio equivalent of page_evictable(). Unfortunately, it's different from !folio_test_unevictable(), but I think it's used in places where you have to be a VM expert and can reasonably be expected to know the difference.
--- mm/internal.h | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-)
mm/internal.h +19 -8
@@ -72,17 +72,28 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, pgoff_t end, struct pagevec *pvec, pgoff_t *indices);/** -* page_evictable - test whether a page is evictable -* @page: the page to test +* folio_evictable - Test whether a folio is evictable. +* @folio: The folio to test. * -* Test whether page is evictable--i.e., should be placed on active/inactive -* lists vs unevictable list. -* -* Reasons page might not be evictable: -* (1) page's mapping marked unevictable -* (2) page is part of an mlocked VMA +* Test whether @folio is evictable -- i.e., should be placed on +* active/inactive lists vs unevictable list. * +* Reasons folio might not be evictable: +* 1. folio's mapping marked unevictable +* 2. One of the pages in the folio is part of an mlocked VMA */ +static inline bool folio_evictable (struct folio *folio) +{ +bool ret; + +/* Prevent address_space of inode and swap cache from being freed */ +rcu_read_lock (); + ret = !mapping_unevictable (folio_mapping (folio)) && + !folio_test_mlocked (folio); +rcu_read_unlock (); +return ret; +} +static inline bool page_evictable (struct page *page) {bool ret;
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 32/39 mm/lru: Convert __pagevec_lru_add_fn to take a folio
In reply to Matthew Wilcox (Oracle)This saves five calls to compound_head(), totalling 60 bytes of text.
--- include/trace/events/pagemap.h | 32 ++++++++++++++++---------------- mm/swap.c | 34 +++++++++++++++++----------------- 2 files changed, 33 insertions(+), 33 deletions(-)
include/trace/events/pagemap.h +16 -16
@@ -16,38 +16,38 @@#define PAGEMAP_MAPPEDDISK 0x0020u#define PAGEMAP_BUFFERS 0x0040u -#define trace_pagemap_flags (page) ( \ - (PageAnon(page) ? PAGEMAP_ANONYMOUS : PAGEMAP_FILE) | \ - (page_mapped(page) ? PAGEMAP_MAPPED : 0) | \ - (PageSwapCache(page) ? PAGEMAP_SWAPCACHE : 0) | \ - (PageSwapBacked(page) ? PAGEMAP_SWAPBACKED : 0) | \ - (PageMappedToDisk(page) ? PAGEMAP_MAPPEDDISK : 0) | \ - (page_has_private(page) ? PAGEMAP_BUFFERS : 0) \ +#define trace_pagemap_flags (folio) ( \ + (folio_test_anon(folio) ? PAGEMAP_ANONYMOUS : PAGEMAP_FILE) | \ + (folio_mapped(folio) ? PAGEMAP_MAPPED : 0) | \ + (folio_test_swapcache(folio) ? PAGEMAP_SWAPCACHE : 0) | \ + (folio_test_swapbacked(folio) ? PAGEMAP_SWAPBACKED : 0) | \ + (folio_test_mappedtodisk(folio) ? PAGEMAP_MAPPEDDISK : 0) | \ + (folio_test_private(folio) ? PAGEMAP_BUFFERS : 0) \ ) TRACE_EVENT(mm_lru_insertion, - TP_PROTO(struct page *page), + TP_PROTO(struct folio *folio), - TP_ARGS(page), + TP_ARGS(folio), TP_STRUCT__entry( - __field(struct page *, page ) + __field(struct folio *, folio ) __field(unsigned long, pfn ) __field(enum lru_list, lru ) __field(unsigned long, flags ) ), TP_fast_assign( - __entry->page = page; - __entry->pfn = page_to_pfn(page); - __entry->lru = folio_lru_list(page_folio(page)); - __entry->flags = trace_pagemap_flags(page); + __entry->folio = folio; + __entry->pfn = folio_pfn(folio); + __entry->lru = folio_lru_list(folio); + __entry->flags = trace_pagemap_flags(folio); ), /* Flag format is based on page-types.c formatting for pagemap */ - TP_printk("page=%p pfn=0x%lx lru=%d flags=%s%s%s%s%s%s", - __entry->page, + TP_printk("folio=%p pfn=0x%lx lru=%d flags=%s%s%s%s%s%s", + __entry->folio, __entry->pfn, __entry->lru, __entry->flags & PAGEMAP_MAPPED ? "M" : " ",
mm/swap.c +17 -17
@@ -1001,17 +1001,18 @@ void __pagevec_release(struct pagevec *pvec) }EXPORT_SYMBOL (__pagevec_release); -static void __pagevec_lru_add_fn (struct page *page,struct lruvec *lruvec) +static void __pagevec_lru_add_fn (struct folio *folio,struct lruvec *lruvec) { -int was_unevictable =TestClearPageUnevictable (page); -int nr_pages =thp_nr_pages (page); +int was_unevictable =folio_test_clear_unevictable (folio); +int nr_pages =folio_nr_pages (folio); -VM_BUG_ON_PAGE (PageLRU (page), page); +VM_BUG_ON_FOLIO (folio_test_lru (folio), folio);/* -* Page becomes evictable in two ways: +* Folio becomes evictable in two ways: * 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()]. -* 2) Before acquiring LRU lock to put the page to correct LRU and then +* 2) Before acquiring LRU lock to put the folio on the correct LRU +* and then * a) do PageLRU check with lock [check_move_unevictable_pages] * b) do PageLRU check before lock [clear_page_mlock] *
@@ -1020,10 +1021,10 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec)* * #0: __pagevec_lru_add_fn #1: clear_page_mlock * -* SetPageLRU() TestClearPageMlocked() +* folio_set_lru() folio_test_clear_mlocked() * smp_mb() // explicit ordering // above provides strict * // ordering -* PageMlocked() PageLRU() +* folio_test_mlocked() folio_test_lru() * * * if '#1' does not observe setting of PG_lru by '#0' and fails
@@ -1034,21 +1035,21 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec)* looking at the same page) and the evictable page will be stranded * in an unevictable LRU. */ -SetPageLRU (page); +folio_set_lru (folio);smp_mb__after_atomic (); -if (page_evictable (page)) { +if (folio_evictable (folio)) {if (was_unevictable)__count_vm_events (UNEVICTABLE_PGRESCUED , nr_pages); }else { -ClearPageActive (page); -SetPageUnevictable (page); +folio_clear_active (folio); +folio_set_unevictable (folio);if (!was_unevictable)__count_vm_events (UNEVICTABLE_PGCULLED , nr_pages); } -add_page_to_lru_list (page, lruvec); -trace_mm_lru_insertion (page); +lruvec_add_folio (lruvec, folio); +trace_mm_lru_insertion (folio); }/*
@@ -1062,11 +1063,10 @@ void __pagevec_lru_add(struct pagevec *pvec)unsigned long flags =0 ;for (i =0 ; i <pagevec_count (pvec); i++) { -struct page *page = pvec->pages[i]; -struct folio *folio =page_folio (page); +struct folio *folio =page_folio (pvec->pages[i]); lruvec =folio_lruvec_relock_irqsave (folio, lruvec, &flags); -__pagevec_lru_add_fn (page, lruvec); +__pagevec_lru_add_fn (folio, lruvec); }if (lruvec)unlock_page_lruvec_irqrestore (lruvec, flags);
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 33/39 mm/lru: Add folio_add_lru()
In reply to Matthew Wilcox (Oracle)Reimplement lru_cache_add() as a wrapper around folio_add_lru(). Saves 159 bytes of kernel text due to removing calls to compound_head().
--- include/linux/swap.h | 1 + mm/folio-compat.c | 6 ++++++ mm/swap.c | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 11 deletions(-)
include/linux/swap.h +1 -0
@@ -351,6 +351,7 @@ extern unsigned long nr_free_buffer_pages(void);extern void lru_note_cost (struct lruvec *lruvec,bool file,unsigned int nr_pages);extern void lru_note_cost_folio (struct folio *); +extern void folio_add_lru (struct folio *);extern void lru_cache_add (struct page *);void mark_page_accessed (struct page *);void folio_mark_accessed (struct folio *);
mm/folio-compat.c +6 -0
@@ -102,3 +102,9 @@ bool redirty_page_for_writepage(struct writeback_control *wbc,return folio_redirty_for_writepage (wbc,page_folio (page)); }EXPORT_SYMBOL (redirty_page_for_writepage); + +void lru_cache_add (struct page *page) +{ +folio_add_lru (page_folio (page)); +} +EXPORT_SYMBOL (lru_cache_add);
mm/swap.c +11 -11
@@ -459,29 +459,29 @@ void folio_mark_accessed(struct folio *folio)EXPORT_SYMBOL (folio_mark_accessed);/** -* lru_cache_add - add a page to a page list -* @page: the page to be added to the LRU. +* folio_add_lru - Add a folio to an LRU list. +* @folio: The folio to be added to the LRU. * -* Queue the page for addition to the LRU via pagevec. The decision on whether +* Queue the folio for addition to the LRU. The decision on whether * to add the page to the [in]active [file|anon] list is deferred until the -* pagevec is drained. This gives a chance for the caller of lru_cache_add() -* have the page added to the active list using mark_page_accessed(). +* pagevec is drained. This gives a chance for the caller of folio_add_lru() +* have the folio added to the active list using folio_mark_accessed(). */ -void lru_cache_add (struct page *page) +void folio_add_lru (struct folio *folio) {struct pagevec *pvec; -VM_BUG_ON_PAGE (PageActive (page) &&PageUnevictable (page), page); -VM_BUG_ON_PAGE (PageLRU (page), page); +VM_BUG_ON_FOLIO (folio_test_active (folio) &&folio_test_unevictable (folio), folio); +VM_BUG_ON_FOLIO (folio_test_lru (folio), folio); -get_page (page); +folio_get (folio);local_lock (&lru_pvecs.lock); pvec =this_cpu_ptr (&lru_pvecs.lru_add); -if (pagevec_add_and_need_flush (pvec, page)) +if (pagevec_add_and_need_flush (pvec, &folio->page))__pagevec_lru_add (pvec);local_unlock (&lru_pvecs.lock); } -EXPORT_SYMBOL (lru_cache_add); +EXPORT_SYMBOL (folio_add_lru);/** * lru_cache_add_inactive_or_unevictable
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 34/39 mm/page_alloc: Add folio allocation functions
In reply to Matthew Wilcox (Oracle)The __folio_alloc(), __folio_alloc_node() and folio_alloc() functions are mostly for type safety, but they also ensure that the page allocator allocates a compound page and initialises the deferred list if the page is large enough to have one.
--- include/linux/gfp.h | 16 ++++++++++++++++ mm/mempolicy.c | 10 ++++++++++ mm/page_alloc.c | 12 ++++++++++++ 3 files changed, 38 insertions(+)
include/linux/gfp.h +16 -0
@@ -523,6 +523,8 @@ static inline void arch_alloc_page(struct page *page, int order) { }struct page *__alloc_pages (gfp_t gfp,unsigned int order,int preferred_nid,nodemask_t *nodemask); +struct folio *__folio_alloc (gfp_t gfp,unsigned int order,int preferred_nid, +nodemask_t *nodemask); unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid, nodemask_t *nodemask, int nr_pages,
@@ -564,6 +566,15 @@ __alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order)return __alloc_pages (gfp_mask, order, nid,NULL ); } +static inline +struct folio *__folio_alloc_node (gfp_t gfp,unsigned int order,int nid) +{ +VM_BUG_ON (nid <0 || nid >=MAX_NUMNODES ); +VM_WARN_ON ((gfp & __GFP_THISNODE) && !node_online (nid)); + +return __folio_alloc (gfp, order, nid,NULL ); +} +/* * Allocate pages, preferring the node given as nid. When nid == NUMA_NO_NODE, * prefer the current CPU's closest node. Otherwise node must be valid and
@@ -580,6 +591,7 @@ static inline struct page *alloc_pages_node(int nid, gfp_t gfp_mask,#ifdef CONFIG_NUMA struct page *alloc_pages (gfp_t gfp,unsigned int order); +struct folio *folio_alloc (gfp_t gfp,unsigned order);extern struct page *alloc_pages_vma (gfp_t gfp_mask,int order,struct vm_area_struct *vma,unsigned long addr,int node,bool hugepage);
@@ -590,6 +602,10 @@ static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order) {return alloc_pages_node (numa_node_id (), gfp_mask, order); } +static inline struct folio *folio_alloc (gfp_t gfp,unsigned int order) +{ +return __folio_alloc_node (gfp, order,numa_node_id ()); +}#define alloc_pages_vma (gfp_mask, order, vma, addr, node, false)\ alloc_pages(gfp_mask, order)#define alloc_hugepage_vma (gfp_mask, vma, addr, order) \
mm/mempolicy.c +10 -0
@@ -2249,6 +2249,16 @@ struct page *alloc_pages(gfp_t gfp, unsigned order) }EXPORT_SYMBOL (alloc_pages); +struct folio *folio_alloc (gfp_t gfp,unsigned order) +{ +struct page *page =alloc_pages (gfp | __GFP_COMP, order); + +if (page && order >1 ) +prep_transhuge_page (page); +return (struct folio *)page; +} +EXPORT_SYMBOL (folio_alloc); +int vma_dup_policy (struct vm_area_struct *src,struct vm_area_struct *dst) {struct mempolicy *pol =mpol_dup (vma_policy (src));
mm/page_alloc.c +12 -0
@@ -5399,6 +5399,18 @@ struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, }EXPORT_SYMBOL (__alloc_pages); +struct folio *__folio_alloc (gfp_t gfp,unsigned int order,int preferred_nid, +nodemask_t *nodemask) +{ +struct page *page =__alloc_pages (gfp | __GFP_COMP, order, + preferred_nid, nodemask); + +if (page && order >1 ) +prep_transhuge_page (page); +return (struct folio *)page; +} +EXPORT_SYMBOL (__folio_alloc); +/* * Common helper functions. Never use with __GFP_HIGHMEM because the returned * address cannot represent highmem pages. Use alloc_pages and then kmap if
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 35/39 mm/filemap: Add filemap_alloc_folio
In reply to Matthew Wilcox (Oracle)Reimplement __page_cache_alloc as a wrapper around filemap_alloc_folio to allow filesystems to be converted at our leisure. Increases kernel text size by 133 bytes, mostly in cachefiles_read_backing_file(). pagecache_get_page() shrinks by 32 bytes, though.
--- include/linux/pagemap.h | 11 ++++++++--- mm/filemap.c | 14 +++++++------- 2 files changed, 15 insertions(+), 10 deletions(-)
include/linux/pagemap.h +8 -3
@@ -262,14 +262,19 @@ static inline void *detach_page_private(struct page *page) }#ifdef CONFIG_NUMA -extern struct page *__page_cache_alloc (gfp_t gfp); +struct folio *filemap_alloc_folio (gfp_t gfp,unsigned int order);#else -static inline struct page *__page_cache_alloc (gfp_t gfp) +static inline struct folio *filemap_alloc_folio (gfp_t gfp,unsigned int order) { -return alloc_pages (gfp,0 ); +return folio_alloc (gfp, order); }#endif +static inline struct page *__page_cache_alloc (gfp_t gfp) +{ +return &filemap_alloc_folio (gfp,0 )->page; +} +static inline struct page *page_cache_alloc (struct address_space *x) {return __page_cache_alloc (mapping_gfp_mask (x));
mm/filemap.c +7 -7
@@ -989,24 +989,24 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,EXPORT_SYMBOL_GPL (add_to_page_cache_lru);#ifdef CONFIG_NUMA -struct page *__page_cache_alloc (gfp_t gfp) +struct folio *filemap_alloc_folio (gfp_t gfp,unsigned int order) {int n; -struct page *page; +struct folio *folio;if (cpuset_do_page_mem_spread ()) {unsigned int cpuset_mems_cookie;do { cpuset_mems_cookie =read_mems_allowed_begin (); n =cpuset_mem_spread_node (); - page =__alloc_pages_node (n, gfp,0 ); - }while (!page &&read_mems_allowed_retry (cpuset_mems_cookie)); + folio =__folio_alloc_node (gfp, order, n); + }while (!folio &&read_mems_allowed_retry (cpuset_mems_cookie)); -return page; +return folio; } -return alloc_pages (gfp,0 ); +return folio_alloc (gfp, order); } -EXPORT_SYMBOL (__page_cache_alloc); +EXPORT_SYMBOL (filemap_alloc_folio);#endif /*
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 36/39 mm/filemap: Add filemap_add_folio()
In reply to Matthew Wilcox (Oracle)Convert __add_to_page_cache_locked() into __filemap_add_folio(). Add an assertion to it that (for !hugetlbfs), the folio is naturally aligned within the file. Move the prototype from mm.h to pagemap.h. Convert add_to_page_cache_lru() into filemap_add_folio(). Add a compatibility wrapper for unconverted callers.
--- include/linux/mm.h | 7 ----- include/linux/pagemap.h | 10 ++++-- kernel/bpf/verifier.c | 2 +- mm/filemap.c | 70 ++++++++++++++++++++--------------------- mm/folio-compat.c | 7 +++++ 5 files changed, 50 insertions(+), 46 deletions(-)
include/linux/mm.h +0 -7
@@ -213,13 +213,6 @@ int overcommit_kbytes_handler(struct ctl_table *, int, void *, size_t *, loff_t *);int overcommit_policy_handler (struct ctl_table *,int ,void *,size_t *,loff_t *); -/* -* Any attempt to mark this function as static leads to build failure -* when CONFIG_DEBUG_INFO_BTF is enabled because __add_to_page_cache_locked() -* is referred to by BPF code. This must be visible for error injection. -*/ -int __add_to_page_cache_locked (struct page *page,struct address_space *mapping, -pgoff_t index,gfp_t gfp,void **shadowp);#if defined(CONFIG_SPARSEMEM ) && !defined(CONFIG_SPARSEMEM_VMEMMAP )#define nth_page (page,n) pfn_to_page(page_to_pfn((page)) + (n))
include/linux/pagemap.h +8 -2
@@ -877,9 +877,11 @@ static inline int fault_in_pages_readable(const char __user *uaddr, int size) }int add_to_page_cache_locked (struct page *page,struct address_space *mapping, -pgoff_t index,gfp_t gfp_mask); +pgoff_t index,gfp_t gfp);int add_to_page_cache_lru (struct page *page,struct address_space *mapping, -pgoff_t index,gfp_t gfp_mask); +pgoff_t index,gfp_t gfp); +int filemap_add_folio (struct address_space *mapping,struct folio *folio, +pgoff_t index,gfp_t gfp);extern void delete_from_page_cache (struct page *page);extern void __delete_from_page_cache (struct page *page,void *shadow);void replace_page_cache_page (struct page *old,struct page *new);
@@ -904,6 +906,10 @@ static inline int add_to_page_cache(struct page *page,return error; } +/* Must be non-static for BPF error injection */ +int __filemap_add_folio (struct address_space *mapping,struct folio *folio, +pgoff_t index,gfp_t gfp,void **shadowp); +/** * struct readahead_control - Describes a readahead request. *
kernel/bpf/verifier.c +1 -1
@@ -13015,7 +13015,7 @@ BTF_SET_START(btf_non_sleepable_error_inject)/* Three functions below can be called from sleepable and non-sleepable context. * Assume non-sleepable from bpf safety point of view. */ -BTF_ID(func, __add_to_page_cache_locked) +BTF_ID(func, __filemap_add_folio) BTF_ID(func, should_fail_alloc_page) BTF_ID(func, should_failslab)BTF_SET_END (btf_non_sleepable_error_inject)
mm/filemap.c +34 -36
145 lines not on this page · raw message
mm/folio-compat.c +7 -0
@@ -108,3 +108,10 @@ void lru_cache_add(struct page *page)folio_add_lru (page_folio (page)); }EXPORT_SYMBOL (lru_cache_add); + +int add_to_page_cache_lru (struct page *page,struct address_space *mapping, +pgoff_t index,gfp_t gfp) +{ +return filemap_add_folio (mapping,page_folio (page), index, gfp); +} +EXPORT_SYMBOL (add_to_page_cache_lru);
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 37/39 mm/filemap: Convert mapping_get_entry to return a folio
In reply to Matthew Wilcox (Oracle)The pagecache only contains folios, so indicate that this is definitely not a tail page. Shrinks mapping_get_entry() by 56 bytes, but grows pagecache_get_page() by 21 bytes as gcc makes slightly different hot/cold code decisions. A net reduction of 35 bytes of text.
--- mm/filemap.c | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-)
mm/filemap.c +14 -21
@@ -1755,49 +1755,42 @@ EXPORT_SYMBOL(page_cache_prev_miss);* @mapping: the address_space to search * @index: The page cache index. * -* Looks up the page cache slot at @mapping & @index. If there is a -* page cache page, the head page is returned with an increased refcount. +* Looks up the page cache entry at @mapping & @index. If it is a folio, +* it is returned with an increased refcount. If it is a shadow entry +* of a previously evicted folio, or a swap entry from shmem/tmpfs, +* it is returned without further action. * -* If the slot holds a shadow entry of a previously evicted page, or a -* swap entry from shmem/tmpfs, it is returned. -* -* Return: The head page or shadow entry, %NULL if nothing is found. +* Return: The folio, swap or shadow entry, %NULL if nothing is found. */ -static struct page *mapping_get_entry (struct address_space *mapping, -pgoff_t index) +static void *mapping_get_entry (struct address_space *mapping,pgoff_t index) {XA_STATE (xas, &mapping->i_pages, index); -struct page *page; +struct folio *folio;rcu_read_lock (); repeat:xas_reset (&xas); - page =xas_load (&xas); -if (xas_retry (&xas, page)) + folio =xas_load (&xas); +if (xas_retry (&xas, folio)) goto repeat;/* * A shadow entry of a recently evicted page, or a swap entry from * shmem/tmpfs. Return it without attempting to raise page count. */ -if (!page ||xa_is_value (page)) +if (!folio ||xa_is_value (folio)) goto out; -if (!page_cache_get_speculative (page)) +if (!folio_try_get_rcu (folio)) goto repeat; -/* -* Has the page moved or been split? -* This is part of the lockless pagecache protocol. See -* include/linux/pagemap.h for details. -*/ -if (unlikely (page !=xas_reload (&xas))) { -put_page (page); +if (unlikely (folio !=xas_reload (&xas))) { +folio_put (folio); goto repeat; } out:rcu_read_unlock (); -return page; +return folio; }/**
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 38/39 mm/filemap: Add filemap_get_folio
In reply to Matthew Wilcox (Oracle)filemap_get_folio() is a replacement for find_get_page(). Turn pagecache_get_page() into a wrapper around __filemap_get_folio(). Remove find_lock_head() as this use case is now covered by filemap_get_folio().
Reduces overall kernel size by 209 bytes. __filemap_get_folio() is 316 bytes shorter than pagecache_get_page() was, but the new pagecache_get_page() is 99 bytes.
--- include/linux/pagemap.h | 41 +++++++++--------- mm/filemap.c | 92 ++++++++++++++++++++--------------------- mm/folio-compat.c | 12 ++++++ 3 files changed, 76 insertions(+), 69 deletions(-)
include/linux/pagemap.h +20 -21
@@ -302,8 +302,26 @@ pgoff_t page_cache_prev_miss(struct address_space *mapping,#define FGP_HEAD 0x00000080#define FGP_ENTRY 0x00000100 -struct page *pagecache_get_page (struct address_space *mapping,pgoff_t offset, -int fgp_flags,gfp_t cache_gfp_mask); +struct folio *__filemap_get_folio (struct address_space *mapping,pgoff_t index, +int fgp_flags,gfp_t gfp); +struct page *pagecache_get_page (struct address_space *mapping,pgoff_t index, +int fgp_flags,gfp_t gfp); + +/** +* filemap_get_folio - Find and get a folio. +* @mapping: The address_space to search. +* @index: The page index. +* +* Looks up the page cache entry at @mapping & @index. If a folio is +* present, it is returned with an increased refcount. +* +* Otherwise, %NULL is returned. +*/ +static inline struct folio *filemap_get_folio (struct address_space *mapping, +pgoff_t index) +{ +return __filemap_get_folio (mapping, index,0 ,0 ); +}/** * find_get_page - find and get a page reference
@@ -346,25 +364,6 @@ static inline struct page *find_lock_page(struct address_space *mapping,return pagecache_get_page (mapping, index,FGP_LOCK ,0 ); } -/** -* find_lock_head - Locate, pin and lock a pagecache page. -* @mapping: The address_space to search. -* @index: The page index. -* -* Looks up the page cache entry at @mapping & @index. If there is a -* page cache page, its head page is returned locked and with an increased -* refcount. -* -* Context: May sleep. -* Return: A struct page which is !PageTail, or %NULL if there is no page -* in the cache for this index. -*/ -static inline struct page *find_lock_head (struct address_space *mapping, -pgoff_t index) -{ -return pagecache_get_page (mapping, index,FGP_LOCK |FGP_HEAD ,0 ); -} -/** * find_or_create_page - locate or add a pagecache page * @mapping: the page's address_space
mm/filemap.c +44 -48
164 lines not on this page · raw message
mm/folio-compat.c +12 -0
@@ -115,3 +115,15 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,return filemap_add_folio (mapping,page_folio (page), index, gfp); }EXPORT_SYMBOL (add_to_page_cache_lru); + +struct page *pagecache_get_page (struct address_space *mapping,pgoff_t index, +int fgp_flags,gfp_t gfp) +{ +struct folio *folio; + + folio =__filemap_get_folio (mapping, index, fgp_flags, gfp); +if ((fgp_flags &FGP_HEAD ) || !folio ||xa_is_value (folio)) +return &folio->page; +return folio_file_page (folio, index); +} +EXPORT_SYMBOL (pagecache_get_page);
Signature · 1 line
2.30.2
Matthew Wilcox (Oracle)
maintainerPATCH v14 39/39 mm/filemap: Add FGP_STABLE
In reply to Matthew Wilcox (Oracle)Allow filemap_get_folio() to wait for writeback to complete (if the filesystem wants that behaviour). This is the folio equivalent of grab_cache_page_write_begin(), which is moved into the folio-compat file as a reminder to migrate all the code using it. This paves the way for getting rid of AOP_FLAG_NOFS once grab_cache_page_write_begin() is removed.
Kernel grows by 11 bytes. filemap_get_folio() grows by 33 bytes but grab_cache_page_write_begin() shrinks by 22 bytes to make up for it.
--- include/linux/pagemap.h | 1 + mm/filemap.c | 25 +++---------------------- mm/folio-compat.c | 13 +++++++++++++ 3 files changed, 17 insertions(+), 22 deletions(-)
include/linux/pagemap.h +1 -0
@@ -301,6 +301,7 @@ pgoff_t page_cache_prev_miss(struct address_space *mapping,#define FGP_FOR_MMAP 0x00000040#define FGP_HEAD 0x00000080#define FGP_ENTRY 0x00000100 +#define FGP_STABLE 0x00000200struct folio *__filemap_get_folio (struct address_space *mapping,pgoff_t index,int fgp_flags,gfp_t gfp);
mm/filemap.c +3 -22
@@ -1817,6 +1817,7 @@ static void *mapping_get_entry(struct address_space *mapping, pgoff_t index)* * %FGP_WRITE - The page will be written to by the caller. * * %FGP_NOFS - __GFP_FS will get cleared in gfp. * * %FGP_NOWAIT - Don't get blocked by page lock. +* * %FGP_STABLE - Wait for the folio to be stable (finished writeback) * * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even * if the %GFP flags specified for %FGP_CREAT are atomic.
@@ -1867,6 +1868,8 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,folio_clear_idle (folio); } +if (fgp_flags &FGP_STABLE ) +folio_wait_stable (folio); no_page:if (!folio && (fgp_flags &FGP_CREAT )) {int err;
@@ -3590,28 +3593,6 @@ generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) }EXPORT_SYMBOL (generic_file_direct_write); -/* -* Find or create a page at the given pagecache position. Return the locked -* page. This function is specifically for buffered writes. -*/ -struct page *grab_cache_page_write_begin (struct address_space *mapping, -pgoff_t index,unsigned flags) -{ -struct page *page; -int fgp_flags =FGP_LOCK |FGP_WRITE |FGP_CREAT ; - -if (flags &AOP_FLAG_NOFS ) - fgp_flags |=FGP_NOFS ; - - page =pagecache_get_page (mapping, index, fgp_flags, -mapping_gfp_mask (mapping)); -if (page) -wait_for_stable_page (page); - -return page; -} -EXPORT_SYMBOL (grab_cache_page_write_begin); -ssize_t generic_perform_write (struct file *file,struct iov_iter *i,loff_t pos) {
mm/folio-compat.c +13 -0
@@ -116,6 +116,7 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping, }EXPORT_SYMBOL (add_to_page_cache_lru); +noinline struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,int fgp_flags,gfp_t gfp) {
@@ -127,3 +128,15 @@ struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,return folio_file_page (folio, index); }EXPORT_SYMBOL (pagecache_get_page); + +struct page *grab_cache_page_write_begin (struct address_space *mapping, +pgoff_t index,unsigned flags) +{ +unsigned fgp_flags =FGP_LOCK |FGP_WRITE |FGP_CREAT |FGP_STABLE ; + +if (flags &AOP_FLAG_NOFS ) + fgp_flags |=FGP_NOFS ; +return pagecache_get_page (mapping, index, fgp_flags, +mapping_gfp_mask (mapping)); +} +EXPORT_SYMBOL (grab_cache_page_write_begin);
Signature · 1 line
2.30.2