Clean up and fixes for swap
Subsystems (3)
v2 cover Clean up and fixes for swap
From: Kairui Song <[email protected]>
This series cleanup some code path, saves a few cycles and reduce the object size by a bit, also fixes some rare race issue of statistics.
Update from V1:
- Collect Huang, Ying's Reviewd-by.
- Use bloat-o-meter to show size change info in commit message as suggested by Matthew Wilcox.
- Rework patch 4/4 as suggested by Huang, Ying.
- Drop a previouly invalid clean up patch.
Kairui Song (4):
swapfile: get rid of volatile and avoid redundant read
swap: avoid a redundant pte map if ra window is 1
swap: fold swap_ra_clamp_pfn into swap_ra_info
swap: avoid holding swap reference in swap_cache_get_folio
mm/shmem.c | 11 +++++++++ mm/swap_state.c | 59 +++++++++++++++++++------------------------------ mm/swapfile.c | 7 +++--- 3 files changed, 38 insertions(+), 39 deletions(-)
Signature · 1 line
2.35.2
PATCH v2 1/4 swapfile: get rid of volatile and avoid redundant read
From: Kairui Song <[email protected]>
Convert a volatile variable to more readable READ_ONCE. And this actually avoids the code from reading the variable twice redundantly when it races.
--- mm/swapfile.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
mm/swapfile.c +4 -3
@@ -1835,13 +1835,13 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,pte_t *pte;struct swap_info_struct *si;int ret =0 ; -volatile unsigned char *swap_map; si = swap_info[type]; pte =pte_offset_map (pmd, addr); do { struct folio *folio; unsigned long offset; + unsigned char swp_count; if (!is_swap_pte(*pte)) continue;
@@ -1852,7 +1852,6 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, offset =swp_offset (entry);pte_unmap (pte); - swap_map = &si->swap_map[offset]; folio =swap_cache_get_folio (entry, vma, addr);if (!folio) {struct page *page;
@@ -1869,8 +1868,10 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio =page_folio (page); }if (!folio) { -if (*swap_map ==0 || *swap_map ==SWAP_MAP_BAD ) + swp_count =READ_ONCE (si->swap_map[offset]); +if (swp_count ==0 || swp_count ==SWAP_MAP_BAD ) goto try_next; +return -ENOMEM ; }
Signature · 1 line
2.35.2
PATCH v2 2/4 swap: avoid a redundant pte map if ra window is 1
In reply to Kairui SongFrom: Kairui Song <[email protected]>
Avoid a redundant pte map/unmap when swap readahead window is 1.
--- mm/swap_state.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-)
mm/swap_state.c +2 -5
@@ -727,8 +727,6 @@ static void swap_ra_info(struct vm_fault *vmf, } faddr = vmf->address; - orig_pte = pte =pte_offset_map (vmf->pmd, faddr); - fpfn =PFN_DOWN (faddr); ra_val =GET_SWAP_RA_VAL (vma); pfn =PFN_DOWN (SWAP_RA_ADDR (ra_val));
@@ -739,12 +737,11 @@ static void swap_ra_info(struct vm_fault *vmf,atomic_long_set (&vma->swap_readahead_info,SWAP_RA_VAL (faddr, win,0 )); -if (win ==1 ) { -pte_unmap (orig_pte); +if (win ==1 )return ; - }/* Copy the PTEs because the page table may be unmapped */ + orig_pte = pte =pte_offset_map (vmf->pmd, faddr);if (fpfn == pfn +1 )swap_ra_clamp_pfn (vma, faddr, fpfn, fpfn + win, &start, &end);else if (pfn == fpfn +1 )
Signature · 1 line
2.35.2
PATCH v2 3/4 swap: fold swap_ra_clamp_pfn into swap_ra_info
In reply to Kairui SongFrom: Kairui Song <[email protected]>
This make the code cleaner. This helper is made of only two line of self explanational code and not reused anywhere else.
And this actually make the compiled object smaller by a bit.
bloat-o-meter results on x86_64 of mm/swap_state.o:
add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-35 (-35)
Function old new delta swap_ra_info.constprop 512 477 -35
Total: Before=8388, After=8353, chg -0.42%
--- mm/swap_state.c | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-)
mm/swap_state.c +19 -25
@@ -693,28 +693,15 @@ void exit_swap_address_space(unsigned int type) swapper_spaces[type] =NULL ; } -static inline void swap_ra_clamp_pfn (struct vm_area_struct *vma, -unsigned long faddr, -unsigned long lpfn, -unsigned long rpfn, -unsigned long *start, -unsigned long *end) -{ - *start =max3 (lpfn,PFN_DOWN (vma->vm_start), -PFN_DOWN (faddr &PMD_MASK )); - *end =min3 (rpfn,PFN_DOWN (vma->vm_end), -PFN_DOWN ((faddr &PMD_MASK ) +PMD_SIZE )); -} -static void swap_ra_info (struct vm_fault *vmf, -struct vma_swap_readahead *ra_info) +struct vma_swap_readahead *ra_info) {struct vm_area_struct *vma = vmf->vma;unsigned long ra_val; -unsigned long faddr, pfn, fpfn; +unsigned long faddr, pfn, fpfn, lpfn, rpfn;unsigned long start, end;pte_t *pte, *orig_pte; -unsigned int max_win, hits, prev_win, win, left; +unsigned int max_win, hits, prev_win, win;#ifndef CONFIG_64BIT pte_t *tpte;#endif
@@ -742,16 +729,23 @@ static void swap_ra_info(struct vm_fault *vmf,/* Copy the PTEs because the page table may be unmapped */ orig_pte = pte =pte_offset_map (vmf->pmd, faddr); -if (fpfn == pfn +1 ) -swap_ra_clamp_pfn (vma, faddr, fpfn, fpfn + win, &start, &end); -else if (pfn == fpfn +1 ) -swap_ra_clamp_pfn (vma, faddr, fpfn - win +1 , fpfn +1 , - &start, &end); -else { - left = (win -1 ) /2 ; -swap_ra_clamp_pfn (vma, faddr, fpfn - left, fpfn + win - left, - &start, &end); +if (fpfn == pfn +1 ) { + lpfn = fpfn; + rpfn = fpfn + win; + }else if (pfn == fpfn +1 ) { + lpfn = fpfn - win +1 ; + rpfn = fpfn +1 ; + }else { +unsigned int left = (win -1 ) /2 ; + + lpfn = fpfn - left; + rpfn = fpfn + win - left; } + start =max3 (lpfn,PFN_DOWN (vma->vm_start), +PFN_DOWN (faddr &PMD_MASK )); + end =min3 (rpfn,PFN_DOWN (vma->vm_end), +PFN_DOWN ((faddr &PMD_MASK ) +PMD_SIZE )); + ra_info->nr_pte = end - start; ra_info->offset = fpfn - start; pte -= ra_info->offset;
Signature · 1 line
2.35.2
PATCH v2 4/4 swap: avoid holding swap reference in swap_cache_get_folio
In reply to Kairui SongFrom: Kairui Song <[email protected]>
All its callers either already hold a reference to, or lock the swap device while calling this function. There is only one exception in shmem_swapin_folio, just make this caller also hold a reference of the swap device, so this helper can be simplified and saves a few cycles.
This also provides finer control of error handling in shmem_swapin_folio, on race (with swap off), it can just try again. For invalid swap entry, it can fail with a proper error code.
--- mm/shmem.c | 11 +++++++++++ mm/swap_state.c | 8 ++------ 2 files changed, 13 insertions(+), 6 deletions(-)
mm/shmem.c +11 -0
@@ -1735,6 +1735,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,struct address_space *mapping = inode->i_mapping;struct shmem_inode_info *info =SHMEM_I (inode);struct mm_struct *charge_mm = vma ? vma->vm_mm :NULL ; +struct swap_info_struct *si;struct folio *folio =NULL ;swp_entry_t swap;int error;
@@ -1746,6 +1747,14 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,if (is_swapin_error_entry (swap))return -EIO ; + si =get_swap_device (swap); +if (!si) { +if (!shmem_confirm_swap (mapping, index, swap)) +return -EEXIST ; +else +return -EINVAL ; + } +/* Look it up and read it in.. */ folio =swap_cache_get_folio (swap,NULL ,0 );if (!folio) {
@@ -1806,6 +1815,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,delete_from_swap_cache (folio);folio_mark_dirty (folio);swap_free (swap); +put_swap_device (si); *foliop = folio;return 0 ;
@@ -1819,6 +1829,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,folio_unlock (folio);folio_put (folio); } +put_swap_device (si);return error; }
mm/swap_state.c +2 -6
@@ -321,19 +321,15 @@ static inline bool swap_use_vma_readahead(void)* unlocked and with its refcount incremented - we rely on the kernel * lock getting page table operations atomic even if we drop the folio * lock before returning. +* +* Caller must lock the swap device or hold a reference to keep it valid. */ struct folio *swap_cache_get_folio (swp_entry_t entry,struct vm_area_struct *vma,unsigned long addr) {struct folio *folio; -struct swap_info_struct *si; - si =get_swap_device (entry); -if (!si) -return NULL ; folio =filemap_get_folio (swap_address_space (entry),swp_offset (entry)); -put_swap_device (si); -if (folio) {bool vma_ra =swap_use_vma_readahead ();bool readahead;
Signature · 1 line
2.35.2