Skip to contentSkip to conversation

Improve the tmpfs large folio read performance

PATCH Superseded Baolin Wang 4 participants 13 messages Oct 16, 2024 – Oct 18, 2024

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Baolin Wang

author reviewer
#0 lore raw

cover Improve the tmpfs large folio read performance

The tmpfs has already supported the PMD-sized large folios, but the tmpfs read operation still performs copying at the PAGE SIZE granularity, which is not perfect. This patch changes to copy data at the folio granularity, which can improve the read performance.

Use 'fio bs=64k' to read a 1G tmpfs file populated with 2M THPs, and I can see about 20% performance improvement, and no regression with bs=4k. I also did some functional test with the xfstests suite, and I did not find any regressions with the following xfstests config.
FSTYP=tmpfs
export TEST_DIR=/mnt/tempfs_mnt
export TEST_DEV=/mnt/tempfs_mnt
export SCRATCH_MNT=/mnt/scratchdir
export SCRATCH_DEV=/mnt/scratchdir

Baolin Wang (2):
mm: shmem: update iocb->ki_pos directly to simplify tmpfs read logic
mm: shmem: improve the tmpfs large folio read performance

 mm/shmem.c | 54 ++++++++++++++++++++++--------------------------------
 1 file changed, 22 insertions(+), 32 deletions(-)
Signature · 1 line
2.39.3
author reviewer
#1 lore raw

PATCH 1/2 mm: shmem: update iocb->ki_pos directly to simplify tmpfs read logic

Use iocb->ki_pos to check if the read bytes exceeds the file size and to calculate the bytes to be read can help simplify the code logic. Meanwhile, this is also a preparation for improving tmpfs large folios read performace in the following patch.

---
 mm/shmem.c | 36 ++++++++++++------------------------
 1 file changed, 12 insertions(+), 24 deletions(-)
mm/shmem.c +12 -24
index 66eae800ffab..edab02a26aac 100644
@@ -3106,26 +3106,18 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	unsigned long offset;
 	int error = 0;
 	ssize_t retval = 0;
-	loff_t *ppos = &iocb->ki_pos;
 
-	index = *ppos >> PAGE_SHIFT;
-	offset = *ppos & ~PAGE_MASK;
+	index = iocb->ki_pos >> PAGE_SHIFT;
+	offset = iocb->ki_pos & ~PAGE_MASK;
 
 	for (;;) {
 		struct folio *folio = NULL;
 		struct page *page = NULL;
-		pgoff_t end_index;
 		unsigned long nr, ret;
-		loff_t i_size = i_size_read(inode);
+		loff_t end_offset, i_size = i_size_read(inode);
 
-		end_index = i_size >> PAGE_SHIFT;
-		if (index > end_index)
+		if (unlikely(iocb->ki_pos >= i_size))
 			break;
-		if (index == end_index) {
-			nr = i_size & ~PAGE_MASK;
-			if (nr <= offset)
-				break;
-		}
 
 		error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
 		if (error) {
@@ -3148,18 +3140,14 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		 * We must evaluate after, since reads (unlike writes)
 		 * are called without i_rwsem protection against truncate
 		 */
-		nr = PAGE_SIZE;
 		i_size = i_size_read(inode);
-		end_index = i_size >> PAGE_SHIFT;
-		if (index == end_index) {
-			nr = i_size & ~PAGE_MASK;
-			if (nr <= offset) {
-				if (folio)
-					folio_put(folio);
-				break;
-			}
+		if (unlikely(iocb->ki_pos >= i_size)) {
+			if (folio)
+				folio_put(folio);
+			break;
 		}
-		nr -= offset;
+		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
+		nr = min_t(loff_t, end_offset - iocb->ki_pos, PAGE_SIZE - offset);
 
 		if (folio) {
 			/*
@@ -3199,8 +3187,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
 		retval += ret;
 		offset += ret;
-		index += offset >> PAGE_SHIFT;
 		offset &= ~PAGE_MASK;
+		iocb->ki_pos += ret;
+		index = iocb->ki_pos >> PAGE_SHIFT;
 
 		if (!iov_iter_count(to))
 			break;
@@ -3211,7 +3200,6 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		cond_resched();
 	}
 
-	*ppos = ((loff_t) index << PAGE_SHIFT) + offset;
 	file_accessed(file);
 	return retval ? retval : error;
 }
Signature · 1 line
2.39.3
author reviewer
#4 lore raw

PATCH 2/2 mm: shmem: improve the tmpfs large folio read performance

In reply to Baolin Wang

The tmpfs has already supported the PMD-sized large folios, but the tmpfs read operation still performs copying at the PAGE SIZE granularity, which is unreasonable. This patch changes to copy data at the folio granularity, which can improve the read performance, as well as changing to use folio related functions.

Use 'fio bs=64k' to read a 1G tmpfs file populated with 2M THPs, and I can see about 20% performance improvement, and no regression with bs=4k. Before the patch:
READ: bw=10.0GiB/s

After the patch:
READ: bw=12.0GiB/s

---
 mm/shmem.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)
mm/shmem.c +12 -10
index edab02a26aac..7e79b6a96da0 100644
@@ -3108,13 +3108,12 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	ssize_t retval = 0;
 
 	index = iocb->ki_pos >> PAGE_SHIFT;
-	offset = iocb->ki_pos & ~PAGE_MASK;
 
 	for (;;) {
 		struct folio *folio = NULL;
-		struct page *page = NULL;
 		unsigned long nr, ret;
 		loff_t end_offset, i_size = i_size_read(inode);
+		size_t fsize;
 
 		if (unlikely(iocb->ki_pos >= i_size))
 			break;
@@ -3128,8 +3127,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		if (folio) {
 			folio_unlock(folio);
 
-			page = folio_file_page(folio, index);
-			if (PageHWPoison(page)) {
+			if (folio_test_hwpoison(folio) ||
+			    (folio_test_large(folio) &&
+			     folio_test_has_hwpoisoned(folio))) {
 				folio_put(folio);
 				error = -EIO;
 				break;
@@ -3147,7 +3147,12 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 			break;
 		}
 		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
-		nr = min_t(loff_t, end_offset - iocb->ki_pos, PAGE_SIZE - offset);
+		if (folio)
+			fsize = folio_size(folio);
+		else
+			fsize = PAGE_SIZE;
+		offset = iocb->ki_pos & (fsize - 1);
+		nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
 
 		if (folio) {
 			/*
@@ -3156,7 +3161,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 			 * before reading the page on the kernel side.
 			 */
 			if (mapping_writably_mapped(mapping))
-				flush_dcache_page(page);
+				flush_dcache_folio(folio);
 			/*
 			 * Mark the page accessed if we read the beginning.
 			 */
@@ -3166,9 +3171,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 			 * Ok, we have the page, and it's up-to-date, so
 			 * now we can copy it to user space...
 			 */
-			ret = copy_page_to_iter(page, offset, nr, to);
+			ret = copy_folio_to_iter(folio, offset, nr, to);
 			folio_put(folio);
-
 		} else if (user_backed_iter(to)) {
 			/*
 			 * Copy to user tends to be so well optimized, but
@@ -3186,8 +3190,6 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		}
 
 		retval += ret;
-		offset += ret;
-		offset &= ~PAGE_MASK;
 		iocb->ki_pos += ret;
 		index = iocb->ki_pos >> PAGE_SHIFT;
 
Signature · 1 line
2.39.3
#12 lore raw
In reply to Baolin Wang
Quoted 5 lines · Baolin Wang

The tmpfs has already supported the PMD-sized large folios, but the tmpfs read operation still performs copying at the PAGE SIZE granularity, which is not perfect. This patch changes to copy data at the folio granularity, which can improve the read performance.

Quoted 10 lines

Use 'fio bs=64k' to read a 1G tmpfs file populated with 2M THPs, and I can see about 20% performance improvement, and no regression with bs=4k. I also did some functional test with the xfstests suite, and I did not find any regressions with the following xfstests config.
FSTYP=tmpfs
export TEST_DIR=/mnt/tempfs_mnt
export TEST_DEV=/mnt/tempfs_mnt
export SCRATCH_MNT=/mnt/scratchdir
export SCRATCH_DEV=/mnt/scratchdir

Oh,we make same changes, my bonnie test(./bonnie -d /tmp -s 1024) see similar improvement(19.2% with huge=always) with out inner changes :)

Quoted 7 lines

Baolin Wang (2):
mm: shmem: update iocb->ki_pos directly to simplify tmpfs read logic
mm: shmem: improve the tmpfs large folio read performance

  mm/shmem.c | 54 ++++++++++++++++++++++--------------------------------
  1 file changed, 22 insertions(+), 32 deletions(-)
#2 lore raw

Re: [PATCH 1/2] mm: shmem: update iocb->ki_pos directly to simplify tmpfs read logic

In reply to Baolin Wang
Quoted 15 lines · Baolin Wang

Use iocb->ki_pos to check if the read bytes exceeds the file size and to calculate the bytes to be read can help simplify the code logic. Meanwhile, this is also a preparation for improving tmpfs large folios read performace in the following patch.

---
  mm/shmem.c | 36 ++++++++++++------------------------
  1 file changed, 12 insertions(+), 24 deletions(-)
mm/shmem.c +0 -0
index 66eae800ffab..edab02a26aac 100644

mm/shmem.c static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · line 3110

Quoted 8 lines
mm/shmem.c +1 -3
@@ -3106,26 +3106,18 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  	unsigned long offset;
  	int error = 0;
  	ssize_t retval = 0;
-	loff_t *ppos = &iocb->ki_pos;
 
-	index = *ppos >> PAGE_SHIFT;
-	offset = *ppos & ~PAGE_MASK;
+	index = iocb->ki_pos >> PAGE_SHIFT;

index calculate could be moved before shmem_get_folio(), then...

Quoted 48 lines
mm/shmem.c +9 -19
+	offset = iocb->ki_pos & ~PAGE_MASK;
 
  	for (;;) {
  		struct folio *folio = NULL;
  		struct page *page = NULL;
-		pgoff_t end_index;
  		unsigned long nr, ret;
-		loff_t i_size = i_size_read(inode);
+		loff_t end_offset, i_size = i_size_read(inode);
 
-		end_index = i_size >> PAGE_SHIFT;
-		if (index > end_index)
+		if (unlikely(iocb->ki_pos >= i_size))
  			break;
-		if (index == end_index) {
-			nr = i_size & ~PAGE_MASK;
-			if (nr <= offset)
-				break;
-		}
 
  		error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
  		if (error) {
@@ -3148,18 +3140,14 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  		 * We must evaluate after, since reads (unlike writes)
  		 * are called without i_rwsem protection against truncate
  		 */
-		nr = PAGE_SIZE;
  		i_size = i_size_read(inode);
-		end_index = i_size >> PAGE_SHIFT;
-		if (index == end_index) {
-			nr = i_size & ~PAGE_MASK;
-			if (nr <= offset) {
-				if (folio)
-					folio_put(folio);
-				break;
-			}
+		if (unlikely(iocb->ki_pos >= i_size)) {
+			if (folio)
+				folio_put(folio);
+			break;
  		}
-		nr -= offset;
+		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
+		nr = min_t(loff_t, end_offset - iocb->ki_pos, PAGE_SIZE - offset);
 
  		if (folio) {
  			/*

mm/shmem.c static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · line 3192

Quoted 7 lines
mm/shmem.c +2 -1
@@ -3199,8 +3187,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
  		retval += ret;
  		offset += ret;
-		index += offset >> PAGE_SHIFT;
  		offset &= ~PAGE_MASK;
+		iocb->ki_pos += ret;
+		index = iocb->ki_pos >> PAGE_SHIFT;

remove this line.

Quoted 11 lines
mm/shmem.c +0 -1
  		if (!iov_iter_count(to))
  			break;
@@ -3211,7 +3200,6 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  		cond_resched();
  	}
 
-	*ppos = ((loff_t) index << PAGE_SHIFT) + offset;
  	file_accessed(file);
  	return retval ? retval : error;
  }
#5 lore raw

Re: [PATCH 2/2] mm: shmem: improve the tmpfs large folio read performance

In reply to Baolin Wang
Quoted 65 lines · Baolin Wang

The tmpfs has already supported the PMD-sized large folios, but the tmpfs read operation still performs copying at the PAGE SIZE granularity, which is unreasonable. This patch changes to copy data at the folio granularity, which can improve the read performance, as well as changing to use folio related functions.

Use 'fio bs=64k' to read a 1G tmpfs file populated with 2M THPs, and I can see about 20% performance improvement, and no regression with bs=4k. Before the patch:
READ: bw=10.0GiB/s

After the patch:
READ: bw=12.0GiB/s

---
  mm/shmem.c | 22 ++++++++++++----------
  1 file changed, 12 insertions(+), 10 deletions(-)
mm/shmem.c +10 -5
index edab02a26aac..7e79b6a96da0 100644
@@ -3108,13 +3108,12 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  	ssize_t retval = 0;
 
  	index = iocb->ki_pos >> PAGE_SHIFT;
-	offset = iocb->ki_pos & ~PAGE_MASK;
 
  	for (;;) {
  		struct folio *folio = NULL;
-		struct page *page = NULL;
  		unsigned long nr, ret;
  		loff_t end_offset, i_size = i_size_read(inode);
+		size_t fsize;
 
  		if (unlikely(iocb->ki_pos >= i_size))
  			break;
@@ -3128,8 +3127,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  		if (folio) {
  			folio_unlock(folio);
 
-			page = folio_file_page(folio, index);
-			if (PageHWPoison(page)) {
+			if (folio_test_hwpoison(folio) ||
+			    (folio_test_large(folio) &&
+			     folio_test_has_hwpoisoned(folio))) {
  				folio_put(folio);
  				error = -EIO;
  				break;
@@ -3147,7 +3147,12 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  			break;
  		}
  		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
-		nr = min_t(loff_t, end_offset - iocb->ki_pos, PAGE_SIZE - offset);
+		if (folio)
+			fsize = folio_size(folio);
+		else
+			fsize = PAGE_SIZE;
+		offset = iocb->ki_pos & (fsize - 1);
+		nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
 
  		if (folio) {
  			/*

mm/shmem.c static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · line 3162

Quoted 2 lines
mm/shmem.c +0 -0
@@ -3156,7 +3161,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  			 * before reading the page on the kernel side.
  			 */

We'd better to update all the comment from page to folio.

Quoted 26 lines
mm/shmem.c +2 -5
  			if (mapping_writably_mapped(mapping))
-				flush_dcache_page(page);
+				flush_dcache_folio(folio);
  			/*
  			 * Mark the page accessed if we read the beginning.
  			 */
@@ -3166,9 +3171,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  			 * Ok, we have the page, and it's up-to-date, so
  			 * now we can copy it to user space...
  			 */
-			ret = copy_page_to_iter(page, offset, nr, to);
+			ret = copy_folio_to_iter(folio, offset, nr, to);
  			folio_put(folio);
-
  		} else if (user_backed_iter(to)) {
  			/*
  			 * Copy to user tends to be so well optimized, but
@@ -3186,8 +3190,6 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  		}
 
  		retval += ret;
-		offset += ret;
-		offset &= ~PAGE_MASK;
  		iocb->ki_pos += ret;
  		index = iocb->ki_pos >> PAGE_SHIFT;
 
#7 lore raw

Re: [PATCH 2/2] mm: shmem: improve the tmpfs large folio read performance

In reply to Baolin Wang

mm/shmem.c static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · line 3132

Quoted 9 lines · Baolin Wang
mm/shmem.c +3 -2
@@ -3128,8 +3127,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		if (folio) {
 			folio_unlock(folio);
 
-			page = folio_file_page(folio, index);
-			if (PageHWPoison(page)) {
+			if (folio_test_hwpoison(folio) ||
+			    (folio_test_large(folio) &&
+			     folio_test_has_hwpoisoned(folio))) {

Hm, so if we have hwpoison set on one page in a folio, we now can't read bytes from any page in the folio? That seems like we've made a bad situation worse.

#8 lore raw

Re: [PATCH 2/2] mm: shmem: improve the tmpfs large folio read performance

Quoted 6 lines · Matthew Wilcox
Quoted 4 lines · Baolin Wang
mm/shmem.c +0 -0
@@ -3128,8 +3127,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
              if (folio) {
                      folio_unlock(folio);
 
Quoted 9 lines

mm/shmem.c static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · line 3132

Quoted 5 lines
mm/shmem.c +3 -2
-                     page = folio_file_page(folio, index);
-                     if (PageHWPoison(page)) {
+                     if (folio_test_hwpoison(folio) ||
+                         (folio_test_large(folio) &&
+                          folio_test_has_hwpoisoned(folio))) {

Hm, so if we have hwpoison set on one page in a folio, we now can't read bytes from any page in the folio? That seems like we've made a bad situation worse.

Yeah, I agree. I think we can fallback to page copy if folio_test_has_hwpoisoned is true. The PG_hwpoison flag is per page.

The folio_test_has_hwpoisoned is kept set if the folio split is failed in memory failure handler.

Quoted 2 lines
author reviewer
#3 lore raw

Re: [PATCH 1/2] mm: shmem: update iocb->ki_pos directly to simplify tmpfs read logic

In reply to Kefeng Wang
Quoted 13 lines · Kefeng Wang
Quoted 10 lines · Baolin Wang

Use iocb->ki_pos to check if the read bytes exceeds the file size and to calculate the bytes to be read can help simplify the code logic. Meanwhile, this is also a preparation for improving tmpfs large folios read performace in the following patch.

---
  mm/shmem.c | 36 ++++++++++++------------------------

  1 file changed, 12 insertions(+), 24 deletions(-)
Quoted 15 lines

mm/shmem.c static ssize_t shmem_file_read_iter(struct · line 3110

Quoted 13 lines
mm/shmem.c +1 -3
index 66eae800ffab..edab02a26aac 100644
@@ -3106,26 +3106,18 @@ static ssize_t shmem_file_read_iter(struct 
 kiocb *iocb, struct iov_iter *to)
       unsigned long offset;
       int error = 0;
       ssize_t retval = 0;
-    loff_t *ppos = &iocb->ki_pos;
-    index = *ppos >> PAGE_SHIFT;
-    offset = *ppos & ~PAGE_MASK;
+    index = iocb->ki_pos >> PAGE_SHIFT;

index calculate could be moved before shmem_get_folio(), then...

Yes. Will do.

author reviewer
#6 lore raw

Re: [PATCH 2/2] mm: shmem: improve the tmpfs large folio read performance

In reply to Kefeng Wang
Quoted 64 lines · Kefeng Wang
Quoted 61 lines · Baolin Wang

The tmpfs has already supported the PMD-sized large folios, but the tmpfs read operation still performs copying at the PAGE SIZE granularity, which is unreasonable. This patch changes to copy data at the folio granularity, which can improve the read performance, as well as changing to use folio related functions.

Use 'fio bs=64k' to read a 1G tmpfs file populated with 2M THPs, and I can see about 20% performance improvement, and no regression with bs=4k. Before the patch:
READ: bw=10.0GiB/s

After the patch:
READ: bw=12.0GiB/s

---
  mm/shmem.c | 22 ++++++++++++----------

  1 file changed, 12 insertions(+), 10 deletions(-)
mm/shmem.c +10 -5
index edab02a26aac..7e79b6a96da0 100644
@@ -3108,13 +3108,12 @@ static ssize_t shmem_file_read_iter(struct 
 kiocb *iocb, struct iov_iter *to)
       ssize_t retval = 0;
       index = iocb->ki_pos >> PAGE_SHIFT;
-    offset = iocb->ki_pos & ~PAGE_MASK;
       for (;;) {
           struct folio *folio = NULL;
-        struct page *page = NULL;
           unsigned long nr, ret;
           loff_t end_offset, i_size = i_size_read(inode);
+        size_t fsize;
           if (unlikely(iocb->ki_pos >= i_size))
               break;
@@ -3128,8 +3127,9 @@ static ssize_t shmem_file_read_iter(struct kiocb 
 *iocb, struct iov_iter *to)
           if (folio) {
               folio_unlock(folio);
-            page = folio_file_page(folio, index);
-            if (PageHWPoison(page)) {
+            if (folio_test_hwpoison(folio) ||
+                (folio_test_large(folio) &&
+                 folio_test_has_hwpoisoned(folio))) {
                   folio_put(folio);
                   error = -EIO;
                   break;
@@ -3147,7 +3147,12 @@ static ssize_t shmem_file_read_iter(struct 
 kiocb *iocb, struct iov_iter *to)
               break;
           }
           end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
-        nr = min_t(loff_t, end_offset - iocb->ki_pos, PAGE_SIZE - 
 offset);
+        if (folio)
+            fsize = folio_size(folio);
+        else
+            fsize = PAGE_SIZE;
+        offset = iocb->ki_pos & (fsize - 1);
+        nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
Quoted 8 lines

mm/shmem.c static ssize_t shmem_file_read_iter(struct kiocb · line 3163

Quoted 6 lines
mm/shmem.c +0 -0
           if (folio) {

              /*

mm/shmem.c +0 -0
@@ -3156,7 +3161,7 @@ static ssize_t shmem_file_read_iter(struct kiocb 
 *iocb, struct iov_iter *to)
                * before reading the page on the kernel side.
                */

We'd better to update all the comment from page to folio.

Ack.

author reviewer
#9 lore raw

Re: [PATCH 2/2] mm: shmem: improve the tmpfs large folio read performance

In reply to Yang Shi
Quoted 17 lines · Yang Shi
Quoted 6 lines · Matthew Wilcox
Quoted 4 lines · Baolin Wang
(quoted diff) +0 -0
@@ -3128,8 +3127,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
               if (folio) {
                       folio_unlock(folio);
 
Quoted 9 lines

Review static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · line 3132

Quoted 5 lines
(quoted diff) +3 -2
-                     page = folio_file_page(folio, index);
-                     if (PageHWPoison(page)) {
+                     if (folio_test_hwpoison(folio) ||
+                         (folio_test_large(folio) &&
+                          folio_test_has_hwpoisoned(folio))) {

Hm, so if we have hwpoison set on one page in a folio, we now can't read bytes from any page in the folio? That seems like we've made a bad situation worse.

Quoted 5 lines

Yeah, I agree. I think we can fallback to page copy if folio_test_has_hwpoisoned is true. The PG_hwpoison flag is per page.

The folio_test_has_hwpoisoned is kept set if the folio split is failed in memory failure handler.

Right. I can still keep the page size copy if folio_test_has_hwpoisoned() is true. Some sample changes are as follow.

Moreover, I noticed shmem splice_read() and write() also simply return an error if the folio_test_has_hwpoisoned() is true, without any fallback to page granularity. I wonder if it is worth adding page granularity support as well?

mm/shmem.c +8 -3
index 7e79b6a96da0..f30e24e529b9 100644
@@ -3111,9 +3111,11 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
        for (;;) {
                struct folio *folio = NULL;
+               struct page *page = NULL;
                unsigned long nr, ret;
                loff_t end_offset, i_size = i_size_read(inode);
                size_t fsize;
+               bool fallback_page_copy = false;
 
                if (unlikely(iocb->ki_pos >= i_size))
                        break;
@@ -3127,13 +3129,16 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
                if (folio) {
                        folio_unlock(folio);
 
-                       if (folio_test_hwpoison(folio) ||
-                           (folio_test_large(folio) &&
-                            folio_test_has_hwpoisoned(folio))) {
+                       page = folio_file_page(folio, index);
+                       if (PageHWPoison(page)) {
                                folio_put(folio);
                                error = -EIO;
                                break;
                        }
+
+                       if (folio_test_large(folio) &&
+                           folio_test_has_hwpoisoned(folio))
+                               fallback_page_copy = true;
                }
 
                /*
@@ -3147,7 +3152,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
                        break;
                }
                end_offset = min_t(loff_t, i_size, iocb->ki_pos + 

to->count);
- if (folio)
+ if (folio && likely(!fallback_page_copy))

                        fsize = folio_size(folio);
                else
                        fsize = PAGE_SIZE;

@@ -3160,8 +3165,13 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
* virtual addresses, take care about potential
aliasing
* before reading the page on the kernel side.
*/
- if (mapping_writably_mapped(mapping))
- flush_dcache_folio(folio);
+ if (mapping_writably_mapped(mapping)) {
+ if (unlikely(fallback_page_copy))
+ flush_dcache_page(page);
+ else
+ flush_dcache_folio(folio);
+ }
+

                        /*
                         * Mark the page accessed if we read the beginning.
                         */

@@ -3171,7 +3181,10 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
* Ok, we have the page, and it's up-to-date, so
* now we can copy it to user space...
*/
- ret = copy_folio_to_iter(folio, offset, nr, to);
+ if (unlikely(fallback_page_copy))
+ ret = copy_page_to_iter(page, offset, nr, to);
+ else

+                               ret = copy_folio_to_iter(folio, offset, nr, to);
                        folio_put(folio);
                } else if (user_backed_iter(to)) {
                        /*
#10 lore raw

Re: [PATCH 2/2] mm: shmem: improve the tmpfs large folio read performance

Quoted 27 lines · Baolin Wang
Quoted 17 lines · Yang Shi
Quoted 6 lines · Matthew Wilcox
Quoted 4 lines · Baolin Wang
mm/shmem.c +0 -0
@@ -3128,8 +3127,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
               if (folio) {
                       folio_unlock(folio);
 
Quoted 9 lines

Review static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · line 3132

Quoted 5 lines
(quoted diff) +3 -2
-                     page = folio_file_page(folio, index);
-                     if (PageHWPoison(page)) {
+                     if (folio_test_hwpoison(folio) ||
+                         (folio_test_large(folio) &&
+                          folio_test_has_hwpoisoned(folio))) {

Hm, so if we have hwpoison set on one page in a folio, we now can't read bytes from any page in the folio? That seems like we've made a bad situation worse.

Quoted 5 lines

Yeah, I agree. I think we can fallback to page copy if folio_test_has_hwpoisoned is true. The PG_hwpoison flag is per page.

The folio_test_has_hwpoisoned is kept set if the folio split is failed in memory failure handler.

Quoted 7 lines

Right. I can still keep the page size copy if folio_test_has_hwpoisoned() is true. Some sample changes are as follow.

Moreover, I noticed shmem splice_read() and write() also simply return an error if the folio_test_has_hwpoisoned() is true, without any fallback to page granularity. I wonder if it is worth adding page granularity support as well?

I think you should do the same.

Quoted 80 lines
mm/shmem.c +2 -0
index 7e79b6a96da0..f30e24e529b9 100644
@@ -3111,9 +3111,11 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
 
         for (;;) {
                 struct folio *folio = NULL;
+               struct page *page = NULL;
                 unsigned long nr, ret;
                 loff_t end_offset, i_size = i_size_read(inode);
                 size_t fsize;
+               bool fallback_page_copy = false;
 
                 if (unlikely(iocb->ki_pos >= i_size))
                         break;
mm/shmem.c +6 -3
@@ -3127,13 +3129,16 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
                 if (folio) {
                         folio_unlock(folio);
 
-                       if (folio_test_hwpoison(folio) ||
-                           (folio_test_large(folio) &&
-                            folio_test_has_hwpoisoned(folio))) {
+                       page = folio_file_page(folio, index);
+                       if (PageHWPoison(page)) {
                                 folio_put(folio);
                                 error = -EIO;
                                 break;
                         }
+
+                       if (folio_test_large(folio) &&
+                           folio_test_has_hwpoisoned(folio))
+                               fallback_page_copy = true;
                 }
 
                 /*
mm/shmem.c +1 -1
@@ -3147,7 +3152,7 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
                         break;
                 }
                 end_offset = min_t(loff_t, i_size, iocb->ki_pos +
 to->count);
-               if (folio)
+               if (folio && likely(!fallback_page_copy))
                         fsize = folio_size(folio);
                 else
                         fsize = PAGE_SIZE;
mm/shmem.c +0 -0
@@ -3160,8 +3165,13 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
                          * virtual addresses, take care about potential

aliasing
* before reading the page on the kernel side. */

mm/shmem.c +11 -3
-                       if (mapping_writably_mapped(mapping))
-                               flush_dcache_folio(folio);
+                       if (mapping_writably_mapped(mapping)) {
+                               if (unlikely(fallback_page_copy))
+                                       flush_dcache_page(page);
+                               else
+                                       flush_dcache_folio(folio);
+                       }
+
                         /*
                          * Mark the page accessed if we read the beginning.
                          */
@@ -3171,7 +3181,10 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
                          * Ok, we have the page, and it's up-to-date, so
                          * now we can copy it to user space...
                          */
-                       ret = copy_folio_to_iter(folio, offset, nr, to);
+                       if (unlikely(fallback_page_copy))
+                               ret = copy_page_to_iter(page, offset,
 nr, to);
+                       else
+                               ret = copy_folio_to_iter(folio, offset,
 nr, to);
Quoted 3 lines
                         folio_put(folio);
                 } else if (user_backed_iter(to)) {
                         /*

The change seems fine to me.

author reviewer
#11 lore raw

Re: [PATCH 2/2] mm: shmem: improve the tmpfs large folio read performance

Quoted 32 lines · Yang Shi
Quoted 30 lines · Baolin Wang
Quoted 17 lines · Yang Shi
Quoted 6 lines · Matthew Wilcox
Quoted 4 lines · Baolin Wang
mm/shmem.c +0 -0
@@ -3128,8 +3127,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
                if (folio) {
                        folio_unlock(folio);
 
Quoted 9 lines

Review static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · line 3132

Quoted 5 lines
(quoted diff) +3 -2
-                     page = folio_file_page(folio, index);
-                     if (PageHWPoison(page)) {
+                     if (folio_test_hwpoison(folio) ||
+                         (folio_test_large(folio) &&
+                          folio_test_has_hwpoisoned(folio))) {

Hm, so if we have hwpoison set on one page in a folio, we now can't read bytes from any page in the folio? That seems like we've made a bad situation worse.

Quoted 5 lines

Yeah, I agree. I think we can fallback to page copy if folio_test_has_hwpoisoned is true. The PG_hwpoison flag is per page.

The folio_test_has_hwpoisoned is kept set if the folio split is failed in memory failure handler.

Right. I can still keep the page size copy if folio_test_has_hwpoisoned() is true. Some sample changes are as follow.

Quoted 6 lines
Quoted 4 lines

Moreover, I noticed shmem splice_read() and write() also simply return an error if the folio_test_has_hwpoisoned() is true, without any fallback to page granularity. I wonder if it is worth adding page granularity support as well?

I think you should do the same.

OK. Let me have a detailed look.

Quoted 76 lines
Quoted 76 lines
mm/shmem.c +2 -0
index 7e79b6a96da0..f30e24e529b9 100644
@@ -3111,9 +3111,11 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
 
          for (;;) {
                  struct folio *folio = NULL;
+               struct page *page = NULL;
                  unsigned long nr, ret;
                  loff_t end_offset, i_size = i_size_read(inode);
                  size_t fsize;
+               bool fallback_page_copy = false;
 
                  if (unlikely(iocb->ki_pos >= i_size))
                          break;
mm/shmem.c +6 -3
@@ -3127,13 +3129,16 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
                  if (folio) {
                          folio_unlock(folio);
 
-                       if (folio_test_hwpoison(folio) ||
-                           (folio_test_large(folio) &&
-                            folio_test_has_hwpoisoned(folio))) {
+                       page = folio_file_page(folio, index);
+                       if (PageHWPoison(page)) {
                                  folio_put(folio);
                                  error = -EIO;
                                  break;
                          }
+
+                       if (folio_test_large(folio) &&
+                           folio_test_has_hwpoisoned(folio))
+                               fallback_page_copy = true;
                  }
 
                  /*
mm/shmem.c +1 -1
@@ -3147,7 +3152,7 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
                          break;
                  }
                  end_offset = min_t(loff_t, i_size, iocb->ki_pos +
 to->count);
-               if (folio)
+               if (folio && likely(!fallback_page_copy))
                          fsize = folio_size(folio);
                  else
                          fsize = PAGE_SIZE;
mm/shmem.c +0 -0
@@ -3160,8 +3165,13 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
                           * virtual addresses, take care about potential

aliasing
* before reading the page on the kernel side. */

mm/shmem.c +9 -3
-                       if (mapping_writably_mapped(mapping))
-                               flush_dcache_folio(folio);
+                       if (mapping_writably_mapped(mapping)) {
+                               if (unlikely(fallback_page_copy))
+                                       flush_dcache_page(page);
+                               else
+                                       flush_dcache_folio(folio);
+                       }
+
                          /*
                           * Mark the page accessed if we read the beginning.
                           */
@@ -3171,7 +3181,10 @@ static ssize_t shmem_file_read_iter(struct kiocb
 *iocb, struct iov_iter *to)
                           * Ok, we have the page, and it's up-to-date, so
                           * now we can copy it to user space...
                           */
-                       ret = copy_folio_to_iter(folio, offset, nr, to);
+                       if (unlikely(fallback_page_copy))
+                               ret = copy_page_to_iter(page, offset,
 nr, to);
Quoted 8 lines
Quoted 6 lines
mm/shmem.c +2 -0
+                       else
+                               ret = copy_folio_to_iter(folio, offset,
 nr, to);
                          folio_put(folio);
                  } else if (user_backed_iter(to)) {
                          /*

The change seems fine to me.

Thanks.