Skip to contentSkip to conversation

z3fold: fix reclaim lock-ups

PATCH In review Jongseok Kim 2 participants 4 messages May 3, 2018 – May 11, 2018 Memory management

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 z3fold: fix reclaim lock-ups

In the processing of headless pages, there was a problem that the zhdr pointed to another page or a page was alread released in z3fold_free(). So, the wrong page is encoded in headless, or test_bit does not work properly in z3fold_reclaim_page(). This patch fixed these problems.

---
 mm/z3fold.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)
mm/z3fold.c +7 -5
index d5b3f49..a36ffa5ab 100644
@@ -819,6 +819,7 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle)
 static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 {
 	int i, ret = 0;
+	bool is_headless;
 	struct z3fold_header *zhdr = NULL;
 	struct page *page = NULL;
 	struct list_head *pos;
@@ -836,11 +837,11 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 		}
 		list_for_each_prev(pos, &pool->lru) {
 			page = list_entry(pos, struct page, lru);
+			zhdr = page_address(page);
 			if (test_bit(PAGE_HEADLESS, &page->private))
 				/* candidate found */
 				break;
 
-			zhdr = page_address(page);
 			if (!z3fold_page_trylock(zhdr))
 				continue; /* can't evict at this point */
 			kref_get(&zhdr->refcount);
@@ -873,9 +874,11 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 			 * reference to this page
 			 */
 			z3fold_page_unlock(zhdr);
+			is_headless = false;
 		} else {
 			first_handle = encode_handle(zhdr, HEADLESS);
 			last_handle = middle_handle = 0;
+			is_headless = true;
 		}
 
 		/* Issue the eviction callback(s) */
@@ -895,11 +898,10 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 				goto next;
 		}
 next:
-		if (test_bit(PAGE_HEADLESS, &page->private)) {
-			if (ret == 0) {
-				free_z3fold_page(page);
+		if (is_headless) {
+			if (ret == 0)
 				return 0;
-			}
+
 			spin_lock(&pool->lock);
 			list_add(&page->lru, &pool->lru);
 			spin_unlock(&pool->lock);
Signature · 1 line
2.7.4
#1 lore raw

Hi Jongseok,

Den tors 3 maj 2018 kl 08:36 skrev Jongseok Kim <[email protected]>:

mm/z3fold.c static void z3fold_free(struct z3fold_pool *pool,

Quoted 16 lines

In the processing of headless pages, there was a problem that the zhdr pointed to another page or a page was alread released in z3fold_free(). So, the wrong page is encoded in headless, or test_bit does not work properly in z3fold_reclaim_page(). This patch fixed these problems.

---
  mm/z3fold.c | 12 +++++++-----
  1 file changed, 7 insertions(+), 5 deletions(-)
mm/z3fold.c +0 -0
index d5b3f49..a36ffa5ab 100644
@@ -819,6 +819,7 @@ static void z3fold_free(struct z3fold_pool *pool,

unsigned long handle)

mm/z3fold.c static void z3fold_free(struct z3fold_pool *pool, · line 819

Quoted 1 line
mm/z3fold.c +0 -0
  static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int

retries)

mm/z3fold.c static int z3fold_reclaim_page(struct z3fold_pool

Quoted 7 lines
mm/z3fold.c +1 -0
  {
         int i, ret = 0;
+       bool is_headless;
         struct z3fold_header *zhdr = NULL;
         struct page *page = NULL;
         struct list_head *pos;
@@ -836,11 +837,11 @@ static int z3fold_reclaim_page(struct z3fold_pool

*pool, unsigned int retries)

mm/z3fold.c static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) · line 847

Quoted 12 lines
mm/z3fold.c +1 -1
                 }
                 list_for_each_prev(pos, &pool->lru) {
                         page = list_entry(pos, struct page, lru);
+                       zhdr = page_address(page);
                         if (test_bit(PAGE_HEADLESS, &page->private))
                                 /* candidate found */
                                 break;
 
-                       zhdr = page_address(page);
                         if (!z3fold_page_trylock(zhdr))
                                 continue; /* can't evict at this point */
                         kref_get(&zhdr->refcount);

I don't see how that is relevant, we don't use zhdr variable if the page is headless anyway.

mm/z3fold.c static int z3fold_reclaim_page(struct z3fold_pool

Quoted 1 line
mm/z3fold.c +0 -0
@@ -873,9 +874,11 @@ static int z3fold_reclaim_page(struct z3fold_pool

*pool, unsigned int retries)

mm/z3fold.c static int z3fold_reclaim_page(struct z3fold_pool

Quoted 12 lines
mm/z3fold.c +2 -0
                          * reference to this page
                          */
                         z3fold_page_unlock(zhdr);
+                       is_headless = false;
                 } else {
                         first_handle = encode_handle(zhdr, HEADLESS);
                         last_handle = middle_handle = 0;
+                       is_headless = true;
                 }
 
                 /* Issue the eviction callback(s) */
@@ -895,11 +898,10 @@ static int z3fold_reclaim_page(struct z3fold_pool

*pool, unsigned int retries)

mm/z3fold.c static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) · line 904

Quoted 11 lines
mm/z3fold.c +3 -4
                                 goto next;
                 }
  next:
-               if (test_bit(PAGE_HEADLESS, &page->private)) {
-                       if (ret == 0) {
-                               free_z3fold_page(page);
+               if (is_headless) {
+                       if (ret == 0)
                                 return 0;
-                       }
+

Thanks, that is indeed a valid fix, but given the operation of eviction in zswap case, I'd prefer to release the page here and not in _free(). Basically we should bail very early from z3fold_free() if the page is headless and UNDER_RECLAIM and keep the code intact in z3fold_reclaim_page().

~Vitaly

#2 lore raw
Quoted 3 lines

Hi Jongseok,

I don't see how that is relevant, we don't use zhdr variable if the page is

headless anyway.

Quoted 7 lines

Thanks, that is indeed a valid fix, but given the operation of eviction in zswap case, I'd prefer to release the page here and not in _free(). Basically we should bail very early from z3fold_free() if the page is headless and UNDER_RECLAIM and keep the code intact in z3fold_reclaim_page().

~Vitaly

Hi, Vitaly. Thanks for your reply.
Then I think the below code is your preferred code.

And explaining about zhdr, this code don't mean that zhdr pointer is used as z3fold_header. Because it's headless like you said. Even though it is headless, however, 'zhdr', the pointer of z3fold_header, is still used in _alloc and encode_handle as an address of a headless page.

So in pervious code, if eviction fails (it's not headless) and the selected page in the next try loop is headless, then 'zhdr' will remain the value of the previously failed page, which will cause a not headless page to be encoded to headless. This is why a page that is not headless is encoded as HEADLESS in _map, which causes 'z3fold: unknown buddy id 0' warning.

And I have a one question. In z3fold_reclaim_page, is it because of performance that you added non-headless pages only to lru list except buddy list?

~Jongseok

---
 mm/z3fold.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
mm/z3fold.c +5 -1
index d5b3f49..5f659ab 100644
@@ -746,6 +746,9 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle)
 	}
 
 	if (bud == HEADLESS) {
+		if (test_bit(UNDER_RECLAIM, &page->private))
+			return;
+
 		spin_lock(&pool->lock);
 		list_del(&page->lru);
 		spin_unlock(&pool->lock);
@@ -836,11 +839,11 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 		}
 		list_for_each_prev(pos, &pool->lru) {
 			page = list_entry(pos, struct page, lru);
+			zhdr = page_address(page);
 			if (test_bit(PAGE_HEADLESS, &page->private))
 				/* candidate found */
 				break;
 
-			zhdr = page_address(page);
 			if (!z3fold_page_trylock(zhdr))
 				continue; /* can't evict at this point */
 			kref_get(&zhdr->refcount);
@@ -898,6 +901,7 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 		if (test_bit(PAGE_HEADLESS, &page->private)) {
 			if (ret == 0) {
 				free_z3fold_page(page);
+				atomic64_dec(&pool->pages_nr);
 				return 0;
 			}
 			spin_lock(&pool->lock);
Signature · 1 line
2.7.4
#3 lore raw

A headless page also need to be set UNDER_RECLAIM in previous reply, but I missed it.

---
 mm/z3fold.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
mm/z3fold.c +2 -1
index 5f659ab..8536a47 100644
@@ -849,10 +849,10 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 			kref_get(&zhdr->refcount);
 			list_del_init(&zhdr->buddy);
 			zhdr->cpu = -1;
-			set_bit(UNDER_RECLAIM, &page->private);
 			break;
 		}
 
+		set_bit(UNDER_RECLAIM, &page->private);
 		list_del_init(&page->lru);
 		spin_unlock(&pool->lock);
 
@@ -899,6 +899,7 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 		}
 next:
 		if (test_bit(PAGE_HEADLESS, &page->private)) {
+			clear_bit(UNDER_RECLAIM, &page->private);
 			if (ret == 0) {
 				free_z3fold_page(page);
 				atomic64_dec(&pool->pages_nr);
Signature · 1 line
2.7.4