Skip to contentSkip to conversation

mm: replace xa_get_order with xas_get_order where appropriate

PATCH Applied Shakeel Butt 4 participants 4 messages Sep 6, 2024 – Sep 10, 2024

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 mm: replace xa_get_order with xas_get_order where appropriate

The tracing of invalidation and truncation operations on large files showed that xa_get_order() is among the top functions where kernel spends a lot of CPUs. xa_get_order() needs to traverse the tree to reach the right node for a given index and then extract the order of the entry. However it seems like at many places it is being called within an already happening tree traversal where there is no need to do another traversal. Just use xas_get_order() at those places.

---
 mm/filemap.c | 6 +++---
 mm/shmem.c   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
mm/filemap.c +3 -3
index 070dee9791a9..7e3412941a8d 100644
@@ -2112,7 +2112,7 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
 			VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
 					folio);
 		} else {
-			nr = 1 << xa_get_order(&mapping->i_pages, xas.xa_index);
+			nr = 1 << xas_get_order(&xas);
 			base = xas.xa_index & ~(nr - 1);
 			/* Omit order>0 value which begins before the start */
 			if (base < *start)
@@ -3001,7 +3001,7 @@ static inline loff_t folio_seek_hole_data(struct xa_state *xas,
 static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio)
 {
 	if (xa_is_value(folio))
-		return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index);
+		return PAGE_SIZE << xas_get_order(xas);
 	return folio_size(folio);
 }
 
@@ -4297,7 +4297,7 @@ static void filemap_cachestat(struct address_space *mapping,
 		if (xas_retry(&xas, folio))
 			continue;
 
-		order = xa_get_order(xas.xa, xas.xa_index);
+		order = xas_get_order(&xas);
 		nr_pages = 1 << order;
 		folio_first_index = round_down(xas.xa_index, 1 << order);
 		folio_last_index = folio_first_index + nr_pages - 1;
mm/shmem.c +1 -1
index 866d46d0c43d..4002c4f47d4d 100644
@@ -893,7 +893,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
 		if (xas_retry(&xas, page))
 			continue;
 		if (xa_is_value(page))
-			swapped += 1 << xa_get_order(xas.xa, xas.xa_index);
+			swapped += 1 << xas_get_order(&xas);
 		if (xas.xa_index == max)
 			break;
 		if (need_resched()) {
Signature · 1 line
2.43.5
Reviewed-by on v1 1/1
#1 lore raw
Quoted 9 lines · Shakeel Butt

The tracing of invalidation and truncation operations on large files showed that xa_get_order() is among the top functions where kernel spends a lot of CPUs. xa_get_order() needs to traverse the tree to reach the right node for a given index and then extract the order of the entry. However it seems like at many places it is being called within an already happening tree traversal where there is no need to do another traversal. Just use xas_get_order() at those places.

This change alters areas that already do the rcu locking and the internal state of the xas isn't altered, so the external loops are not affected, afaict.

Looks good to me.

Quoted 53 lines
---
 mm/filemap.c | 6 +++---
 mm/shmem.c   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
mm/filemap.c +3 -3
index 070dee9791a9..7e3412941a8d 100644
@@ -2112,7 +2112,7 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
 			VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
 					folio);
 		} else {
-			nr = 1 << xa_get_order(&mapping->i_pages, xas.xa_index);
+			nr = 1 << xas_get_order(&xas);
 			base = xas.xa_index & ~(nr - 1);
 			/* Omit order>0 value which begins before the start */
 			if (base < *start)
@@ -3001,7 +3001,7 @@ static inline loff_t folio_seek_hole_data(struct xa_state *xas,
 static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio)
 {
 	if (xa_is_value(folio))
-		return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index);
+		return PAGE_SIZE << xas_get_order(xas);
 	return folio_size(folio);
 }
 
@@ -4297,7 +4297,7 @@ static void filemap_cachestat(struct address_space *mapping,
 		if (xas_retry(&xas, folio))
 			continue;
 
-		order = xa_get_order(xas.xa, xas.xa_index);
+		order = xas_get_order(&xas);
 		nr_pages = 1 << order;
 		folio_first_index = round_down(xas.xa_index, 1 << order);
 		folio_last_index = folio_first_index + nr_pages - 1;
mm/shmem.c +1 -1
index 866d46d0c43d..4002c4f47d4d 100644
@@ -893,7 +893,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
 		if (xas_retry(&xas, page))
 			continue;
 		if (xa_is_value(page))
-			swapped += 1 << xa_get_order(xas.xa, xas.xa_index);
+			swapped += 1 << xas_get_order(&xas);
 		if (xas.xa_index == max)
 			break;
 		if (need_resched()) {
Signature · 1 line
2.43.5
reviewer Reviewed-by on v1 1/1
#2 lore raw
In reply to Shakeel Butt
Quoted 9 lines · Shakeel Butt

The tracing of invalidation and truncation operations on large files showed that xa_get_order() is among the top functions where kernel spends a lot of CPUs. xa_get_order() needs to traverse the tree to reach the right node for a given index and then extract the order of the entry. However it seems like at many places it is being called within an already happening tree traversal where there is no need to do another traversal. Just use xas_get_order() at those places.

LGTM. Thanks.

Quoted 49 lines
---
  mm/filemap.c | 6 +++---
  mm/shmem.c   | 2 +-
  2 files changed, 4 insertions(+), 4 deletions(-)
mm/filemap.c +3 -3
index 070dee9791a9..7e3412941a8d 100644
@@ -2112,7 +2112,7 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
  			VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
  					folio);
  		} else {
-			nr = 1 << xa_get_order(&mapping->i_pages, xas.xa_index);
+			nr = 1 << xas_get_order(&xas);
  			base = xas.xa_index & ~(nr - 1);
  			/* Omit order>0 value which begins before the start */
  			if (base < *start)
@@ -3001,7 +3001,7 @@ static inline loff_t folio_seek_hole_data(struct xa_state *xas,
  static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio)
  {
  	if (xa_is_value(folio))
-		return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index);
+		return PAGE_SIZE << xas_get_order(xas);
  	return folio_size(folio);
  }
 
@@ -4297,7 +4297,7 @@ static void filemap_cachestat(struct address_space *mapping,
  		if (xas_retry(&xas, folio))
  			continue;
 
-		order = xa_get_order(xas.xa, xas.xa_index);
+		order = xas_get_order(&xas);
  		nr_pages = 1 << order;
  		folio_first_index = round_down(xas.xa_index, 1 << order);
  		folio_last_index = folio_first_index + nr_pages - 1;
mm/shmem.c +1 -1
index 866d46d0c43d..4002c4f47d4d 100644
@@ -893,7 +893,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
  		if (xas_retry(&xas, page))
  			continue;
  		if (xa_is_value(page))
-			swapped += 1 << xa_get_order(xas.xa, xas.xa_index);
+			swapped += 1 << xas_get_order(&xas);
  		if (xas.xa_index == max)
  			break;
  		if (need_resched()) {
Reviewed-by on v1 1/1
#3 lore raw
In reply to Shakeel Butt
Quoted 38 lines · Shakeel Butt

The tracing of invalidation and truncation operations on large files showed that xa_get_order() is among the top functions where kernel spends a lot of CPUs. xa_get_order() needs to traverse the tree to reach the right node for a given index and then extract the order of the entry. However it seems like at many places it is being called within an already happening tree traversal where there is no need to do another traversal. Just use xas_get_order() at those places.

---
 mm/filemap.c | 6 +++---
 mm/shmem.c   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
mm/filemap.c +2 -2
index 070dee9791a9..7e3412941a8d 100644
@@ -2112,7 +2112,7 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
                        VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index),
                                        folio);
                } else {
-                       nr = 1 << xa_get_order(&mapping->i_pages, xas.xa_index);
+                       nr = 1 << xas_get_order(&xas);
                        base = xas.xa_index & ~(nr - 1);
                        /* Omit order>0 value which begins before the start */
                        if (base < *start)
@@ -3001,7 +3001,7 @@ static inline loff_t folio_seek_hole_data(struct xa_state *xas,
 static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio)
 {
        if (xa_is_value(folio))
-               return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index);
+               return PAGE_SIZE << xas_get_order(xas);
        return folio_size(folio);
 }
 

mm/filemap.c static void filemap_cachestat(struct address_space *mapping, · line 4300

Quoted 5 lines
mm/filemap.c +1 -1
@@ -4297,7 +4297,7 @@ static void filemap_cachestat(struct address_space *mapping,
                if (xas_retry(&xas, folio))
                        continue;
 
-               order = xa_get_order(xas.xa, xas.xa_index);
+               order = xas_get_order(&xas);

Yikesy that's my bad. This is late, but FWIW: