mm/mmzone: Introduce a new macro for_each_node_zonelist()
Subsystems (6)
PATCH v2 1/1 mm/mmzone: Introduce a new macro for_each_node_zonelist()
There are some codes using for_each_zone_zonelist() even when only iterating each node is needed. This commit introduces a new macro for_each_node_zonelist() which iterates through valid nodes in the zonelist.
By using this new macro, code can be written in a much simpler form. Also, slab/slub can now skip trying to allocate from the node which was previously tried and failed.
---
V1 -> V2: Fix a compile error
include/linux/mmzone.h | 36 ++++++++++++++++++++++++++++++++++++ mm/hugetlb.c | 17 +++++++---------- mm/mmzone.c | 17 +++++++++++++++++ mm/slab.c | 7 ++----- mm/slub.c | 8 ++++---- mm/vmscan.c | 16 ++++++---------- 6 files changed, 72 insertions(+), 29 deletions(-)
include/linux/mmzone.h +36 -0
@@ -1464,6 +1464,42 @@ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,#define for_each_zone_zonelist (zone, z, zlist, highidx) \ for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL) + +struct zoneref *next_node_zones_zonelist (struct zoneref *z, +int prev_nid, +enum zone_type highest_zoneidx, +nodemask_t *nodes); + +/** +* for_each_node_zonelist_nodemask - helper macro to iterate over valid nodes in a zonelist which have at least one zone at or below a given zone index and within a nodemask +* @node: The current node in the iterator +* @z: First matched zoneref within current node +* @zlist: The zonelist being iterated +* @highidx: The zone index of the highest zone in the node +* @nodemask: Nodemask allowed by the allocator +* +* This iterator iterates through all nodes which have at least one zone at or below a given zone index and +* within a given nodemask +*/ +#define for_each_node_zonelist_nodemask (node, z, zlist, highidx, nodemask) \ + for (z = first_zones_zonelist(zlist, highidx, nodemask), \ + node = zonelist_zone(z) ? zonelist_node_idx(z) : NUMA_NO_NODE; \ + zonelist_zone(z); \ + z = next_node_zones_zonelist(++z, node, highidx, nodemask), \ + node = zonelist_zone(z) ? zonelist_node_idx(z) : NUMA_NO_NODE) + +/** +* for_each_node_zonelist - helper macro to iterate over nodes in a zonelist which have at least one zone at or below a given zone index +* @node: The current node in the iterator +* @z: First matched zoneref within current node +* @zlist: The zonelist being iterated +* @highidx: The zone index of the highest zone in the node +* +* This iterator iterates through all nodes which have at least one zone at or below a given zone index. +*/ +#define for_each_node_zonelist (node, z, zlist, highidx) \ + for_each_node_zonelist_nodemask(node, z, zlist, highidx, NULL) +/* Whether the 'nodes' are all movable nodes */ static inline bool movable_only_nodes (nodemask_t *nodes) {
mm/hugetlb.c +7 -10
@@ -1157,7 +1157,6 @@ static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, {unsigned int cpuset_mems_cookie;struct zonelist *zonelist; -struct zone *zone;struct zoneref *z;int node =NUMA_NO_NODE ;
@@ -1165,18 +1164,16 @@ static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, retry_cpuset: cpuset_mems_cookie =read_mems_allowed_begin (); -for_each_zone_zonelist_nodemask (zone, z, zonelist,gfp_zone (gfp_mask), nmask) { + +/* +* no need to ask again on the same node. Pool is node rather than +* zone aware +*/ +for_each_node_zonelist_nodemask (node, z, zonelist,gfp_zone (gfp_mask), nmask) {struct page *page; -if (!cpuset_zone_allowed (zone, gfp_mask)) -continue ; -/* -* no need to ask again on the same node. Pool is node rather than -* zone aware -*/ -if (zone_to_nid (zone) == node) +if (!cpuset_node_allowed (node, gfp_mask))continue ; - node =zone_to_nid (zone); page =dequeue_huge_page_node_exact (h, node);if (page)
mm/mmzone.c +17 -0
@@ -72,6 +72,23 @@ struct zoneref *__next_zones_zonelist(struct zoneref *z,return z; } +/* Returns the zone in the next node and at or below highest_zoneidx in a zonelist */ +struct zoneref *next_node_zones_zonelist (struct zoneref *z, +int prev_nid, +enum zone_type highest_zoneidx, +nodemask_t *nodes) +{ +if (likely (nodes ==NULL )) +while (z->zone && (zonelist_node_idx (z) == prev_nid ||zonelist_zone_idx (z) > highest_zoneidx)) + z++; +else +while (z->zone && (zonelist_node_idx (z) == prev_nid ||zonelist_zone_idx (z) > highest_zoneidx || + !zref_in_nodemask (z, nodes))) + z++; + +return z; +} +void lruvec_init (struct lruvec *lruvec) {enum lru_list lru;
mm/slab.c +2 -5
@@ -3077,7 +3077,6 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) {struct zonelist *zonelist;struct zoneref *z; -struct zone *zone;enum zone_type highest_zoneidx =gfp_zone (flags);void *obj =NULL ;struct slab *slab;
@@ -3096,10 +3095,8 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)* Look through allowed nodes for objects available * from existing per node queues. */ -for_each_zone_zonelist (zone, z, zonelist, highest_zoneidx) { - nid =zone_to_nid (zone); - -if (cpuset_zone_allowed (zone, flags) && +for_each_node_zonelist (nid, z, zonelist, highest_zoneidx) { +if (cpuset_node_allowed (nid, flags) &&get_node (cache, nid) &&get_node (cache, nid)->free_objects) { obj = ____cache_alloc_node(cache,
mm/slub.c +4 -4
@@ -2192,7 +2192,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags,#ifdef CONFIG_NUMA struct zonelist *zonelist;struct zoneref *z; -struct zone *zone; +int nid;enum zone_type highest_zoneidx =gfp_zone (flags);void *object;unsigned int cpuset_mems_cookie;
@@ -2222,12 +2222,12 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, do { cpuset_mems_cookie = read_mems_allowed_begin(); zonelist = node_zonelist(mempolicy_slab_node(), flags); - for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { + for_each_node_zonelist(nid, z, zonelist, highest_zoneidx) { struct kmem_cache_node *n; - n = get_node(s, zone_to_nid(zone)); + n = get_node(s, nid); - if (n && cpuset_zone_allowed(zone, flags) && + if (n && cpuset_node_allowed(nid, flags) && n->nr_partial > s->min_partial) { object = get_partial_node(s, n, ret_slab, flags); if (object) {
mm/vmscan.c +6 -10
@@ -6176,9 +6176,9 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, struct scan_control *sc) {int initial_priority = sc->priority; -pg_data_t *last_pgdat; +pg_data_t *pgdat;struct zoneref *z; -struct zone *zone; +int nid; retry:delayacct_freepages_start ();
@@ -6205,20 +6205,16 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, sc->may_writepage =1 ; }while (--sc->priority >=0 ); - last_pgdat =NULL ; -for_each_zone_zonelist_nodemask (zone, z, zonelist, sc->reclaim_idx, +for_each_node_zonelist_nodemask (nid, z, zonelist, sc->reclaim_idx, sc->nodemask) { -if (zone->zone_pgdat == last_pgdat) -continue ; - last_pgdat = zone->zone_pgdat; + pgdat =NODE_DATA (nid); -snapshot_refaults (sc->target_mem_cgroup, zone->zone_pgdat); +snapshot_refaults (sc->target_mem_cgroup, pgdat);if (cgroup_reclaim (sc)) {struct lruvec *lruvec; - lruvec =mem_cgroup_lruvec (sc->target_mem_cgroup, - zone->zone_pgdat); + lruvec =mem_cgroup_lruvec (sc->target_mem_cgroup, pgdat);clear_bit (LRUVEC_CONGESTED , &lruvec->flags); } }
Signature · 1 line
2.17.1
Mike Rapoport
reviewerQuoted 15 lines · Donghyeok Kim
There are some codes using
for_each_zone_zonelist()even when only iterating each node is needed. This commit introduces a new macrofor_each_node_zonelist()which iterates through valid nodes in the zonelist.By using this new macro, code can be written in a much simpler form. Also, slab/slub can now skip trying to allocate from the node which was previously tried and failed.
Co-developed-by Ohhoon Kwon <[email protected]>Signed-off-by Ohhoon Kwon <[email protected]>Signed-off-by Donghyeok Kim <[email protected]>---
V1 -> V2: Fix a compile error
Quoted 7 lines
include/linux/mmzone.h | 36 ++++++++++++++++++++++++++++++++++++ mm/hugetlb.c | 17 +++++++---------- mm/mmzone.c | 17 +++++++++++++++++ mm/slab.c | 7 ++----- mm/slub.c | 8 ++++---- mm/vmscan.c | 16 ++++++---------- 6 files changed, 72 insertions(+), 29 deletions(-)
Even counting the comments this does not look like a great simplification.
Quoted 1 line
...
Quoted 25 lines
mm/hugetlb.c +6
-2@@ -1157,7 +1157,6 @@ static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, {unsigned int cpuset_mems_cookie;struct zonelist *zonelist; -struct zone *zone;struct zoneref *z;int node =NUMA_NO_NODE ;@@ -1165,18 +1164,16 @@ static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, retry_cpuset: cpuset_mems_cookie =read_mems_allowed_begin (); -for_each_zone_zonelist_nodemask (zone, z, zonelist,gfp_zone (gfp_mask), nmask) { + +/* +* no need to ask again on the same node. Pool is node rather than +* zone aware +*/ +for_each_node_zonelist_nodemask (node, z, zonelist,gfp_zone (gfp_mask), nmask) {struct page *page;
mm/hugetlb.c static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, · line 1179
Quoted 10 lines
mm/hugetlb.c +1
-8-if (!cpuset_zone_allowed (zone, gfp_mask)) -continue ; -/* -* no need to ask again on the same node. Pool is node rather than -* zone aware -*/ -if (zone_to_nid (zone) == node) +if (!cpuset_node_allowed (node, gfp_mask))continue ; - node =zone_to_nid (zone);
The actual change here seems to be the deletion of the node assignment. I'd say it's not worth the churn.
Quoted 3 lines
mm/hugetlb.c +0
-0page =dequeue_huge_page_node_exact (h, node);if (page)
...
Quoted 13 lines
mm/slab.c +0
-1@@ -3077,7 +3077,6 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) {struct zonelist *zonelist;struct zoneref *z; -struct zone *zone;enum zone_type highest_zoneidx =gfp_zone (flags);void *obj =NULL ;struct slab *slab;
mm/slab.c static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) · line 3099
Quoted 9 lines
mm/slab.c +2
-4@@ -3096,10 +3095,8 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)* Look through allowed nodes for objects available * from existing per node queues. */ -for_each_zone_zonelist (zone, z, zonelist, highest_zoneidx) { - nid =zone_to_nid (zone); - - if (cpuset_zone_allowed(zone, flags) && +for_each_node_zonelist (nid, z, zonelist, highest_zoneidx) { + if (cpuset_node_allowed(nid, flags) &&
Same here.
Quoted 20 lines
mm/slab.c +0
-0get_node(cache, nid) && get_node(cache, nid)->free_objects) { obj = ____cache_alloc_node(cache,mm/slub.c +1
-1@@ -2192,7 +2192,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags,#ifdef CONFIG_NUMA struct zonelist *zonelist;struct zoneref *z; -struct zone *zone; +int nid;enum zone_type highest_zoneidx =gfp_zone (flags);void *object;unsigned int cpuset_mems_cookie;@@ -2222,12 +2222,12 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, do { cpuset_mems_cookie = read_mems_allowed_begin(); zonelist = node_zonelist(mempolicy_slab_node(), flags);
mm/slub.c static void *get_any_partial(struct kmem_cache *s, gfp_t flags, · line 2231
Quoted 10 lines
mm/slub.c +3
-3-for_each_zone_zonelist (zone, z, zonelist, highest_zoneidx) { +for_each_node_zonelist (nid, z, zonelist, highest_zoneidx) {struct kmem_cache_node *n; - n =get_node (s,zone_to_nid (zone)); + n =get_node (s, nid); -if (n &&cpuset_zone_allowed (zone, flags) && +if (n &&cpuset_node_allowed (nid, flags) && n->nr_partial > s->min_partial) {
And the changes here look rather like using different iterator variable.
Quoted 20 lines
mm/slub.c +0
-0object =get_partial_node (s, n, ret_slab, flags);if (object) {mm/vmscan.c +2
-2@@ -6176,9 +6176,9 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, struct scan_control *sc) {int initial_priority = sc->priority; -pg_data_t *last_pgdat; +pg_data_t *pgdat;struct zoneref *z; -struct zone *zone; +int nid; retry:delayacct_freepages_start ();@@ -6205,20 +6205,16 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, sc->may_writepage =1 ;
mm/vmscan.c static unsigned long do_try_to_free_pages(struct zonelist *zonelist, · line 6210
Quoted 10 lines
mm/vmscan.c +2
-5}while (--sc->priority >=0 ); - last_pgdat =NULL ; -for_each_zone_zonelist_nodemask (zone, z, zonelist, sc->reclaim_idx, +for_each_node_zonelist_nodemask (nid, z, zonelist, sc->reclaim_idx, sc->nodemask) { -if (zone->zone_pgdat == last_pgdat) -continue ; - last_pgdat = zone->zone_pgdat; + pgdat =NODE_DATA (nid);
I don't think this simplification is compelling enough to justify all the changes.
Quoted 17 lines
mm/vmscan.c +2
-3-snapshot_refaults (sc->target_mem_cgroup, zone->zone_pgdat); +snapshot_refaults (sc->target_mem_cgroup, pgdat);if (cgroup_reclaim (sc)) {struct lruvec *lruvec; - lruvec =mem_cgroup_lruvec (sc->target_mem_cgroup, - zone->zone_pgdat); + lruvec =mem_cgroup_lruvec (sc->target_mem_cgroup, pgdat);clear_bit (LRUVEC_CONGESTED , &lruvec->flags); } }Signature · 1 line
2.17.1
Signature · 2 lines
Sincerely yours, Mike.
Quoted 24 lines · Mike Rapoport
Quoted 15 lines · Donghyeok Kim
There are some codes using
for_each_zone_zonelist()even when only iterating each node is needed. This commit introduces a new macrofor_each_node_zonelist()which iterates through valid nodes in the zonelist.By using this new macro, code can be written in a much simpler form. Also, slab/slub can now skip trying to allocate from the node which was previously tried and failed.
Co-developed-by Ohhoon Kwon <[email protected]>Signed-off-by Ohhoon Kwon <[email protected]>Signed-off-by Donghyeok Kim <[email protected]>---
V1 -> V2: Fix a compile errorQuoted 7 lines
include/linux/mmzone.h | 36 ++++++++++++++++++++++++++++++++++++ mm/hugetlb.c | 17 +++++++---------- mm/mmzone.c | 17 +++++++++++++++++ mm/slab.c | 7 ++----- mm/slub.c | 8 ++++---- mm/vmscan.c | 16 ++++++---------- 6 files changed, 72 insertions(+), 29 deletions(-)
Quoted 2 lines
Even counting the comments this does not look like a great simplification.
I tend to agree.
Signature · 3 lines
Thanks, David / dhildenb