Skip to contentSkip to conversation

mm: lru_lock splitting

PATCH In review Konstantin Khlebnikov 6 participants 65 messages Feb 23, 2012 – Feb 28, 2012

Subsystems (11)
lore
Share & tools
lore
#0 lore raw

v3 cover mm: lru_lock splitting

v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving

This patchset is based on next-20120210

git: https://github.com/koct9i/linux/commits/lruvec-v3

---

Konstantin Khlebnikov (21):
memcg: unify inactive_ratio calculation
memcg: make mm_match_cgroup() hirarchical
memcg: fix page_referencies cgroup filter on global reclaim
memcg: use vm_swappiness from target memory cgroup
mm: rename lruvec->lists into lruvec->pages_lru
mm: lruvec linking functions
mm: add lruvec->pages_count
mm: unify inactive_list_is_low()
mm: add lruvec->reclaim_stat
mm: kill struct mem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec into update_page_reclaim_stat()
mm: push lruvecs from pagevec_lru_move_fn() to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: split zone->lru_lock
mm: zone lru vectors interleaving

 include/linux/huge_mm.h    |    3 
 include/linux/memcontrol.h |   75 ------
 include/linux/mm.h         |   66 +++++
 include/linux/mm_inline.h  |   19 +-
 include/linux/mmzone.h     |   39 ++-
 include/linux/swap.h       |    6 
 mm/Kconfig                 |   16 +
 mm/compaction.c            |   31 +--
 mm/huge_memory.c           |   14 +
 mm/internal.h              |  204 +++++++++++++++++
 mm/ksm.c                   |    2 
 mm/memcontrol.c            |  343 +++++++++++-----------------
 mm/migrate.c               |    2 
 mm/page_alloc.c            |   70 +-----
 mm/rmap.c                  |    2 
 mm/swap.c                  |  217 ++++++++++--------
 mm/vmscan.c                |  534 ++++++++++++++++++++++++--------------------
 mm/vmstat.c                |    6 
 18 files changed, 932 insertions(+), 717 deletions(-)
Signature · 8 lines
Signature

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
#1 lore raw

PATCH v3 1/21 memcg: unify inactive_ratio calculation

This patch removes precalculated zone->inactive_ratio.
Now it always calculated in inactive_anon_is_low() from current lru sizes. After that we can merge memcg and non-memcg cases and drop duplicated code.

---
 include/linux/memcontrol.h |   16 --------
 include/linux/mmzone.h     |    7 ----
 mm/memcontrol.c            |   38 -------------------
 mm/page_alloc.c            |   44 ----------------------
 mm/vmscan.c                |   88 ++++++++++++++++++++++++++++----------------
 mm/vmstat.c                |    6 +--
 6 files changed, 58 insertions(+), 141 deletions(-)
include/linux/memcontrol.h +0 -16
index bf4e1f4..8c4d74f 100644
@@ -113,10 +113,6 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
 /*
  * For memory reclaim.
  */
-int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg,
-				    struct zone *zone);
-int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg,
-				    struct zone *zone);
 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
 unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
 					int nid, int zid, unsigned int lrumask);
@@ -319,18 +315,6 @@ static inline bool mem_cgroup_disabled(void)
 	return true;
 }
 
-static inline int
-mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
-{
-	return 1;
-}
-
-static inline int
-mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone)
-{
-	return 1;
-}
-
 static inline unsigned long
 mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
 				unsigned int lru_mask)
include/linux/mmzone.h +0 -7
index f10a54c..3e1f7ff 100644
@@ -382,13 +382,6 @@ struct zone {
 	/* Zone statistics */
 	atomic_long_t		vm_stat[NR_VM_ZONE_STAT_ITEMS];
 
-	/*
-	 * The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on
-	 * this zone's LRU.  Maintained by the pageout code.
-	 */
-	unsigned int inactive_ratio;
-
-
 	ZONE_PADDING(_pad2_)
 	/* Rarely used or read-mostly fields */
 
mm/memcontrol.c +0 -38
index ab315ab..b8039d2 100644
@@ -1157,44 +1157,6 @@ int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
 	return ret;
 }
 
-int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
-{
-	unsigned long inactive_ratio;
-	int nid = zone_to_nid(zone);
-	int zid = zone_idx(zone);
-	unsigned long inactive;
-	unsigned long active;
-	unsigned long gb;
-
-	inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
-						BIT(LRU_INACTIVE_ANON));
-	active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
-					      BIT(LRU_ACTIVE_ANON));
-
-	gb = (inactive + active) >> (30 - PAGE_SHIFT);
-	if (gb)
-		inactive_ratio = int_sqrt(10 * gb);
-	else
-		inactive_ratio = 1;
-
-	return inactive * inactive_ratio < active;
-}
-
-int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone)
-{
-	unsigned long active;
-	unsigned long inactive;
-	int zid = zone_idx(zone);
-	int nid = zone_to_nid(zone);
-
-	inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
-						BIT(LRU_INACTIVE_FILE));
-	active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
-					      BIT(LRU_ACTIVE_FILE));
-
-	return (active > inactive);
-}
-
 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
 						      struct zone *zone)
 {
mm/page_alloc.c +0 -44

56 lines not on this page · raw message

mm/vmscan.c +56 -32

123 lines not on this page · raw message

mm/vmstat.c +2 -4
index f600557..2c813e1 100644
@@ -1017,11 +1017,9 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
 	}
 	seq_printf(m,
 		   "\n  all_unreclaimable: %u"
-		   "\n  start_pfn:         %lu"
-		   "\n  inactive_ratio:    %u",
+		   "\n  start_pfn:         %lu",
 		   zone->all_unreclaimable,
-		   zone->zone_start_pfn,
-		   zone->inactive_ratio);
+		   zone->zone_start_pfn);
 	seq_putc(m, '\n');
 }
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 1/21
#2 lore raw

Re: [PATCH v3 01/21] memcg: unify inactive_ratio calculation

Quoted 5 lines · On Thu, 23 Feb 2012 17:51:41 +0400 Konstantin Khlebnikov…

This patch removes precalculated zone->inactive_ratio.
Now it always calculated in inactive_anon_is_low() from current lru sizes. After that we can merge memcg and non-memcg cases and drop duplicated code.

seems good to me.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#3 lore raw

PATCH v3 2/21 memcg: make mm_match_cgroup() hirarchical

In reply to Konstantin Khlebnikov

Check mm-owner cgroup membership hierarchically.

---
 include/linux/memcontrol.h |   11 ++---------
 mm/memcontrol.c            |   20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 9 deletions(-)
include/linux/memcontrol.h +2 -9
index 8c4d74f..4822d53 100644
@@ -87,15 +87,8 @@ extern struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm);
 extern struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg);
 extern struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont);
 
-static inline
-int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
-{
-	struct mem_cgroup *memcg;
-	rcu_read_lock();
-	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
-	rcu_read_unlock();
-	return cgroup == memcg;
-}
+extern int mm_match_cgroup(const struct mm_struct *mm,
+			   const struct mem_cgroup *cgroup);
 
 extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg);
 
mm/memcontrol.c +20 -0
index b8039d2..77f5d48 100644
@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 				struct mem_cgroup, css);
 }
 
+/**
+ * mm_match_cgroup - cgroup hierarchy mm membership test
+ * @mm		mm_struct to test
+ * @cgroup	target cgroup
+ *
+ * Returns true if mm belong this cgroup or any its child in hierarchy
+ */
+int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
+{
+	struct mem_cgroup *memcg;
+
+	rcu_read_lock();
+	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
+	while (memcg != cgroup && memcg && memcg->use_hierarchy)
+		memcg = parent_mem_cgroup(memcg);
+	rcu_read_unlock();
+
+	return cgroup == memcg;
+}
+
 struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
 {
 	struct mem_cgroup *memcg = NULL;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

maintainer
#4 lore raw

Re: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical

Quoted 1 line · Konstantin Khlebnikov

Check mm-owner cgroup membership hierarchically.

I think this one cat just beat up this other cat in front of my window, yelling something about money and missing product. Anyway, I already forgot why we want this patch. Could you describe that in the changelog, please?

Quoted 13 lines
mm/memcontrol.c +9 -0
@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 				struct mem_cgroup, css);
 }
 
+/**
+ * mm_match_cgroup - cgroup hierarchy mm membership test
+ * @mm		mm_struct to test
+ * @cgroup	target cgroup
+ *
+ * Returns true if mm belong this cgroup or any its child in hierarchy
+ */
+int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
+{

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 842

Quoted 10 lines
mm/memcontrol.c +10 -0
+	struct mem_cgroup *memcg;
+
+	rcu_read_lock();
+	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
+	while (memcg != cgroup && memcg && memcg->use_hierarchy)
+		memcg = parent_mem_cgroup(memcg);
+	rcu_read_unlock();
+
+	return cgroup == memcg;
+}

Please don't duplicate mem_cgroup_same_or_subtree()'s functionality in a worse way. The hierarchy information is kept in a stack such that ancestry can be detected in linear time, check out css_is_ancestor().

If you don't want to nest rcu_read_lock(), you could push the rcu_read_lock() from css_is_ancestor() into its sole user and provide a __mem_cgroup_is_ancestor() that assumes rcu already read-locked.

No?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#5 lore raw

Re: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical

Quoted 7 lines · Johannes Weiner
Quoted 1 line · Konstantin Khlebnikov

Check mm-owner cgroup membership hierarchically.

I think this one cat just beat up this other cat in front of my window, yelling something about money and missing product. Anyway, I already forgot why we want this patch. Could you describe that in the changelog, please?

Yeah, sorry for lack of comment.

This test is used in rmap walker at checling page referencies in reclaimer. Memory cgroup shrinker want to skip all referencies outside of cgroup hierarchy which is currently under reclaim.

Actually this patch does not important for this set and can be dropped without problems, it does not share any context with other patches. Next patch is more important because it fixes global reclaimer and required for further cleanups.

Quoted 20 lines
Quoted 20 lines
mm/memcontrol.c +15 -0
@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
  				struct mem_cgroup, css);
  }
 
+/**
+ * mm_match_cgroup - cgroup hierarchy mm membership test
+ * @mm		mm_struct to test
+ * @cgroup	target cgroup
+ *
+ * Returns true if mm belong this cgroup or any its child in hierarchy
+ */
+int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
+{
+	struct mem_cgroup *memcg;
+
+	rcu_read_lock();
+	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
+	while (memcg != cgroup&&  memcg&&  memcg->use_hierarchy)
+		memcg = parent_mem_cgroup(memcg);
Quoted 8 lines

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 842

Quoted 4 lines
mm/memcontrol.c +4 -0
+	rcu_read_unlock();
+
+	return cgroup == memcg;
+}

Please don't duplicate mem_cgroup_same_or_subtree()'s functionality in a worse way. The hierarchy information is kept in a stack such that ancestry can be detected in linear time, check out css_is_ancestor().

Ok, there will be something like that:

+bool mm_match_cgroup(const struct mm_struct *mm,
+ const struct mem_cgroup *cgroup)
+{
+ struct mem_cgroup *memcg;
+ bool ret;
+

+       rcu_read_lock();
+       memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
+       ret = memcg && mem_cgroup_same_or_subtree(cgroup, memcg);
+       rcu_read_unlock();

+
+ return ret;
+}
+

Quoted 6 lines

If you don't want to nest rcu_read_lock(), you could push the rcu_read_lock() from css_is_ancestor() into its sole user and provide a __mem_cgroup_is_ancestor() that assumes rcu already read-locked.

No?

It is not a problem.

looks like mem_cgroup_same_or_subtree() check something different, because it does not check ->use_hierarchy flag on tested cgroup, only on target cgroup.

Or just all this hierarchical stuff is out of sync in different parts of code. For example memcg_get_hierarchical_limit() start from deepest cgroup and go upper while ->use_hierarchy is set.

Quoted 7 lines

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email:<a href=mailto:"[email protected]"> [email protected]</a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

maintainer
#6 lore raw

Re: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical

Quoted 36 lines · Konstantin Khlebnikov
Quoted 20 lines · Johannes Weiner
Quoted 19 lines · Konstantin Khlebnikov
mm/memcontrol.c +15 -0
@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 				struct mem_cgroup, css);
 }
 
+/**
+ * mm_match_cgroup - cgroup hierarchy mm membership test
+ * @mm		mm_struct to test
+ * @cgroup	target cgroup
+ *
+ * Returns true if mm belong this cgroup or any its child in hierarchy
+ */
+int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
+{
+	struct mem_cgroup *memcg;
+
+	rcu_read_lock();
+	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
+	while (memcg != cgroup&&  memcg&&  memcg->use_hierarchy)
+		memcg = parent_mem_cgroup(memcg);
Quoted 8 lines

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 842

Quoted 4 lines
mm/memcontrol.c +4 -0
+	rcu_read_unlock();
+
+	return cgroup == memcg;
+}

Please don't duplicate mem_cgroup_same_or_subtree()'s functionality in a worse way. The hierarchy information is kept in a stack such that ancestry can be detected in linear time, check out css_is_ancestor().

Ok, there will be something like that:

mm/memcontrol.c +4 -0
+bool mm_match_cgroup(const struct mm_struct *mm,
+                    const struct mem_cgroup *cgroup)
+{
+       struct mem_cgroup *memcg;

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 843

Quoted 10 lines
mm/memcontrol.c +10 -0
+       bool ret;
+
+       rcu_read_lock();
+       memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
+       ret = memcg && mem_cgroup_same_or_subtree(cgroup, memcg);
+       rcu_read_unlock();
+
+       return ret;
+}
+

It would be unfortunate to nest rcu_read_lock(), but I think this looks good otherwise.

Quoted 10 lines
Quoted 5 lines

If you don't want to nest rcu_read_lock(), you could push the rcu_read_lock() from css_is_ancestor() into its sole user and provide a __mem_cgroup_is_ancestor() that assumes rcu already read-locked.

No?

It is not a problem.

looks like mem_cgroup_same_or_subtree() check something different, because it does not check ->use_hierarchy flag on tested cgroup, only on target cgroup.

If a memcg has hierarchy enabled, any memcg that turns out to be its child is guaranteed to have hierarchy enabled.

Quoted 3 lines

Or just all this hierarchical stuff is out of sync in different parts of code. For example memcg_get_hierarchical_limit() start from deepest cgroup and go upper while ->use_hierarchy is set.

This one really has to walk up and find the smallest applicable limit, there is no way around looking at every single level.

But checking for ancestry can and has been optimized.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#7 lore raw

Re: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical

In reply to Konstantin Khlebnikov
Quoted 3 lines · On Thu, 23 Feb 2012 17:51:46 +0400 Konstantin Khlebnikov…

Check mm-owner cgroup membership hierarchically.

Ack. but ... see below.

Quoted 32 lines
---
 include/linux/memcontrol.h |   11 ++---------
 mm/memcontrol.c            |   20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 9 deletions(-)
include/linux/memcontrol.h +2 -9
index 8c4d74f..4822d53 100644
@@ -87,15 +87,8 @@ extern struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm);
 extern struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg);
 extern struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont);
 
-static inline
-int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
-{
-	struct mem_cgroup *memcg;
-	rcu_read_lock();
-	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
-	rcu_read_unlock();
-	return cgroup == memcg;
-}
+extern int mm_match_cgroup(const struct mm_struct *mm,
+			   const struct mem_cgroup *cgroup);
 
 extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg);
 
mm/memcontrol.c +0 -0
index b8039d2..77f5d48 100644

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 829

Quoted 9 lines
mm/memcontrol.c +6 -0
@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 				struct mem_cgroup, css);
 }
 
+/**
+ * mm_match_cgroup - cgroup hierarchy mm membership test
+ * @mm		mm_struct to test
+ * @cgroup	target cgroup
+ *
+ * Returns true if mm belong this cgroup or any its child in hierarchy

belongs to ?

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 832

Quoted 3 lines
mm/memcontrol.c +3 -0
+ */
+int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
+{

Please use "memcg" for representing "memory cgroup" (other function's arguments uses "memcg")

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 833

Quoted 1 line
mm/memcontrol.c +1 -0
+	struct mem_cgroup *memcg;

So, rename this as *cur_memcg or some.

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 838

Quoted 5 lines
mm/memcontrol.c +5 -0
+
+	rcu_read_lock();
+	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
+	while (memcg != cgroup && memcg && memcg->use_hierarchy)
+		memcg = parent_mem_cgroup(memcg);

IIUC, parent_mem_cgroup() checks mem->res.parent. mem->res.parent is set only when parent->use_hierarchy == true. Then,

	while (memcg != cgroup)
		memcg = parent_mem_cgroup(memcg);

will be enough.

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#8 lore raw

Re: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical

Quoted 73 lines · KAMEZAWA Hiroyuki
Quoted 3 lines · On Thu, 23 Feb 2012 17:51:46 +0400 Konstantin…

Check mm-owner cgroup membership hierarchically.

Ack. but ... see below.

Quoted 32 lines
---
  include/linux/memcontrol.h |   11 ++---------
  mm/memcontrol.c            |   20 ++++++++++++++++++++
  2 files changed, 22 insertions(+), 9 deletions(-)
include/linux/memcontrol.h +2 -9
index 8c4d74f..4822d53 100644
@@ -87,15 +87,8 @@ extern struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm);
  extern struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg);
  extern struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont);
 
-static inline
-int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
-{
-	struct mem_cgroup *memcg;
-	rcu_read_lock();
-	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
-	rcu_read_unlock();
-	return cgroup == memcg;
-}
+extern int mm_match_cgroup(const struct mm_struct *mm,
+			   const struct mem_cgroup *cgroup);
 
  extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg);
 
mm/memcontrol.c +0 -0
index b8039d2..77f5d48 100644

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 829

Quoted 9 lines
mm/memcontrol.c +6 -0
@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
  				struct mem_cgroup, css);
  }
 
+/**
+ * mm_match_cgroup - cgroup hierarchy mm membership test
+ * @mm		mm_struct to test
+ * @cgroup	target cgroup
+ *
+ * Returns true if mm belong this cgroup or any its child in hierarchy

belongs to ?

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 832

Quoted 3 lines
mm/memcontrol.c +3 -0
+ */
+int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
+{

Please use "memcg" for representing "memory cgroup" (other function's arguments uses "memcg")

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 833

Quoted 1 line
mm/memcontrol.c +1 -0
+	struct mem_cgroup *memcg;

So, rename this as *cur_memcg or some.

mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 838

Quoted 5 lines
mm/memcontrol.c +5 -0
+
+	rcu_read_lock();
+	memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner));
+	while (memcg != cgroup&&  memcg&&  memcg->use_hierarchy)
+		memcg = parent_mem_cgroup(memcg);

IIUC, parent_mem_cgroup() checks mem->res.parent. mem->res.parent is set only when parent->use_hierarchy == true. Then,

Quoted 4 lines
	while (memcg != cgroup)
		memcg = parent_mem_cgroup(memcg);

will be enough.

Here will be mem_cgroup_same_or_subtree(), see reply from Johannes Weiner.

Quoted 4 lines

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#9 lore raw

PATCH v3 3/21 memcg: fix page_referencies cgroup filter on global reclaim

In reply to Konstantin Khlebnikov

Global memory reclaimer shouldn't skip any page referencies.

This patch pass sc->target_mem_cgroup into page_referenced().
On global memory reclaim it always NULL, so we will account all. Cgroup reclaimer will account only referencies from target cgroup and its childs.

---
 mm/vmscan.c |    9 +++++----
 1 files changed, 5 insertions(+), 4 deletions(-)
mm/vmscan.c +5 -4
index 39aa4d7..d133ac6 100644
@@ -701,13 +701,13 @@ enum page_references {
 };
 
 static enum page_references page_check_references(struct page *page,
-						  struct mem_cgroup_zone *mz,
 						  struct scan_control *sc)
 {
 	int referenced_ptes, referenced_page;
 	unsigned long vm_flags;
 
-	referenced_ptes = page_referenced(page, 1, mz->mem_cgroup, &vm_flags);
+	referenced_ptes = page_referenced(page, 1,
+					  sc->target_mem_cgroup, &vm_flags);
 	referenced_page = TestClearPageReferenced(page);
 
 	/* Lumpy reclaim - ignore references */
@@ -828,7 +828,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			}
 		}
 
-		references = page_check_references(page, mz, sc);
+		references = page_check_references(page, sc);
 		switch (references) {
 		case PAGEREF_ACTIVATE:
 			goto activate_locked;
@@ -1735,7 +1735,8 @@ static void shrink_active_list(unsigned long nr_to_scan,
 			continue;
 		}
 
-		if (page_referenced(page, 0, mz->mem_cgroup, &vm_flags)) {
+		if (page_referenced(page, 0,
+				    sc->target_mem_cgroup, &vm_flags)) {
 			nr_rotated += hpage_nr_pages(page);
 			/*
 			 * Identify referenced, file-backed active pages and

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 3/21
#10 lore raw

Re: [PATCH v3 03/21] memcg: fix page_referencies cgroup filter on global reclaim

Quoted 7 lines · On Thu, 23 Feb 2012 17:51:51 +0400 Konstantin Khlebnikov…

Global memory reclaimer shouldn't skip any page referencies.

This patch pass sc->target_mem_cgroup into page_referenced().
On global memory reclaim it always NULL, so we will account all. Cgroup reclaimer will account only referencies from target cgroup and its childs.

seems nice to me.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#11 lore raw

PATCH v3 4/21 memcg: use vm_swappiness from target memory cgroup

In reply to Konstantin Khlebnikov

Use vm_swappiness from memory cgroup which is triggered this memory reclaim. This is more reasonable and allows to kill one argument.

---
 mm/vmscan.c |    9 ++++-----
 1 files changed, 4 insertions(+), 5 deletions(-)
mm/vmscan.c +4 -5
index d133ac6..8b59cb5 100644
@@ -1903,12 +1903,11 @@ static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
 	return shrink_inactive_list(nr_to_scan, mz, sc, priority, file);
 }
 
-static int vmscan_swappiness(struct mem_cgroup_zone *mz,
-			     struct scan_control *sc)
+static int vmscan_swappiness(struct scan_control *sc)
 {
 	if (global_reclaim(sc))
 		return vm_swappiness;
-	return mem_cgroup_swappiness(mz->mem_cgroup);
+	return mem_cgroup_swappiness(sc->target_mem_cgroup);
 }
 
 /*
@@ -1976,8 +1975,8 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,
 	 * With swappiness at 100, anonymous and file have the same priority.
 	 * This scanning priority is essentially the inverse of IO cost.
 	 */
-	anon_prio = vmscan_swappiness(mz, sc);
-	file_prio = 200 - vmscan_swappiness(mz, sc);
+	anon_prio = vmscan_swappiness(sc);
+	file_prio = 200 - vmscan_swappiness(sc);
 
 	/*
 	 * OK, so we have swap space and a fair amount of page cache

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 4/21
#12 lore raw

Re: [PATCH v3 04/21] memcg: use vm_swappiness from target memory cgroup

Quoted 4 lines · On Thu, 23 Feb 2012 17:51:56 +0400 Konstantin Khlebnikov…

Use vm_swappiness from memory cgroup which is triggered this memory reclaim. This is more reasonable and allows to kill one argument.

Hmm... it may be better to disallow to have different swappiness in a hierarchy..

Quoted 1 line

From me,

But I wonder other guys may have different idea.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#13 lore raw

PATCH v3 5/21 mm: rename lruvec->lists into lruvec->pages_lru

In reply to Konstantin Khlebnikov

This is much more unique and grep-friendly name.

---
 include/linux/mm_inline.h |    2 +-
 include/linux/mmzone.h    |    2 +-
 mm/memcontrol.c           |    6 +++---
 mm/page_alloc.c           |    2 +-
 mm/swap.c                 |    4 ++--
 mm/vmscan.c               |    6 +++---
 6 files changed, 11 insertions(+), 11 deletions(-)
include/linux/mm_inline.h +1 -1
index 227fd3e..8415596 100644
@@ -27,7 +27,7 @@ add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
 	struct lruvec *lruvec;
 
 	lruvec = mem_cgroup_lru_add_list(zone, page, lru);
-	list_add(&page->lru, &lruvec->lists[lru]);
+	list_add(&page->lru, &lruvec->pages_lru[lru]);
 	__mod_zone_page_state(zone, NR_LRU_BASE + lru, hpage_nr_pages(page));
 }
 
include/linux/mmzone.h +1 -1
index 3e1f7ff..ddd0fd2 100644
@@ -160,7 +160,7 @@ static inline int is_unevictable_lru(enum lru_list lru)
 }
 
 struct lruvec {
-	struct list_head lists[NR_LRU_LISTS];
+	struct list_head pages_lru[NR_LRU_LISTS];
 };
 
 /* Mask used at gathering information at once (see memcontrol.c) */
mm/memcontrol.c +3 -3
index 77f5d48..8f8c7c4 100644
@@ -1050,7 +1050,7 @@ struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
  * the lruvec for the given @zone and the memcg @page is charged to.
  *
  * The callsite is then responsible for physically linking the page to
- * the returned lruvec->lists[@lru].
+ * the returned lruvec->pages_lru[@lru].
  */
 struct lruvec *mem_cgroup_lru_add_list(struct zone *zone, struct page *page,
 				       enum lru_list lru)
@@ -3592,7 +3592,7 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
 
 	zone = &NODE_DATA(node)->node_zones[zid];
 	mz = mem_cgroup_zoneinfo(memcg, node, zid);
-	list = &mz->lruvec.lists[lru];
+	list = &mz->lruvec.pages_lru[lru];
 
 	loop = mz->lru_size[lru];
 	/* give some margin against EBUSY etc...*/
@@ -4716,7 +4716,7 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
 		mz = &pn->zoneinfo[zone];
 		for_each_lru(lru)
-			INIT_LIST_HEAD(&mz->lruvec.lists[lru]);
+			INIT_LIST_HEAD(&mz->lruvec.pages_lru[lru]);
 		mz->usage_in_excess = 0;
 		mz->on_tree = false;
 		mz->memcg = memcg;
mm/page_alloc.c +1 -1
index 38f6744..5f19392 100644
@@ -4363,7 +4363,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 
 		zone_pcp_init(zone);
 		for_each_lru(lru)
-			INIT_LIST_HEAD(&zone->lruvec.lists[lru]);
+			INIT_LIST_HEAD(&zone->lruvec.pages_lru[lru]);
 		zone->reclaim_stat.recent_rotated[0] = 0;
 		zone->reclaim_stat.recent_rotated[1] = 0;
 		zone->reclaim_stat.recent_scanned[0] = 0;
mm/swap.c +2 -2
index fff1ff7..17993c0 100644
@@ -238,7 +238,7 @@ static void pagevec_move_tail_fn(struct page *page, void *arg)
 
 		lruvec = mem_cgroup_lru_move_lists(page_zone(page),
 						   page, lru, lru);
-		list_move_tail(&page->lru, &lruvec->lists[lru]);
+		list_move_tail(&page->lru, &lruvec->pages_lru[lru]);
 		(*pgmoved)++;
 	}
 }
@@ -482,7 +482,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 		 * We moves tha page into tail of inactive.
 		 */
 		lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
-		list_move_tail(&page->lru, &lruvec->lists[lru]);
+		list_move_tail(&page->lru, &lruvec->pages_lru[lru]);
 		__count_vm_event(PGROTATED);
 	}
 
mm/vmscan.c +3 -3
index 8b59cb5..e41ad52 100644
@@ -1164,7 +1164,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 		lru += LRU_ACTIVE;
 	if (file)
 		lru += LRU_FILE;
-	src = &lruvec->lists[lru];
+	src = &lruvec->pages_lru[lru];
 
 	for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
 		struct page *page;
@@ -1663,7 +1663,7 @@ static void move_active_pages_to_lru(struct zone *zone,
 		SetPageLRU(page);
 
 		lruvec = mem_cgroup_lru_add_list(zone, page, lru);
-		list_move(&page->lru, &lruvec->lists[lru]);
+		list_move(&page->lru, &lruvec->pages_lru[lru]);
 		pgmoved += hpage_nr_pages(page);
 
 		if (put_page_testzero(page)) {
@@ -3592,7 +3592,7 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
 			__dec_zone_state(zone, NR_UNEVICTABLE);
 			lruvec = mem_cgroup_lru_move_lists(zone, page,
 						LRU_UNEVICTABLE, lru);
-			list_move(&page->lru, &lruvec->lists[lru]);
+			list_move(&page->lru, &lruvec->pages_lru[lru]);
 			__inc_zone_state(zone, NR_INACTIVE_ANON + lru);
 			pgrescued++;
 		}

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Reviewed-by on v3 5/21
#14 lore raw

Re: [PATCH v3 05/21] mm: rename lruvec->lists into lruvec->pages_lru

Quoted 3 lines · On Thu, 23 Feb 2012 17:52:00 +0400 Konstantin Khlebnikov…

This is much more unique and grep-friendly name.

I worries this kind of change can cause many hunks and make merging difficult.. But this seems not very destructive..

I have no strong opinions to this naming. How other mm developpers think ?

I personally think making this kind of changes in the head of patch set tend do make it difficult to merge full sets of patche series.

Thanks,
-Kame

Quoted 134 lines
---
 include/linux/mm_inline.h |    2 +-
 include/linux/mmzone.h    |    2 +-
 mm/memcontrol.c           |    6 +++---
 mm/page_alloc.c           |    2 +-
 mm/swap.c                 |    4 ++--
 mm/vmscan.c               |    6 +++---
 6 files changed, 11 insertions(+), 11 deletions(-)
include/linux/mm_inline.h +1 -1
index 227fd3e..8415596 100644
@@ -27,7 +27,7 @@ add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
 	struct lruvec *lruvec;
 
 	lruvec = mem_cgroup_lru_add_list(zone, page, lru);
-	list_add(&page->lru, &lruvec->lists[lru]);
+	list_add(&page->lru, &lruvec->pages_lru[lru]);
 	__mod_zone_page_state(zone, NR_LRU_BASE + lru, hpage_nr_pages(page));
 }
 
include/linux/mmzone.h +1 -1
index 3e1f7ff..ddd0fd2 100644
@@ -160,7 +160,7 @@ static inline int is_unevictable_lru(enum lru_list lru)
 }
 
 struct lruvec {
-	struct list_head lists[NR_LRU_LISTS];
+	struct list_head pages_lru[NR_LRU_LISTS];
 };
 
 /* Mask used at gathering information at once (see memcontrol.c) */
mm/memcontrol.c +3 -3
index 77f5d48..8f8c7c4 100644
@@ -1050,7 +1050,7 @@ struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
  * the lruvec for the given @zone and the memcg @page is charged to.
  *
  * The callsite is then responsible for physically linking the page to
- * the returned lruvec->lists[@lru].
+ * the returned lruvec->pages_lru[@lru].
  */
 struct lruvec *mem_cgroup_lru_add_list(struct zone *zone, struct page *page,
 				       enum lru_list lru)
@@ -3592,7 +3592,7 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
 
 	zone = &NODE_DATA(node)->node_zones[zid];
 	mz = mem_cgroup_zoneinfo(memcg, node, zid);
-	list = &mz->lruvec.lists[lru];
+	list = &mz->lruvec.pages_lru[lru];
 
 	loop = mz->lru_size[lru];
 	/* give some margin against EBUSY etc...*/
@@ -4716,7 +4716,7 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
 		mz = &pn->zoneinfo[zone];
 		for_each_lru(lru)
-			INIT_LIST_HEAD(&mz->lruvec.lists[lru]);
+			INIT_LIST_HEAD(&mz->lruvec.pages_lru[lru]);
 		mz->usage_in_excess = 0;
 		mz->on_tree = false;
 		mz->memcg = memcg;
mm/page_alloc.c +1 -1
index 38f6744..5f19392 100644
@@ -4363,7 +4363,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 
 		zone_pcp_init(zone);
 		for_each_lru(lru)
-			INIT_LIST_HEAD(&zone->lruvec.lists[lru]);
+			INIT_LIST_HEAD(&zone->lruvec.pages_lru[lru]);
 		zone->reclaim_stat.recent_rotated[0] = 0;
 		zone->reclaim_stat.recent_rotated[1] = 0;
 		zone->reclaim_stat.recent_scanned[0] = 0;
mm/swap.c +2 -2
index fff1ff7..17993c0 100644
@@ -238,7 +238,7 @@ static void pagevec_move_tail_fn(struct page *page, void *arg)
 
 		lruvec = mem_cgroup_lru_move_lists(page_zone(page),
 						   page, lru, lru);
-		list_move_tail(&page->lru, &lruvec->lists[lru]);
+		list_move_tail(&page->lru, &lruvec->pages_lru[lru]);
 		(*pgmoved)++;
 	}
 }
@@ -482,7 +482,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 		 * We moves tha page into tail of inactive.
 		 */
 		lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
-		list_move_tail(&page->lru, &lruvec->lists[lru]);
+		list_move_tail(&page->lru, &lruvec->pages_lru[lru]);
 		__count_vm_event(PGROTATED);
 	}
 
mm/vmscan.c +3 -4
index 8b59cb5..e41ad52 100644
@@ -1164,7 +1164,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 		lru += LRU_ACTIVE;
 	if (file)
 		lru += LRU_FILE;
-	src = &lruvec->lists[lru];
+	src = &lruvec->pages_lru[lru];
 
 	for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
 		struct page *page;
@@ -1663,7 +1663,7 @@ static void move_active_pages_to_lru(struct zone *zone,
 		SetPageLRU(page);
 
 		lruvec = mem_cgroup_lru_add_list(zone, page, lru);
-		list_move(&page->lru, &lruvec->lists[lru]);
+		list_move(&page->lru, &lruvec->pages_lru[lru]);
 		pgmoved += hpage_nr_pages(page);
 
 		if (put_page_testzero(page)) {
@@ -3592,7 +3592,7 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
 			__dec_zone_state(zone, NR_UNEVICTABLE);
 			lruvec = mem_cgroup_lru_move_lists(zone, page,
 						LRU_UNEVICTABLE, lru);
-			list_move(&page->lru, &lruvec->lists[lru]);
+			list_move(&page->lru, &lruvec->pages_lru[lru]);
 			__inc_zone_state(zone, NR_INACTIVE_ANON + lru);
 			pgrescued++;
 		}
 
--
Quoted 5 lines

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#15 lore raw

Re: [PATCH v3 05/21] mm: rename lruvec->lists into lruvec->pages_lru

Quoted 10 lines · KAMEZAWA Hiroyuki
Quoted 3 lines · On Thu, 23 Feb 2012 17:52:00 +0400 Konstantin…

This is much more unique and grep-friendly name.

I worries this kind of change can cause many hunks and make merging difficult.. But this seems not very destructive..

Quoted 6 lines

I have no strong opinions to this naming. How other mm developpers think ?

I personally think making this kind of changes in the head of patch set tend do make it difficult to merge full sets of patche series.

This rename allows to easily highlight lru-lists manipulations, because some of them are in non-trivial places.

Quoted 137 lines

Thanks,
-Kame

Quoted 133 lines
---
  include/linux/mm_inline.h |    2 +-
  include/linux/mmzone.h    |    2 +-
  mm/memcontrol.c           |    6 +++---
  mm/page_alloc.c           |    2 +-
  mm/swap.c                 |    4 ++--
  mm/vmscan.c               |    6 +++---
  6 files changed, 11 insertions(+), 11 deletions(-)
include/linux/mm_inline.h +1 -1
index 227fd3e..8415596 100644
@@ -27,7 +27,7 @@ add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
  	struct lruvec *lruvec;
 
  	lruvec = mem_cgroup_lru_add_list(zone, page, lru);
-	list_add(&page->lru,&lruvec->lists[lru]);
+	list_add(&page->lru,&lruvec->pages_lru[lru]);
  	__mod_zone_page_state(zone, NR_LRU_BASE + lru, hpage_nr_pages(page));
  }
 
include/linux/mmzone.h +1 -1
index 3e1f7ff..ddd0fd2 100644
@@ -160,7 +160,7 @@ static inline int is_unevictable_lru(enum lru_list lru)
  }
 
  struct lruvec {
-	struct list_head lists[NR_LRU_LISTS];
+	struct list_head pages_lru[NR_LRU_LISTS];
  };
 
  /* Mask used at gathering information at once (see memcontrol.c) */
mm/memcontrol.c +3 -3
index 77f5d48..8f8c7c4 100644
@@ -1050,7 +1050,7 @@ struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
   * the lruvec for the given @zone and the memcg @page is charged to.
   *
   * The callsite is then responsible for physically linking the page to
- * the returned lruvec->lists[@lru].
+ * the returned lruvec->pages_lru[@lru].
   */
  struct lruvec *mem_cgroup_lru_add_list(struct zone *zone, struct page *page,
  				       enum lru_list lru)
@@ -3592,7 +3592,7 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
 
  	zone =&NODE_DATA(node)->node_zones[zid];
  	mz = mem_cgroup_zoneinfo(memcg, node, zid);
-	list =&mz->lruvec.lists[lru];
+	list =&mz->lruvec.pages_lru[lru];
 
  	loop = mz->lru_size[lru];
  	/* give some margin against EBUSY etc...*/
@@ -4716,7 +4716,7 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
  	for (zone = 0; zone<  MAX_NR_ZONES; zone++) {
  		mz =&pn->zoneinfo[zone];
  		for_each_lru(lru)
-			INIT_LIST_HEAD(&mz->lruvec.lists[lru]);
+			INIT_LIST_HEAD(&mz->lruvec.pages_lru[lru]);
  		mz->usage_in_excess = 0;
  		mz->on_tree = false;
  		mz->memcg = memcg;
mm/page_alloc.c +1 -1
index 38f6744..5f19392 100644
@@ -4363,7 +4363,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 
  		zone_pcp_init(zone);
  		for_each_lru(lru)
-			INIT_LIST_HEAD(&zone->lruvec.lists[lru]);
+			INIT_LIST_HEAD(&zone->lruvec.pages_lru[lru]);
  		zone->reclaim_stat.recent_rotated[0] = 0;
  		zone->reclaim_stat.recent_rotated[1] = 0;
  		zone->reclaim_stat.recent_scanned[0] = 0;
mm/swap.c +2 -2
index fff1ff7..17993c0 100644
@@ -238,7 +238,7 @@ static void pagevec_move_tail_fn(struct page *page, void *arg)
 
  		lruvec = mem_cgroup_lru_move_lists(page_zone(page),
  						   page, lru, lru);
-		list_move_tail(&page->lru,&lruvec->lists[lru]);
+		list_move_tail(&page->lru,&lruvec->pages_lru[lru]);
  		(*pgmoved)++;
  	}
  }
@@ -482,7 +482,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)
  		 * We moves tha page into tail of inactive.
  		 */
  		lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
-		list_move_tail(&page->lru,&lruvec->lists[lru]);
+		list_move_tail(&page->lru,&lruvec->pages_lru[lru]);
  		__count_vm_event(PGROTATED);
  	}
 
mm/vmscan.c +3 -3
index 8b59cb5..e41ad52 100644
@@ -1164,7 +1164,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
  		lru += LRU_ACTIVE;
  	if (file)
  		lru += LRU_FILE;
-	src =&lruvec->lists[lru];
+	src =&lruvec->pages_lru[lru];
 
  	for (scan = 0; scan<  nr_to_scan&&  !list_empty(src); scan++) {
  		struct page *page;
@@ -1663,7 +1663,7 @@ static void move_active_pages_to_lru(struct zone *zone,
  		SetPageLRU(page);
 
  		lruvec = mem_cgroup_lru_add_list(zone, page, lru);
-		list_move(&page->lru,&lruvec->lists[lru]);
+		list_move(&page->lru,&lruvec->pages_lru[lru]);
  		pgmoved += hpage_nr_pages(page);
 
  		if (put_page_testzero(page)) {
@@ -3592,7 +3592,7 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
  			__dec_zone_state(zone, NR_UNEVICTABLE);
  			lruvec = mem_cgroup_lru_move_lists(zone, page,
  						LRU_UNEVICTABLE, lru);
-			list_move(&page->lru,&lruvec->lists[lru]);
+			list_move(&page->lru,&lruvec->pages_lru[lru]);
  			__inc_zone_state(zone, NR_INACTIVE_ANON + lru);
  			pgrescued++;
  		}
Quoted 7 lines
Quoted 6 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#16 lore raw

PATCH v3 6/21 mm: lruvec linking functions

In reply to Konstantin Khlebnikov

This patch adds links from page to its lruvec and from lruvec to its zone and node. If CONFIG_CGROUP_MEM_RES_CTLR=n they just page_zone() and container_of().

---
 include/linux/mm.h     |   37 +++++++++++++++++++++++++++++++++++++
 include/linux/mmzone.h |   12 ++++++++----
 mm/internal.h          |    1 +
 mm/memcontrol.c        |   27 ++++++++++++++++++++++++---
 mm/page_alloc.c        |   17 ++++++++++++++---
 5 files changed, 84 insertions(+), 10 deletions(-)
include/linux/mm.h +37 -0
index ee3ebc1..c6dc4ab 100644
@@ -728,6 +728,43 @@ static inline void set_page_links(struct page *page, enum zone_type zone,
 #endif
 }
 
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+
+/* Multiple lruvecs in zone */
+
+extern struct lruvec *page_lruvec(struct page *page);
+
+static inline struct zone *lruvec_zone(struct lruvec *lruvec)
+{
+	return lruvec->zone;
+}
+
+static inline struct pglist_data *lruvec_node(struct lruvec *lruvec)
+{
+	return lruvec->node;
+}
+
+#else /* CONFIG_CGROUP_MEM_RES_CTLR */
+
+/* Single lruvec in zone */
+
+static inline struct lruvec *page_lruvec(struct page *page)
+{
+	return &page_zone(page)->lruvec;
+}
+
+static inline struct zone *lruvec_zone(struct lruvec *lruvec)
+{
+	return container_of(lruvec, struct zone, lruvec);
+}
+
+static inline struct pglist_data *lruvec_node(struct lruvec *lruvec)
+{
+	return lruvec_zone(lruvec)->zone_pgdat;
+}
+
+#endif /* CONFIG_CGROUP_MEM_RES_CTLR */
+
 /*
  * Some inline functions in vmstat.h depend on page_zone()
  */
include/linux/mmzone.h +8 -4
index ddd0fd2..be8873a 100644
@@ -159,10 +159,6 @@ static inline int is_unevictable_lru(enum lru_list lru)
 	return (lru == LRU_UNEVICTABLE);
 }
 
-struct lruvec {
-	struct list_head pages_lru[NR_LRU_LISTS];
-};
-
 /* Mask used at gathering information at once (see memcontrol.c) */
 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
@@ -300,6 +296,14 @@ struct zone_reclaim_stat {
 	unsigned long		recent_scanned[2];
 };
 
+struct lruvec {
+	struct list_head	pages_lru[NR_LRU_LISTS];
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+	struct zone		*zone;
+	struct pglist_data	*node;
+#endif
+};
+
 struct zone {
 	/* Fields commonly accessed by the page allocator */
 
mm/internal.h +1 -0
index 2189af4..ef49dbf 100644
@@ -100,6 +100,7 @@ extern void prep_compound_page(struct page *page, unsigned long order);
 extern bool is_free_buddy_page(struct page *page);
 #endif
 
+extern void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec);
 
 /*
  * function for dealing with page's order in buddy system.
mm/memcontrol.c +24 -3
index 8f8c7c4..8b53150 100644
@@ -1026,6 +1026,28 @@ struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
 	return &mz->lruvec;
 }
 
+/**
+ * page_lruvec - get the lruvec there this page is located
+ * @page: the struct page pointer with stable reference
+ *
+ * page_cgroup->mem_cgroup pointer validity guaranteed by caller.
+ *
+ * Returns pointer to struct lruvec.
+ */
+struct lruvec *page_lruvec(struct page *page)
+{
+	struct mem_cgroup_per_zone *mz;
+	struct page_cgroup *pc;
+
+	if (mem_cgroup_disabled())
+		return &page_zone(page)->lruvec;
+
+	pc = lookup_page_cgroup(page);
+	mz = mem_cgroup_zoneinfo(pc->mem_cgroup,
+			page_to_nid(page), page_zonenum(page));
+	return &mz->lruvec;
+}
+
 /*
  * Following LRU functions are allowed to be used without PCG_LOCK.
  * Operations are called by routine of global LRU independently from memcg.
@@ -4697,7 +4719,6 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 {
 	struct mem_cgroup_per_node *pn;
 	struct mem_cgroup_per_zone *mz;
-	enum lru_list lru;
 	int zone, tmp = node;
 	/*
 	 * This routine is called against possible nodes.
@@ -4715,8 +4736,8 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 
 	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
 		mz = &pn->zoneinfo[zone];
-		for_each_lru(lru)
-			INIT_LIST_HEAD(&mz->lruvec.pages_lru[lru]);
+		init_zone_lruvec(&NODE_DATA(node)->node_zones[zone],
+				 &mz->lruvec);
 		mz->usage_in_excess = 0;
 		mz->on_tree = false;
 		mz->memcg = memcg;
mm/page_alloc.c +14 -3
index 5f19392..1cc3afe 100644
@@ -4289,6 +4289,19 @@ static inline int pageblock_default_order(unsigned int order)
 
 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
 
+void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)
+{
+	enum lru_list lru;
+
+	memset(lruvec, 0, sizeof(struct lruvec));
+	for_each_lru(lru)
+		INIT_LIST_HEAD(&lruvec->pages_lru[lru]);
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+	lruvec->node = zone->zone_pgdat;
+	lruvec->zone = zone;
+#endif
+}
+
 /*
  * Set up the zone data structures:
  *   - mark all pages reserved
@@ -4312,7 +4325,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 	for (j = 0; j < MAX_NR_ZONES; j++) {
 		struct zone *zone = pgdat->node_zones + j;
 		unsigned long size, realsize, memmap_pages;
-		enum lru_list lru;
 
 		size = zone_spanned_pages_in_node(nid, j, zones_size);
 		realsize = size - zone_absent_pages_in_node(nid, j,
@@ -4362,8 +4374,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 		zone->zone_pgdat = pgdat;
 
 		zone_pcp_init(zone);
-		for_each_lru(lru)
-			INIT_LIST_HEAD(&zone->lruvec.pages_lru[lru]);
+		init_zone_lruvec(zone, &zone->lruvec);
 		zone->reclaim_stat.recent_rotated[0] = 0;
 		zone->reclaim_stat.recent_rotated[1] = 0;
 		zone->reclaim_stat.recent_scanned[0] = 0;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#17 lore raw

Re: [PATCH v3 06/21] mm: lruvec linking functions

Quoted 4 lines · On Thu, 23 Feb 2012 17:52:04 +0400 Konstantin Khlebnikov…

This patch adds links from page to its lruvec and from lruvec to its zone and node. If CONFIG_CGROUP_MEM_RES_CTLR=n they just page_zone() and container_of().

small comments in below.

Quoted 72 lines
---
 include/linux/mm.h     |   37 +++++++++++++++++++++++++++++++++++++
 include/linux/mmzone.h |   12 ++++++++----
 mm/internal.h          |    1 +
 mm/memcontrol.c        |   27 ++++++++++++++++++++++++---
 mm/page_alloc.c        |   17 ++++++++++++++---
 5 files changed, 84 insertions(+), 10 deletions(-)
include/linux/mm.h +37 -0
index ee3ebc1..c6dc4ab 100644
@@ -728,6 +728,43 @@ static inline void set_page_links(struct page *page, enum zone_type zone,
 #endif
 }
 
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+
+/* Multiple lruvecs in zone */
+
+extern struct lruvec *page_lruvec(struct page *page);
+
+static inline struct zone *lruvec_zone(struct lruvec *lruvec)
+{
+	return lruvec->zone;
+}
+
+static inline struct pglist_data *lruvec_node(struct lruvec *lruvec)
+{
+	return lruvec->node;
+}
+
+#else /* CONFIG_CGROUP_MEM_RES_CTLR */
+
+/* Single lruvec in zone */
+
+static inline struct lruvec *page_lruvec(struct page *page)
+{
+	return &page_zone(page)->lruvec;
+}
+
+static inline struct zone *lruvec_zone(struct lruvec *lruvec)
+{
+	return container_of(lruvec, struct zone, lruvec);
+}
+
+static inline struct pglist_data *lruvec_node(struct lruvec *lruvec)
+{
+	return lruvec_zone(lruvec)->zone_pgdat;
+}
+
+#endif /* CONFIG_CGROUP_MEM_RES_CTLR */
+
 /*
  * Some inline functions in vmstat.h depend on page_zone()
  */
include/linux/mmzone.h +0 -4
index ddd0fd2..be8873a 100644
@@ -159,10 +159,6 @@ static inline int is_unevictable_lru(enum lru_list lru)
 	return (lru == LRU_UNEVICTABLE);
 }
 
-struct lruvec {
-	struct list_head pages_lru[NR_LRU_LISTS];
-};
-
 /* Mask used at gathering information at once (see memcontrol.c) */
 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))

include/linux/mmzone.h struct zone_reclaim_stat { · line 304

Quoted 9 lines
include/linux/mmzone.h +6 -0
@@ -300,6 +296,14 @@ struct zone_reclaim_stat {
 	unsigned long		recent_scanned[2];
 };
 
+struct lruvec {
+	struct list_head	pages_lru[NR_LRU_LISTS];
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+	struct zone		*zone;
+	struct pglist_data	*node;
+#endif

I don't think this #ifdef is very good ....this adds other #ifdefs in other headers. How bad if we remove this #ifdef and use ->zone, ->pgdat in lruvec_zone, lruvec_page always ?

There may be concerns to fit lruvec at el into cache-line...but this set will add a (big) hash here later..

I'm sorry if you're asked to add this #ifdef in v1 or v2.

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#18 lore raw

Re: [PATCH v3 06/21] mm: lruvec linking functions

Quoted 85 lines · KAMEZAWA Hiroyuki
Quoted 4 lines · On Thu, 23 Feb 2012 17:52:04 +0400 Konstantin…

This patch adds links from page to its lruvec and from lruvec to its zone and node. If CONFIG_CGROUP_MEM_RES_CTLR=n they just page_zone() and container_of().

small comments in below.

Quoted 75 lines
---
  include/linux/mm.h     |   37 +++++++++++++++++++++++++++++++++++++
  include/linux/mmzone.h |   12 ++++++++----
  mm/internal.h          |    1 +
  mm/memcontrol.c        |   27 ++++++++++++++++++++++++---
  mm/page_alloc.c        |   17 ++++++++++++++---
  5 files changed, 84 insertions(+), 10 deletions(-)
include/linux/mm.h +37 -0
index ee3ebc1..c6dc4ab 100644
@@ -728,6 +728,43 @@ static inline void set_page_links(struct page *page, enum zone_type zone,
  #endif
  }
 
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+
+/* Multiple lruvecs in zone */
+
+extern struct lruvec *page_lruvec(struct page *page);
+
+static inline struct zone *lruvec_zone(struct lruvec *lruvec)
+{
+	return lruvec->zone;
+}
+
+static inline struct pglist_data *lruvec_node(struct lruvec *lruvec)
+{
+	return lruvec->node;
+}
+
+#else /* CONFIG_CGROUP_MEM_RES_CTLR */
+
+/* Single lruvec in zone */
+
+static inline struct lruvec *page_lruvec(struct page *page)
+{
+	return&page_zone(page)->lruvec;
+}
+
+static inline struct zone *lruvec_zone(struct lruvec *lruvec)
+{
+	return container_of(lruvec, struct zone, lruvec);
+}
+
+static inline struct pglist_data *lruvec_node(struct lruvec *lruvec)
+{
+	return lruvec_zone(lruvec)->zone_pgdat;
+}
+
+#endif /* CONFIG_CGROUP_MEM_RES_CTLR */
+
  /*
   * Some inline functions in vmstat.h depend on page_zone()
   */
include/linux/mmzone.h +0 -4
index ddd0fd2..be8873a 100644
@@ -159,10 +159,6 @@ static inline int is_unevictable_lru(enum lru_list lru)
  	return (lru == LRU_UNEVICTABLE);
  }
 
-struct lruvec {
-	struct list_head pages_lru[NR_LRU_LISTS];
-};
-
  /* Mask used at gathering information at once (see memcontrol.c) */
  #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
  #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
@@ -300,6 +296,14 @@ struct zone_reclaim_stat {
  	unsigned long		recent_scanned[2];
  };
 
Quoted 10 lines

include/linux/mmzone.h struct zone_reclaim_stat { · line 304

Quoted 6 lines
include/linux/mmzone.h +6 -0
+struct lruvec {
+	struct list_head	pages_lru[NR_LRU_LISTS];
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+	struct zone		*zone;
+	struct pglist_data	*node;
+#endif

I don't think this #ifdef is very good ....this adds other #ifdefs in other headers. How bad if we remove this #ifdef and use ->zone, ->pgdat in lruvec_zone, lruvec_page always ?

This adds one dereference in lruvec_zone() if memcg is disabled in config.
We can remove ifdef from declaration and initialization, but keep optimized variant of lruvec_zone()

Quoted 9 lines

There may be concerns to fit lruvec at el into cache-line...but this set will add a (big) hash here later..

I'm sorry if you're asked to add this #ifdef in v1 or v2.

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#19 lore raw

PATCH v3 7/21 mm: add lruvec->pages_count

In reply to Konstantin Khlebnikov

Move lru pages counter from mem_cgroup_per_zone->count[] to lruvec->pages_count[]

Account pages in all lruvecs, incuding root,
this isn't a huge overhead, but it greatly simplifies all code.

Redundant page_lruvec() calls will be optimized in further patches.

---
 include/linux/memcontrol.h |   29 --------------
 include/linux/mm_inline.h  |   15 +++++--
 include/linux/mmzone.h     |    1 
 mm/memcontrol.c            |   93 +-------------------------------------------
 mm/swap.c                  |    7 +--
 mm/vmscan.c                |   25 +++++++++---
 6 files changed, 34 insertions(+), 136 deletions(-)
include/linux/memcontrol.h +0 -29
index 4822d53..b9d555b 100644
@@ -63,12 +63,6 @@ extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
 					gfp_t gfp_mask);
 
 struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
-struct lruvec *mem_cgroup_lru_add_list(struct zone *, struct page *,
-				       enum lru_list);
-void mem_cgroup_lru_del_list(struct page *, enum lru_list);
-void mem_cgroup_lru_del(struct page *);
-struct lruvec *mem_cgroup_lru_move_lists(struct zone *, struct page *,
-					 enum lru_list, enum lru_list);
 
 /* For coalescing uncharge for reducing memcg' overhead*/
 extern void mem_cgroup_uncharge_start(void);
@@ -212,29 +206,6 @@ static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
 	return &zone->lruvec;
 }
 
-static inline struct lruvec *mem_cgroup_lru_add_list(struct zone *zone,
-						     struct page *page,
-						     enum lru_list lru)
-{
-	return &zone->lruvec;
-}
-
-static inline void mem_cgroup_lru_del_list(struct page *page, enum lru_list lru)
-{
-}
-
-static inline void mem_cgroup_lru_del(struct page *page)
-{
-}
-
-static inline struct lruvec *mem_cgroup_lru_move_lists(struct zone *zone,
-						       struct page *page,
-						       enum lru_list from,
-						       enum lru_list to)
-{
-	return &zone->lruvec;
-}
-
 static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
 {
 	return NULL;
include/linux/mm_inline.h +10 -5
index 8415596..daa3d15 100644
@@ -24,19 +24,24 @@ static inline int page_is_file_cache(struct page *page)
 static inline void
 add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
 {
-	struct lruvec *lruvec;
+	struct lruvec *lruvec = page_lruvec(page);
+	int numpages = hpage_nr_pages(page);
 
-	lruvec = mem_cgroup_lru_add_list(zone, page, lru);
 	list_add(&page->lru, &lruvec->pages_lru[lru]);
-	__mod_zone_page_state(zone, NR_LRU_BASE + lru, hpage_nr_pages(page));
+	lruvec->pages_count[lru] += numpages;
+	__mod_zone_page_state(zone, NR_LRU_BASE + lru, numpages);
 }
 
 static inline void
 del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
 {
-	mem_cgroup_lru_del_list(page, lru);
+	struct lruvec *lruvec = page_lruvec(page);
+	int numpages = hpage_nr_pages(page);
+
 	list_del(&page->lru);
-	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -hpage_nr_pages(page));
+	lruvec->pages_count[lru] -= numpages;
+	VM_BUG_ON((long)lruvec->pages_count[lru] < 0);
+	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -numpages);
 }
 
 /**
include/linux/mmzone.h +1 -0
index be8873a..69b0f31 100644
@@ -298,6 +298,7 @@ struct zone_reclaim_stat {
 
 struct lruvec {
 	struct list_head	pages_lru[NR_LRU_LISTS];
+	unsigned long		pages_count[NR_LRU_LISTS];
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
 	struct zone		*zone;
 	struct pglist_data	*node;
mm/memcontrol.c +2 -91

117 lines not on this page · raw message

mm/swap.c +2 -5
index 17993c0..2afe02c 100644
@@ -234,10 +234,8 @@ static void pagevec_move_tail_fn(struct page *page, void *arg)
 
 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 		enum lru_list lru = page_lru_base_type(page);
-		struct lruvec *lruvec;
+		struct lruvec *lruvec = page_lruvec(page);
 
-		lruvec = mem_cgroup_lru_move_lists(page_zone(page),
-						   page, lru, lru);
 		list_move_tail(&page->lru, &lruvec->pages_lru[lru]);
 		(*pgmoved)++;
 	}
@@ -476,12 +474,11 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 		 */
 		SetPageReclaim(page);
 	} else {
-		struct lruvec *lruvec;
+		struct lruvec *lruvec = page_lruvec(page);
 		/*
 		 * The page's writeback ends up during pagevec
 		 * We moves tha page into tail of inactive.
 		 */
-		lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
 		list_move_tail(&page->lru, &lruvec->pages_lru[lru]);
 		__count_vm_event(PGROTATED);
 	}
mm/vmscan.c +19 -6
index e41ad52..f3c0fbe 100644
@@ -1180,7 +1180,6 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 
 		switch (__isolate_lru_page(page, mode, file)) {
 		case 0:
-			mem_cgroup_lru_del(page);
 			list_move(&page->lru, dst);
 			nr_taken += hpage_nr_pages(page);
 			break;
@@ -1238,10 +1237,16 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 
 			if (__isolate_lru_page(cursor_page, mode, file) == 0) {
 				unsigned int isolated_pages;
+				struct lruvec *cursor_lruvec;
+				int cursor_lru = page_lru(cursor_page);
 
-				mem_cgroup_lru_del(cursor_page);
 				list_move(&cursor_page->lru, dst);
 				isolated_pages = hpage_nr_pages(cursor_page);
+				cursor_lruvec = page_lruvec(cursor_page);
+				cursor_lruvec->pages_count[cursor_lru] -=
+								isolated_pages;
+				VM_BUG_ON((long)cursor_lruvec->
+						pages_count[cursor_lru] < 0);
 				nr_taken += isolated_pages;
 				nr_lumpy_taken += isolated_pages;
 				if (PageDirty(cursor_page))
@@ -1273,6 +1278,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 			nr_lumpy_failed++;
 	}
 
+	lruvec->pages_count[lru] -= nr_taken - nr_lumpy_taken;
+	VM_BUG_ON((long)lruvec->pages_count[lru] < 0);
+
 	*nr_scanned = scan;
 
 	trace_mm_vmscan_lru_isolate(sc->order,
@@ -1656,15 +1664,18 @@ static void move_active_pages_to_lru(struct zone *zone,
 
 	while (!list_empty(list)) {
 		struct lruvec *lruvec;
+		int numpages;
 
 		page = lru_to_page(list);
 
 		VM_BUG_ON(PageLRU(page));
 		SetPageLRU(page);
 
-		lruvec = mem_cgroup_lru_add_list(zone, page, lru);
+		lruvec = page_lruvec(page);
 		list_move(&page->lru, &lruvec->pages_lru[lru]);
-		pgmoved += hpage_nr_pages(page);
+		numpages = hpage_nr_pages(page);
+		lruvec->pages_count[lru] += numpages;
+		pgmoved += numpages;
 
 		if (put_page_testzero(page)) {
 			__ClearPageLRU(page);
@@ -3590,8 +3601,10 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
 			VM_BUG_ON(PageActive(page));
 			ClearPageUnevictable(page);
 			__dec_zone_state(zone, NR_UNEVICTABLE);
-			lruvec = mem_cgroup_lru_move_lists(zone, page,
-						LRU_UNEVICTABLE, lru);
+			lruvec = page_lruvec(page);
+			lruvec->pages_count[LRU_UNEVICTABLE]--;
+			VM_BUG_ON((long)lruvec->pages_count[LRU_UNEVICTABLE] < 0);
+			lruvec->pages_count[lru]++;
 			list_move(&page->lru, &lruvec->pages_lru[lru]);
 			__inc_zone_state(zone, NR_INACTIVE_ANON + lru);
 			pgrescued++;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#20 lore raw

Re: [PATCH v3 07/21] mm: add lruvec->pages_count

Quoted 8 lines · On Thu, 23 Feb 2012 17:52:08 +0400 Konstantin Khlebnikov…

Move lru pages counter from mem_cgroup_per_zone->count[] to lruvec->pages_count[]

Account pages in all lruvecs, incuding root,
this isn't a huge overhead, but it greatly simplifies all code.

Redundant page_lruvec() calls will be optimized in further patches.

Hmm, I like this but..a question below.

Quoted 95 lines
---
 include/linux/memcontrol.h |   29 --------------
 include/linux/mm_inline.h  |   15 +++++--
 include/linux/mmzone.h     |    1 
 mm/memcontrol.c            |   93 +-------------------------------------------
 mm/swap.c                  |    7 +--
 mm/vmscan.c                |   25 +++++++++---
 6 files changed, 34 insertions(+), 136 deletions(-)
include/linux/memcontrol.h +0 -29
index 4822d53..b9d555b 100644
@@ -63,12 +63,6 @@ extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
 					gfp_t gfp_mask);
 
 struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
-struct lruvec *mem_cgroup_lru_add_list(struct zone *, struct page *,
-				       enum lru_list);
-void mem_cgroup_lru_del_list(struct page *, enum lru_list);
-void mem_cgroup_lru_del(struct page *);
-struct lruvec *mem_cgroup_lru_move_lists(struct zone *, struct page *,
-					 enum lru_list, enum lru_list);
 
 /* For coalescing uncharge for reducing memcg' overhead*/
 extern void mem_cgroup_uncharge_start(void);
@@ -212,29 +206,6 @@ static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
 	return &zone->lruvec;
 }
 
-static inline struct lruvec *mem_cgroup_lru_add_list(struct zone *zone,
-						     struct page *page,
-						     enum lru_list lru)
-{
-	return &zone->lruvec;
-}
-
-static inline void mem_cgroup_lru_del_list(struct page *page, enum lru_list lru)
-{
-}
-
-static inline void mem_cgroup_lru_del(struct page *page)
-{
-}
-
-static inline struct lruvec *mem_cgroup_lru_move_lists(struct zone *zone,
-						       struct page *page,
-						       enum lru_list from,
-						       enum lru_list to)
-{
-	return &zone->lruvec;
-}
-
 static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
 {
 	return NULL;
include/linux/mm_inline.h +10 -5
index 8415596..daa3d15 100644
@@ -24,19 +24,24 @@ static inline int page_is_file_cache(struct page *page)
 static inline void
 add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
 {
-	struct lruvec *lruvec;
+	struct lruvec *lruvec = page_lruvec(page);
+	int numpages = hpage_nr_pages(page);
 
-	lruvec = mem_cgroup_lru_add_list(zone, page, lru);
 	list_add(&page->lru, &lruvec->pages_lru[lru]);
-	__mod_zone_page_state(zone, NR_LRU_BASE + lru, hpage_nr_pages(page));
+	lruvec->pages_count[lru] += numpages;
+	__mod_zone_page_state(zone, NR_LRU_BASE + lru, numpages);
 }
 
 static inline void
 del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
 {
-	mem_cgroup_lru_del_list(page, lru);
+	struct lruvec *lruvec = page_lruvec(page);
+	int numpages = hpage_nr_pages(page);
+
 	list_del(&page->lru);
-	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -hpage_nr_pages(page));
+	lruvec->pages_count[lru] -= numpages;
+	VM_BUG_ON((long)lruvec->pages_count[lru] < 0);
+	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -numpages);
 }
 
 /**
include/linux/mmzone.h +0 -0
index be8873a..69b0f31 100644

include/linux/mmzone.h struct zone_reclaim_stat { · line 301

Quoted 4 lines
include/linux/mmzone.h +1 -0
@@ -298,6 +298,7 @@ struct zone_reclaim_stat {
 
 struct lruvec {
 	struct list_head	pages_lru[NR_LRU_LISTS];
+	unsigned long		pages_count[NR_LRU_LISTS];

In this time, you don't put the objects under #ifdef...why ?

How do you handle duplication "the number of pages in LRU" of zone->vm_stat and this ?

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#21 lore raw

Re: [PATCH v3 07/21] mm: add lruvec->pages_count

Quoted 110 lines · KAMEZAWA Hiroyuki
Quoted 8 lines · On Thu, 23 Feb 2012 17:52:08 +0400 Konstantin…

Move lru pages counter from mem_cgroup_per_zone->count[] to lruvec->pages_count[]

Account pages in all lruvecs, incuding root,
this isn't a huge overhead, but it greatly simplifies all code.

Redundant page_lruvec() calls will be optimized in further patches.

Hmm, I like this but..a question below.

Quoted 96 lines
---
  include/linux/memcontrol.h |   29 --------------
  include/linux/mm_inline.h  |   15 +++++--
  include/linux/mmzone.h     |    1
  mm/memcontrol.c            |   93 +-------------------------------------------
  mm/swap.c                  |    7 +--
  mm/vmscan.c                |   25 +++++++++---
  6 files changed, 34 insertions(+), 136 deletions(-)
include/linux/memcontrol.h +0 -29
index 4822d53..b9d555b 100644
@@ -63,12 +63,6 @@ extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
  					gfp_t gfp_mask);
 
  struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
-struct lruvec *mem_cgroup_lru_add_list(struct zone *, struct page *,
-				       enum lru_list);
-void mem_cgroup_lru_del_list(struct page *, enum lru_list);
-void mem_cgroup_lru_del(struct page *);
-struct lruvec *mem_cgroup_lru_move_lists(struct zone *, struct page *,
-					 enum lru_list, enum lru_list);
 
  /* For coalescing uncharge for reducing memcg' overhead*/
  extern void mem_cgroup_uncharge_start(void);
@@ -212,29 +206,6 @@ static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
  	return&zone->lruvec;
  }
 
-static inline struct lruvec *mem_cgroup_lru_add_list(struct zone *zone,
-						     struct page *page,
-						     enum lru_list lru)
-{
-	return&zone->lruvec;
-}
-
-static inline void mem_cgroup_lru_del_list(struct page *page, enum lru_list lru)
-{
-}
-
-static inline void mem_cgroup_lru_del(struct page *page)
-{
-}
-
-static inline struct lruvec *mem_cgroup_lru_move_lists(struct zone *zone,
-						       struct page *page,
-						       enum lru_list from,
-						       enum lru_list to)
-{
-	return&zone->lruvec;
-}
-
  static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
  {
  	return NULL;
include/linux/mm_inline.h +10 -5
index 8415596..daa3d15 100644
@@ -24,19 +24,24 @@ static inline int page_is_file_cache(struct page *page)
  static inline void
  add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
  {
-	struct lruvec *lruvec;
+	struct lruvec *lruvec = page_lruvec(page);
+	int numpages = hpage_nr_pages(page);
 
-	lruvec = mem_cgroup_lru_add_list(zone, page, lru);
  	list_add(&page->lru,&lruvec->pages_lru[lru]);
-	__mod_zone_page_state(zone, NR_LRU_BASE + lru, hpage_nr_pages(page));
+	lruvec->pages_count[lru] += numpages;
+	__mod_zone_page_state(zone, NR_LRU_BASE + lru, numpages);
  }
 
  static inline void
  del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
  {
-	mem_cgroup_lru_del_list(page, lru);
+	struct lruvec *lruvec = page_lruvec(page);
+	int numpages = hpage_nr_pages(page);
+
  	list_del(&page->lru);
-	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -hpage_nr_pages(page));
+	lruvec->pages_count[lru] -= numpages;
+	VM_BUG_ON((long)lruvec->pages_count[lru]<  0);
+	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -numpages);
  }
 
  /**
include/linux/mmzone.h +0 -0
index be8873a..69b0f31 100644
@@ -298,6 +298,7 @@ struct zone_reclaim_stat {
 
Quoted 5 lines

include/linux/mmzone.h struct zone_reclaim_stat { · line 301

Quoted 3 lines
include/linux/mmzone.h +1 -0
  struct lruvec {
  	struct list_head	pages_lru[NR_LRU_LISTS];
+	unsigned long		pages_count[NR_LRU_LISTS];

In this time, you don't put the objects under #ifdef...why ?

It will make the code much uglier and does not speed up it at all.

Quoted 2 lines

How do you handle duplication "the number of pages in LRU" of zone->vm_stat and this ?

I don't think this is totally bad, vmstat usually has per-cpu drift, these numbers will be exact.

Quoted 4 lines

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#22 lore raw

PATCH v3 8/21 mm: unify inactive_list_is_low()

In reply to Konstantin Khlebnikov

Unify memcg and non-memcg logic, always use exact counters from struct lruvec.

---
 mm/vmscan.c |   30 ++++++++----------------------
 1 files changed, 8 insertions(+), 22 deletions(-)
mm/vmscan.c +8 -22
index f3c0fbe..b3e8bab 100644
@@ -1822,6 +1822,7 @@ static int inactive_anon_is_low(struct mem_cgroup_zone *mz)
 {
 	unsigned long active, inactive;
 	unsigned int gb, ratio;
+	struct lruvec *lruvec;
 
 	/*
 	 * If we don't have swap space, anonymous page deactivation
@@ -1830,17 +1831,9 @@ static int inactive_anon_is_low(struct mem_cgroup_zone *mz)
 	if (!total_swap_pages)
 		return 0;
 
-	if (scanning_global_lru(mz)) {
-		active = zone_page_state(mz->zone, NR_ACTIVE_ANON);
-		inactive = zone_page_state(mz->zone, NR_INACTIVE_ANON);
-	} else {
-		active = mem_cgroup_zone_nr_lru_pages(mz->mem_cgroup,
-				zone_to_nid(mz->zone), zone_idx(mz->zone),
-				BIT(LRU_ACTIVE_ANON));
-		inactive = mem_cgroup_zone_nr_lru_pages(mz->mem_cgroup,
-				zone_to_nid(mz->zone), zone_idx(mz->zone),
-				BIT(LRU_INACTIVE_ANON));
-	}
+	lruvec = mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup);
+	active = lruvec->pages_count[LRU_ACTIVE_ANON];
+	inactive = lruvec->pages_count[LRU_INACTIVE_ANON];
 
 	/* Total size in gigabytes */
 	gb = (active + inactive) >> (30 - PAGE_SHIFT);
@@ -1875,18 +1868,11 @@ static inline int inactive_anon_is_low(struct mem_cgroup_zone *mz)
 static int inactive_file_is_low(struct mem_cgroup_zone *mz)
 {
 	unsigned long active, inactive;
+	struct lruvec *lruvec;
 
-	if (scanning_global_lru(mz)) {
-		active = zone_page_state(mz->zone, NR_ACTIVE_FILE);
-		inactive = zone_page_state(mz->zone, NR_INACTIVE_FILE);
-	} else {
-		active = mem_cgroup_zone_nr_lru_pages(mz->mem_cgroup,
-				zone_to_nid(mz->zone), zone_idx(mz->zone),
-				BIT(LRU_ACTIVE_FILE));
-		inactive = mem_cgroup_zone_nr_lru_pages(mz->mem_cgroup,
-				zone_to_nid(mz->zone), zone_idx(mz->zone),
-				BIT(LRU_INACTIVE_FILE));
-	}
+	lruvec = mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup);
+	active = lruvec->pages_count[LRU_ACTIVE_FILE];
+	inactive = lruvec->pages_count[LRU_INACTIVE_FILE];
 
 	return inactive < active;
 }

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 8/21
#23 lore raw

Re: [PATCH v3 08/21] mm: unify inactive_list_is_low()

Quoted 3 lines · On Thu, 23 Feb 2012 17:52:19 +0400 Konstantin Khlebnikov…

Unify memcg and non-memcg logic, always use exact counters from struct lruvec.

Nice.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#24 lore raw

PATCH v3 9/21 mm: add lruvec->reclaim_stat

In reply to Konstantin Khlebnikov

Merge memcg and non-memcg reclaim stat. We need to update only one. Move zone->reclaimer_stat and mem_cgroup_per_zone->reclaimer_stat to struct lruvec.

struct lruvec will become operating unit for recalimer logic, thus this is perfect place for these counters.

---
 include/linux/memcontrol.h |   17 --------------
 include/linux/mmzone.h     |    5 +++-
 mm/memcontrol.c            |   52 +++++---------------------------------------
 mm/page_alloc.c            |    4 ---
 mm/swap.c                  |   12 ++--------
 mm/vmscan.c                |    5 +++-
 6 files changed, 14 insertions(+), 81 deletions(-)
include/linux/memcontrol.h +0 -17
index b9d555b..c3e46b0 100644
@@ -103,10 +103,6 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
 unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
 					int nid, int zid, unsigned int lrumask);
-struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
-						      struct zone *zone);
-struct zone_reclaim_stat*
-mem_cgroup_get_reclaim_stat_from_page(struct page *page);
 extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
 					struct task_struct *p);
 extern void mem_cgroup_replace_page_cache(struct page *oldpage,
@@ -286,19 +282,6 @@ mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
 	return 0;
 }
 
-
-static inline struct zone_reclaim_stat*
-mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, struct zone *zone)
-{
-	return NULL;
-}
-
-static inline struct zone_reclaim_stat*
-mem_cgroup_get_reclaim_stat_from_page(struct page *page)
-{
-	return NULL;
-}
-
 static inline void
 mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
 {
include/linux/mmzone.h +3 -2
index 69b0f31..82d5ff3 100644
@@ -299,6 +299,9 @@ struct zone_reclaim_stat {
 struct lruvec {
 	struct list_head	pages_lru[NR_LRU_LISTS];
 	unsigned long		pages_count[NR_LRU_LISTS];
+
+	struct zone_reclaim_stat	reclaim_stat;
+
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
 	struct zone		*zone;
 	struct pglist_data	*node;
@@ -379,8 +382,6 @@ struct zone {
 	spinlock_t		lru_lock;
 	struct lruvec		lruvec;
 
-	struct zone_reclaim_stat reclaim_stat;
-
 	unsigned long		pages_scanned;	   /* since last reclaim */
 	unsigned long		flags;		   /* zone flags, see below */
 
mm/memcontrol.c +6 -46

83 lines not on this page · raw message

mm/page_alloc.c +0 -4
index 1cc3afe..ab42446 100644
@@ -4375,10 +4375,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 
 		zone_pcp_init(zone);
 		init_zone_lruvec(zone, &zone->lruvec);
-		zone->reclaim_stat.recent_rotated[0] = 0;
-		zone->reclaim_stat.recent_rotated[1] = 0;
-		zone->reclaim_stat.recent_scanned[0] = 0;
-		zone->reclaim_stat.recent_scanned[1] = 0;
 		zap_zone_vm_stats(zone);
 		zone->flags = 0;
 		if (!size)
mm/swap.c +2 -10
index 2afe02c..70d1542 100644
@@ -277,21 +277,13 @@ void rotate_reclaimable_page(struct page *page)
 static void update_page_reclaim_stat(struct zone *zone, struct page *page,
 				     int file, int rotated)
 {
-	struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat;
-	struct zone_reclaim_stat *memcg_reclaim_stat;
+	struct zone_reclaim_stat *reclaim_stat;
 
-	memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
+	reclaim_stat = &page_lruvec(page)->reclaim_stat;
 
 	reclaim_stat->recent_scanned[file]++;
 	if (rotated)
 		reclaim_stat->recent_rotated[file]++;
-
-	if (!memcg_reclaim_stat)
-		return;
-
-	memcg_reclaim_stat->recent_scanned[file]++;
-	if (rotated)
-		memcg_reclaim_stat->recent_rotated[file]++;
 }
 
 static void __activate_page(struct page *page, void *arg)
mm/vmscan.c +3 -2
index b3e8bab..98bd61f 100644
@@ -184,9 +184,10 @@ static bool scanning_global_lru(struct mem_cgroup_zone *mz)
 static struct zone_reclaim_stat *get_reclaim_stat(struct mem_cgroup_zone *mz)
 {
 	if (!scanning_global_lru(mz))
-		return mem_cgroup_get_reclaim_stat(mz->mem_cgroup, mz->zone);
+		return &mem_cgroup_zone_lruvec(mz->zone,
+				mz->mem_cgroup)->reclaim_stat;
 
-	return &mz->zone->reclaim_stat;
+	return &mz->zone->lruvec.reclaim_stat;
 }
 
 static unsigned long zone_nr_lru_pages(struct mem_cgroup_zone *mz,

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 9/21
#25 lore raw

Re: [PATCH v3 09/21] mm: add lruvec->reclaim_stat

Quoted 7 lines · On Thu, 23 Feb 2012 17:52:24 +0400 Konstantin Khlebnikov…

Merge memcg and non-memcg reclaim stat. We need to update only one. Move zone->reclaimer_stat and mem_cgroup_per_zone->reclaimer_stat to struct lruvec.

struct lruvec will become operating unit for recalimer logic, thus this is perfect place for these counters.

I like this.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#26 lore raw

PATCH v3 10/21 mm: kill struct mem_cgroup_zone

In reply to Konstantin Khlebnikov

struct mem_cgroup_zone always points to one lruvec, either root zone->lruvec or to some from memcg. So this fancy pointer can be replaced with direct pointer to struct lruvec, because all required infromation already collected on lruvec.

---
 mm/vmscan.c |  186 ++++++++++++++++++++++-------------------------------------
 1 files changed, 70 insertions(+), 116 deletions(-)
mm/vmscan.c +70 -116
index 98bd61f..f5e7046 100644
@@ -115,11 +115,6 @@ struct scan_control {
 	nodemask_t	*nodemask;
 };
 
-struct mem_cgroup_zone {
-	struct mem_cgroup *mem_cgroup;
-	struct zone *zone;
-};
-
 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
 
 #ifdef ARCH_HAS_PREFETCH
@@ -164,45 +159,13 @@ static bool global_reclaim(struct scan_control *sc)
 {
 	return !sc->target_mem_cgroup;
 }
-
-static bool scanning_global_lru(struct mem_cgroup_zone *mz)
-{
-	return !mz->mem_cgroup;
-}
 #else
 static bool global_reclaim(struct scan_control *sc)
 {
 	return true;
 }
-
-static bool scanning_global_lru(struct mem_cgroup_zone *mz)
-{
-	return true;
-}
 #endif
 
-static struct zone_reclaim_stat *get_reclaim_stat(struct mem_cgroup_zone *mz)
-{
-	if (!scanning_global_lru(mz))
-		return &mem_cgroup_zone_lruvec(mz->zone,
-				mz->mem_cgroup)->reclaim_stat;
-
-	return &mz->zone->lruvec.reclaim_stat;
-}
-
-static unsigned long zone_nr_lru_pages(struct mem_cgroup_zone *mz,
-				       enum lru_list lru)
-{
-	if (!scanning_global_lru(mz))
-		return mem_cgroup_zone_nr_lru_pages(mz->mem_cgroup,
-						    zone_to_nid(mz->zone),
-						    zone_idx(mz->zone),
-						    BIT(lru));
-
-	return zone_page_state(mz->zone, NR_LRU_BASE + lru);
-}
-
-
 /*
  * Add a shrinker callback to be called from the vm
  */
@@ -764,7 +727,7 @@ static enum page_references page_check_references(struct page *page,
  * shrink_page_list() returns the number of reclaimed pages
  */
 static unsigned long shrink_page_list(struct list_head *page_list,
-				      struct mem_cgroup_zone *mz,
+				      struct lruvec *lruvec,
 				      struct scan_control *sc,
 				      int priority,
 				      unsigned long *ret_nr_dirty,
@@ -795,7 +758,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			goto keep;
 
 		VM_BUG_ON(PageActive(page));
-		VM_BUG_ON(page_zone(page) != mz->zone);
+		VM_BUG_ON(page_zone(page) != lruvec_zone(lruvec));
 
 		sc->nr_scanned++;
 
@@ -1021,7 +984,7 @@ keep_lumpy:
 	 * will encounter the same problem
 	 */
 	if (nr_dirty && nr_dirty == nr_congested && global_reclaim(sc))
-		zone_set_flag(mz->zone, ZONE_CONGESTED);
+		zone_set_flag(lruvec_zone(lruvec), ZONE_CONGESTED);
 
 	free_hot_cold_page_list(&free_pages, 1);
 
@@ -1136,7 +1099,7 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode, int file)
  * Appropriate locks must be held before calling this function.
  *
  * @nr_to_scan:	The number of pages to look through on the list.
- * @mz:		The mem_cgroup_zone to pull pages from.
+ * @lruvec	The struct lruvec to pull pages from.
  * @dst:	The temp list to put pages on to.
  * @nr_scanned:	The number of pages that were scanned.
  * @sc:		The scan_control struct for this reclaim session
@@ -1147,11 +1110,10 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode, int file)
  * returns how many pages were moved onto *@dst.
  */
 static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
-		struct mem_cgroup_zone *mz, struct list_head *dst,
+		struct lruvec *lruvec, struct list_head *dst,
 		unsigned long *nr_scanned, struct scan_control *sc,
 		isolate_mode_t mode, int active, int file)
 {
-	struct lruvec *lruvec;
 	struct list_head *src;
 	unsigned long nr_taken = 0;
 	unsigned long nr_lumpy_taken = 0;
@@ -1160,7 +1122,6 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 	unsigned long scan;
 	int lru = LRU_BASE;
 
-	lruvec = mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup);
 	if (active)
 		lru += LRU_ACTIVE;
 	if (file)
@@ -1366,11 +1327,11 @@ static int too_many_isolated(struct zone *zone, int file,
 }
 
 static noinline_for_stack void
-putback_inactive_pages(struct mem_cgroup_zone *mz,
+putback_inactive_pages(struct lruvec *lruvec,
 		       struct list_head *page_list)
 {
-	struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
-	struct zone *zone = mz->zone;
+	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
+	struct zone *zone = lruvec_zone(lruvec);
 	LIST_HEAD(pages_to_free);
 
 	/*
@@ -1417,13 +1378,13 @@ putback_inactive_pages(struct mem_cgroup_zone *mz,
 }
 
 static noinline_for_stack void
-update_isolated_counts(struct mem_cgroup_zone *mz,
+update_isolated_counts(struct lruvec *lruvec,
 		       struct list_head *page_list,
 		       unsigned long *nr_anon,
 		       unsigned long *nr_file)
 {
-	struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
-	struct zone *zone = mz->zone;
+	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
+	struct zone *zone = lruvec_zone(lruvec);
 	unsigned int count[NR_LRU_LISTS] = { 0, };
 	unsigned long nr_active = 0;
 	struct page *page;
@@ -1507,7 +1468,7 @@ static inline bool should_reclaim_stall(unsigned long nr_taken,
  * of reclaimed pages
  */
 static noinline_for_stack unsigned long
-shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
+shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
 		     struct scan_control *sc, int priority, int file)
 {
 	LIST_HEAD(page_list);
@@ -1519,7 +1480,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
 	unsigned long nr_dirty = 0;
 	unsigned long nr_writeback = 0;
 	isolate_mode_t isolate_mode = ISOLATE_INACTIVE;
-	struct zone *zone = mz->zone;
+	struct zone *zone = lruvec_zone(lruvec);
 
 	while (unlikely(too_many_isolated(zone, file, sc))) {
 		congestion_wait(BLK_RW_ASYNC, HZ/10);
@@ -1542,8 +1503,9 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
 
 	spin_lock_irq(&zone->lru_lock);
 
-	nr_taken = isolate_lru_pages(nr_to_scan, mz, &page_list, &nr_scanned,
-				     sc, isolate_mode, 0, file);
+	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
+				     &nr_scanned, sc, isolate_mode, 0, file);
+
 	if (global_reclaim(sc)) {
 		zone->pages_scanned += nr_scanned;
 		if (current_is_kswapd())
@@ -1559,20 +1521,20 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
 		return 0;
 	}
 
-	update_isolated_counts(mz, &page_list, &nr_anon, &nr_file);
+	update_isolated_counts(lruvec, &page_list, &nr_anon, &nr_file);
 
 	__mod_zone_page_state(zone, NR_ISOLATED_ANON, nr_anon);
 	__mod_zone_page_state(zone, NR_ISOLATED_FILE, nr_file);
 
 	spin_unlock_irq(&zone->lru_lock);
 
-	nr_reclaimed = shrink_page_list(&page_list, mz, sc, priority,
+	nr_reclaimed = shrink_page_list(&page_list, lruvec, sc, priority,
 						&nr_dirty, &nr_writeback);
 
 	/* Check if we should syncronously wait for writeback */
 	if (should_reclaim_stall(nr_taken, nr_reclaimed, priority, sc)) {
 		set_reclaim_mode(priority, sc, true);
-		nr_reclaimed += shrink_page_list(&page_list, mz, sc,
+		nr_reclaimed += shrink_page_list(&page_list, lruvec, sc,
 					priority, &nr_dirty, &nr_writeback);
 	}
 
@@ -1582,7 +1544,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
 		__count_vm_events(KSWAPD_STEAL, nr_reclaimed);
 	__count_zone_vm_events(PGSTEAL, zone, nr_reclaimed);
 
-	putback_inactive_pages(mz, &page_list);
+	putback_inactive_pages(lruvec, &page_list);
 
 	__mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon);
 	__mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file);
@@ -1697,7 +1659,7 @@ static void move_active_pages_to_lru(struct zone *zone,
 }
 
 static void shrink_active_list(unsigned long nr_to_scan,
-			       struct mem_cgroup_zone *mz,
+			       struct lruvec *lruvec,
 			       struct scan_control *sc,
 			       int priority, int file)
 {
@@ -1708,10 +1670,10 @@ static void shrink_active_list(unsigned long nr_to_scan,
 	LIST_HEAD(l_active);
 	LIST_HEAD(l_inactive);
 	struct page *page;
-	struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
+	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
 	unsigned long nr_rotated = 0;
 	isolate_mode_t isolate_mode = ISOLATE_ACTIVE;
-	struct zone *zone = mz->zone;
+	struct zone *zone = lruvec_zone(lruvec);
 
 	lru_add_drain();
 
@@ -1722,8 +1684,9 @@ static void shrink_active_list(unsigned long nr_to_scan,
 
 	spin_lock_irq(&zone->lru_lock);
 
-	nr_taken = isolate_lru_pages(nr_to_scan, mz, &l_hold, &nr_scanned, sc,
-				     isolate_mode, 1, file);
+	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold, &nr_scanned,
+				     sc, isolate_mode, 1, file);
+
 	if (global_reclaim(sc))
 		zone->pages_scanned += nr_scanned;
 
@@ -1819,11 +1782,10 @@ static void shrink_active_list(unsigned long nr_to_scan,
  *    1TB     101        10GB
  *   10TB     320        32GB
  */
-static int inactive_anon_is_low(struct mem_cgroup_zone *mz)
+static int inactive_anon_is_low(struct lruvec *lruvec)
 {
 	unsigned long active, inactive;
 	unsigned int gb, ratio;
-	struct lruvec *lruvec;
 
 	/*
 	 * If we don't have swap space, anonymous page deactivation
@@ -1832,7 +1794,6 @@ static int inactive_anon_is_low(struct mem_cgroup_zone *mz)
 	if (!total_swap_pages)
 		return 0;
 
-	lruvec = mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup);
 	active = lruvec->pages_count[LRU_ACTIVE_ANON];
 	inactive = lruvec->pages_count[LRU_INACTIVE_ANON];
 
@@ -1846,7 +1807,7 @@ static int inactive_anon_is_low(struct mem_cgroup_zone *mz)
 	return inactive * ratio < active;
 }
 #else
-static inline int inactive_anon_is_low(struct mem_cgroup_zone *mz)
+static inline int inactive_anon_is_low(struct lruvec *lruvec)
 {
 	return 0;
 }
@@ -1866,39 +1827,38 @@ static inline int inactive_anon_is_low(struct mem_cgroup_zone *mz)
  * This uses a different ratio than the anonymous pages, because
  * the page cache uses a use-once replacement algorithm.
  */
-static int inactive_file_is_low(struct mem_cgroup_zone *mz)
+static int inactive_file_is_low(struct lruvec *lruvec)
 {
 	unsigned long active, inactive;
-	struct lruvec *lruvec;
 
-	lruvec = mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup);
 	active = lruvec->pages_count[LRU_ACTIVE_FILE];
 	inactive = lruvec->pages_count[LRU_INACTIVE_FILE];
 
 	return inactive < active;
 }
 
-static int inactive_list_is_low(struct mem_cgroup_zone *mz, int file)
+static int inactive_list_is_low(struct lruvec *lruvec, int file)
 {
 	if (file)
-		return inactive_file_is_low(mz);
+		return inactive_file_is_low(lruvec);
 	else
-		return inactive_anon_is_low(mz);
+		return inactive_anon_is_low(lruvec);
 }
 
 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
-				 struct mem_cgroup_zone *mz,
+				 struct lruvec *lruvec,
 				 struct scan_control *sc, int priority)
 {
 	int file = is_file_lru(lru);
 
 	if (is_active_lru(lru)) {
-		if (inactive_list_is_low(mz, file))
-			shrink_active_list(nr_to_scan, mz, sc, priority, file);
+		if (inactive_list_is_low(lruvec, file))
+			shrink_active_list(nr_to_scan, lruvec,
+					   sc, priority, file);
 		return 0;
 	}
 
-	return shrink_inactive_list(nr_to_scan, mz, sc, priority, file);
+	return shrink_inactive_list(nr_to_scan, lruvec, sc, priority, file);
 }
 
 static int vmscan_swappiness(struct scan_control *sc)
@@ -1916,17 +1876,18 @@ static int vmscan_swappiness(struct scan_control *sc)
  *
  * nr[0] = anon pages to scan; nr[1] = file pages to scan
  */
-static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,
+static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
 			   unsigned long *nr, int priority)
 {
 	unsigned long anon, file, free;
 	unsigned long anon_prio, file_prio;
 	unsigned long ap, fp;
-	struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz);
+	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
 	u64 fraction[2], denominator;
 	enum lru_list lru;
 	int noswap = 0;
 	bool force_scan = false;
+	struct zone *zone = lruvec_zone(lruvec);
 
 	/*
 	 * If the zone or memcg is small, nr[l] can be 0.  This
@@ -1938,7 +1899,7 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,
 	 * latencies, so it's better to scan a minimum amount there as
 	 * well.
 	 */
-	if (current_is_kswapd() && mz->zone->all_unreclaimable)
+	if (current_is_kswapd() && zone->all_unreclaimable)
 		force_scan = true;
 	if (!global_reclaim(sc))
 		force_scan = true;
@@ -1952,16 +1913,16 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,
 		goto out;
 	}
 
-	anon  = zone_nr_lru_pages(mz, LRU_ACTIVE_ANON) +
-		zone_nr_lru_pages(mz, LRU_INACTIVE_ANON);
-	file  = zone_nr_lru_pages(mz, LRU_ACTIVE_FILE) +
-		zone_nr_lru_pages(mz, LRU_INACTIVE_FILE);
+	anon  = lruvec->pages_count[LRU_ACTIVE_ANON] +
+		lruvec->pages_count[LRU_INACTIVE_ANON];
+	file  = lruvec->pages_count[LRU_ACTIVE_FILE] +
+		lruvec->pages_count[LRU_INACTIVE_FILE];
 
 	if (global_reclaim(sc)) {
-		free  = zone_page_state(mz->zone, NR_FREE_PAGES);
+		free  = zone_page_state(zone, NR_FREE_PAGES);
 		/* If we have very few page cache pages,
 		   force-scan anon pages. */
-		if (unlikely(file + free <= high_wmark_pages(mz->zone))) {
+		if (unlikely(file + free <= high_wmark_pages(zone))) {
 			fraction[0] = 1;
 			fraction[1] = 0;
 			denominator = 1;
@@ -1987,7 +1948,7 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,
 	 *
 	 * anon in [0], file in [1]
 	 */
-	spin_lock_irq(&mz->zone->lru_lock);
+	spin_lock_irq(&zone->lru_lock);
 	if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
 		reclaim_stat->recent_scanned[0] /= 2;
 		reclaim_stat->recent_rotated[0] /= 2;
@@ -2008,7 +1969,7 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,
 
 	fp = (file_prio + 1) * (reclaim_stat->recent_scanned[1] + 1);
 	fp /= reclaim_stat->recent_rotated[1] + 1;
-	spin_unlock_irq(&mz->zone->lru_lock);
+	spin_unlock_irq(&zone->lru_lock);
 
 	fraction[0] = ap;
 	fraction[1] = fp;
@@ -2018,7 +1979,7 @@ out:
 		int file = is_file_lru(lru);
 		unsigned long scan;
 
-		scan = zone_nr_lru_pages(mz, lru);
+		scan = lruvec->pages_count[lru];
 		if (priority || noswap) {
 			scan >>= priority;
 			if (!scan && force_scan)
@@ -2036,7 +1997,7 @@ out:
  * back to the allocator and call try_to_compact_zone(), we ensure that
  * there are enough free pages for it to be likely successful
  */
-static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,
+static inline bool should_continue_reclaim(struct lruvec *lruvec,
 					unsigned long nr_reclaimed,
 					unsigned long nr_scanned,
 					struct scan_control *sc)
@@ -2076,15 +2037,15 @@ static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,
 	 * inactive lists are large enough, continue reclaiming
 	 */
 	pages_for_compaction = (2UL << sc->order);
-	inactive_lru_pages = zone_nr_lru_pages(mz, LRU_INACTIVE_FILE);
+	inactive_lru_pages = lruvec->pages_count[LRU_INACTIVE_FILE];
 	if (nr_swap_pages > 0)
-		inactive_lru_pages += zone_nr_lru_pages(mz, LRU_INACTIVE_ANON);
+		inactive_lru_pages += lruvec->pages_count[LRU_INACTIVE_ANON];
 	if (sc->nr_reclaimed < pages_for_compaction &&
 			inactive_lru_pages > pages_for_compaction)
 		return true;
 
 	/* If compaction would go ahead or the allocation would succeed, stop */
-	switch (compaction_suitable(mz->zone, sc->order)) {
+	switch (compaction_suitable(lruvec_zone(lruvec), sc->order)) {
 	case COMPACT_PARTIAL:
 	case COMPACT_CONTINUE:
 		return false;
@@ -2096,8 +2057,8 @@ static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,
 /*
  * This is a basic per-zone page freer.  Used by both kswapd and direct reclaim.
  */
-static void shrink_mem_cgroup_zone(int priority, struct mem_cgroup_zone *mz,
-				   struct scan_control *sc)
+static void shrink_lruvec(int priority, struct lruvec *lruvec,
+			struct scan_control *sc)
 {
 	unsigned long nr[NR_LRU_LISTS];
 	unsigned long nr_to_scan;
@@ -2109,7 +2070,7 @@ static void shrink_mem_cgroup_zone(int priority, struct mem_cgroup_zone *mz,
 restart:
 	nr_reclaimed = 0;
 	nr_scanned = sc->nr_scanned;
-	get_scan_count(mz, sc, nr, priority);
+	get_scan_count(lruvec, sc, nr, priority);
 
 	blk_start_plug(&plug);
 	while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
@@ -2121,7 +2082,7 @@ restart:
 				nr[lru] -= nr_to_scan;
 
 				nr_reclaimed += shrink_list(lru, nr_to_scan,
-							    mz, sc, priority);
+							    lruvec, sc, priority);
 			}
 		}
 		/*
@@ -2147,11 +2108,11 @@ restart:
 	 * Even if we did not try to evict anon pages at all, we want to
 	 * rebalance the anon lru active/inactive ratio.
 	 */
-	if (inactive_anon_is_low(mz))
-		shrink_active_list(SWAP_CLUSTER_MAX, mz, sc, priority, 0);
+	if (inactive_anon_is_low(lruvec))
+		shrink_active_list(SWAP_CLUSTER_MAX, lruvec, sc, priority, 0);
 
 	/* reclaim/compaction might need reclaim to continue */
-	if (should_continue_reclaim(mz, nr_reclaimed,
+	if (should_continue_reclaim(lruvec, nr_reclaimed,
 					sc->nr_scanned - nr_scanned, sc))
 		goto restart;
 
@@ -2167,15 +2128,14 @@ static void shrink_zone(int priority, struct zone *zone,
 		.priority = priority,
 	};
 	struct mem_cgroup *memcg;
+	struct lruvec *lruvec;
 
 	memcg = mem_cgroup_iter(root, NULL, &reclaim);
 	do {
-		struct mem_cgroup_zone mz = {
-			.mem_cgroup = memcg,
-			.zone = zone,
-		};
+		lruvec = mem_cgroup_zone_lruvec(zone, memcg);
+
+		shrink_lruvec(priority, lruvec, sc);
 
-		shrink_mem_cgroup_zone(priority, &mz, sc);
 		/*
 		 * Limit reclaim has historically picked one memcg and
 		 * scanned it with decreasing priority levels until
@@ -2495,10 +2455,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,
 		.order = 0,
 		.target_mem_cgroup = memcg,
 	};
-	struct mem_cgroup_zone mz = {
-		.mem_cgroup = memcg,
-		.zone = zone,
-	};
+	struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
 
 	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
 			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
@@ -2514,7 +2471,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,
 	 * will pick up pages from other mem cgroup's as well. We hack
 	 * the priority and make it zero.
 	 */
-	shrink_mem_cgroup_zone(0, &mz, &sc);
+	shrink_lruvec(0, lruvec, &sc);
 
 	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
 
@@ -2575,13 +2532,10 @@ static void age_active_anon(struct zone *zone, struct scan_control *sc,
 
 	memcg = mem_cgroup_iter(NULL, NULL, NULL);
 	do {
-		struct mem_cgroup_zone mz = {
-			.mem_cgroup = memcg,
-			.zone = zone,
-		};
+		struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
 
-		if (inactive_anon_is_low(&mz))
-			shrink_active_list(SWAP_CLUSTER_MAX, &mz,
+		if (inactive_anon_is_low(lruvec))
+			shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
 					   sc, priority, 0);
 
 		memcg = mem_cgroup_iter(NULL, memcg, NULL);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 10/21
#27 lore raw

Re: [PATCH v3 10/21] mm: kill struct mem_cgroup_zone

Quoted 5 lines · On Thu, 23 Feb 2012 17:52:29 +0400 Konstantin Khlebnikov…

struct mem_cgroup_zone always points to one lruvec, either root zone->lruvec or to some from memcg. So this fancy pointer can be replaced with direct pointer to struct lruvec, because all required infromation already collected on lruvec.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#28 lore raw

PATCH v3 11/21 mm: move page-to-lruvec translation upper

In reply to Konstantin Khlebnikov

move page_lruvec() out of add_page_to_lru_list() and del_page_from_lru_list() switch its first argument from zone to lruvec.

---
 include/linux/mm_inline.h |   10 ++++------
 mm/compaction.c           |    4 +++-
 mm/memcontrol.c           |    7 +++++--
 mm/swap.c                 |   33 +++++++++++++++++++++------------
 mm/vmscan.c               |   14 +++++++++-----
 5 files changed, 42 insertions(+), 26 deletions(-)
include/linux/mm_inline.h +4 -6
index daa3d15..143a2e8 100644
@@ -22,26 +22,24 @@ static inline int page_is_file_cache(struct page *page)
 }
 
 static inline void
-add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
+add_page_to_lru_list(struct lruvec *lruvec, struct page *page, enum lru_list lru)
 {
-	struct lruvec *lruvec = page_lruvec(page);
 	int numpages = hpage_nr_pages(page);
 
 	list_add(&page->lru, &lruvec->pages_lru[lru]);
 	lruvec->pages_count[lru] += numpages;
-	__mod_zone_page_state(zone, NR_LRU_BASE + lru, numpages);
+	__mod_zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru, numpages);
 }
 
 static inline void
-del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list lru)
+del_page_from_lru_list(struct lruvec *lruvec, struct page *page, enum lru_list lru)
 {
-	struct lruvec *lruvec = page_lruvec(page);
 	int numpages = hpage_nr_pages(page);
 
 	list_del(&page->lru);
 	lruvec->pages_count[lru] -= numpages;
 	VM_BUG_ON((long)lruvec->pages_count[lru] < 0);
-	__mod_zone_page_state(zone, NR_LRU_BASE + lru, -numpages);
+	__mod_zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru, -numpages);
 }
 
 /**
mm/compaction.c +3 -1
index 74a8c82..a976b28 100644
@@ -262,6 +262,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 	unsigned long nr_scanned = 0, nr_isolated = 0;
 	struct list_head *migratelist = &cc->migratepages;
 	isolate_mode_t mode = ISOLATE_ACTIVE|ISOLATE_INACTIVE;
+	struct lruvec *lruvec;
 
 	/* Do not scan outside zone boundaries */
 	low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn);
@@ -381,7 +382,8 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 		VM_BUG_ON(PageTransCompound(page));
 
 		/* Successfully isolated */
-		del_page_from_lru_list(zone, page, page_lru(page));
+		lruvec = page_lruvec(page);
+		del_page_from_lru_list(lruvec, page, page_lru(page));
 		list_add(&page->lru, migratelist);
 		cc->nr_migratepages++;
 		nr_isolated++;
mm/memcontrol.c +5 -2
index bef57db..83fa99b 100644
@@ -2543,6 +2543,7 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,
 {
 	struct page_cgroup *pc = lookup_page_cgroup(page);
 	struct zone *zone = page_zone(page);
+	struct lruvec *lruvec;
 	unsigned long flags;
 	bool removed = false;
 
@@ -2553,13 +2554,15 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,
 	 */
 	spin_lock_irqsave(&zone->lru_lock, flags);
 	if (PageLRU(page)) {
-		del_page_from_lru_list(zone, page, page_lru(page));
+		lruvec = page_lruvec(page);
+		del_page_from_lru_list(lruvec, page, page_lru(page));
 		ClearPageLRU(page);
 		removed = true;
 	}
 	__mem_cgroup_commit_charge(memcg, page, 1, pc, ctype);
 	if (removed) {
-		add_page_to_lru_list(zone, page, page_lru(page));
+		lruvec = page_lruvec(page);
+		add_page_to_lru_list(lruvec, page, page_lru(page));
 		SetPageLRU(page);
 	}
 	spin_unlock_irqrestore(&zone->lru_lock, flags);
mm/swap.c +21 -12
index 70d1542..0cbc558 100644
@@ -49,11 +49,13 @@ static void __page_cache_release(struct page *page)
 	if (PageLRU(page)) {
 		unsigned long flags;
 		struct zone *zone = page_zone(page);
+		struct lruvec *lruvec;
 
 		spin_lock_irqsave(&zone->lru_lock, flags);
 		VM_BUG_ON(!PageLRU(page));
 		__ClearPageLRU(page);
-		del_page_from_lru_list(zone, page, page_off_lru(page));
+		lruvec = page_lruvec(page);
+		del_page_from_lru_list(lruvec, page, page_off_lru(page));
 		spin_unlock_irqrestore(&zone->lru_lock, flags);
 	}
 }
@@ -293,11 +295,13 @@ static void __activate_page(struct page *page, void *arg)
 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 		int file = page_is_file_cache(page);
 		int lru = page_lru_base_type(page);
-		del_page_from_lru_list(zone, page, lru);
+		struct lruvec *lruvec = page_lruvec(page);
+
+		del_page_from_lru_list(lruvec, page, lru);
 
 		SetPageActive(page);
 		lru += LRU_ACTIVE;
-		add_page_to_lru_list(zone, page, lru);
+		add_page_to_lru_list(lruvec, page, lru);
 		__count_vm_event(PGACTIVATE);
 
 		update_page_reclaim_stat(zone, page, file, 1);
@@ -404,11 +408,13 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)
 void add_page_to_unevictable_list(struct page *page)
 {
 	struct zone *zone = page_zone(page);
+	struct lruvec *lruvec;
 
 	spin_lock_irq(&zone->lru_lock);
 	SetPageUnevictable(page);
 	SetPageLRU(page);
-	add_page_to_lru_list(zone, page, LRU_UNEVICTABLE);
+	lruvec = page_lruvec(page);
+	add_page_to_lru_list(lruvec, page, LRU_UNEVICTABLE);
 	spin_unlock_irq(&zone->lru_lock);
 }
 
@@ -438,6 +444,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 	int lru, file;
 	bool active;
 	struct zone *zone = page_zone(page);
+	struct lruvec *lruvec;
 
 	if (!PageLRU(page))
 		return;
@@ -453,10 +460,11 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 
 	file = page_is_file_cache(page);
 	lru = page_lru_base_type(page);
-	del_page_from_lru_list(zone, page, lru + active);
+	lruvec = page_lruvec(page);
+	del_page_from_lru_list(lruvec, page, lru + active);
 	ClearPageActive(page);
 	ClearPageReferenced(page);
-	add_page_to_lru_list(zone, page, lru);
+	add_page_to_lru_list(lruvec, page, lru);
 
 	if (PageWriteback(page) || PageDirty(page)) {
 		/*
@@ -466,7 +474,6 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 		 */
 		SetPageReclaim(page);
 	} else {
-		struct lruvec *lruvec = page_lruvec(page);
 		/*
 		 * The page's writeback ends up during pagevec
 		 * We moves tha page into tail of inactive.
@@ -596,6 +603,7 @@ void release_pages(struct page **pages, int nr, int cold)
 
 		if (PageLRU(page)) {
 			struct zone *pagezone = page_zone(page);
+			struct lruvec *lruvec = page_lruvec(page);
 
 			if (pagezone != zone) {
 				if (zone)
@@ -606,7 +614,7 @@ void release_pages(struct page **pages, int nr, int cold)
 			}
 			VM_BUG_ON(!PageLRU(page));
 			__ClearPageLRU(page);
-			del_page_from_lru_list(zone, page, page_off_lru(page));
+			del_page_from_lru_list(lruvec, page, page_off_lru(page));
 		}
 
 		list_add(&page->lru, &pages_to_free);
@@ -644,6 +652,7 @@ void lru_add_page_tail(struct zone* zone,
 	int active;
 	enum lru_list lru;
 	const int file = 0;
+	struct lruvec *lruvec = page_lruvec(page);
 
 	VM_BUG_ON(!PageHead(page));
 	VM_BUG_ON(PageCompound(page_tail));
@@ -678,7 +687,7 @@ void lru_add_page_tail(struct zone* zone,
 		 * Use the standard add function to put page_tail on the list,
 		 * but then correct its position so they all end up in order.
 		 */
-		add_page_to_lru_list(zone, page_tail, lru);
+		add_page_to_lru_list(lruvec, page_tail, lru);
 		list_head = page_tail->lru.prev;
 		list_move_tail(&page_tail->lru, list_head);
 	}
@@ -688,7 +697,7 @@ void lru_add_page_tail(struct zone* zone,
 static void __pagevec_lru_add_fn(struct page *page, void *arg)
 {
 	enum lru_list lru = (enum lru_list)arg;
-	struct zone *zone = page_zone(page);
+	struct lruvec *lruvec = page_lruvec(page);
 	int file = is_file_lru(lru);
 	int active = is_active_lru(lru);
 
@@ -699,8 +708,8 @@ static void __pagevec_lru_add_fn(struct page *page, void *arg)
 	SetPageLRU(page);
 	if (active)
 		SetPageActive(page);
-	update_page_reclaim_stat(zone, page, file, active);
-	add_page_to_lru_list(zone, page, lru);
+	update_page_reclaim_stat(lruvec_zone(lruvec), page, file, active);
+	add_page_to_lru_list(lruvec, page, lru);
 }
 
 /*
mm/vmscan.c +9 -5
index f5e7046..ebb5d99 100644
@@ -1286,6 +1286,7 @@ int isolate_lru_page(struct page *page)
 
 	if (PageLRU(page)) {
 		struct zone *zone = page_zone(page);
+		struct lruvec *lruvec;
 
 		spin_lock_irq(&zone->lru_lock);
 		if (PageLRU(page)) {
@@ -1293,8 +1294,8 @@ int isolate_lru_page(struct page *page)
 			ret = 0;
 			get_page(page);
 			ClearPageLRU(page);
-
-			del_page_from_lru_list(zone, page, lru);
+			lruvec = page_lruvec(page);
+			del_page_from_lru_list(lruvec, page, lru);
 		}
 		spin_unlock_irq(&zone->lru_lock);
 	}
@@ -1339,6 +1340,7 @@ putback_inactive_pages(struct lruvec *lruvec,
 	 */
 	while (!list_empty(page_list)) {
 		struct page *page = lru_to_page(page_list);
+		struct lruvec *lruvec;
 		int lru;
 
 		VM_BUG_ON(PageLRU(page));
@@ -1351,7 +1353,9 @@ putback_inactive_pages(struct lruvec *lruvec,
 		}
 		SetPageLRU(page);
 		lru = page_lru(page);
-		add_page_to_lru_list(zone, page, lru);
+
+		lruvec = page_lruvec(page);
+		add_page_to_lru_list(lruvec, page, lru);
 		if (is_active_lru(lru)) {
 			int file = is_file_lru(lru);
 			int numpages = hpage_nr_pages(page);
@@ -1360,7 +1364,7 @@ putback_inactive_pages(struct lruvec *lruvec,
 		if (put_page_testzero(page)) {
 			__ClearPageLRU(page);
 			__ClearPageActive(page);
-			del_page_from_lru_list(zone, page, lru);
+			del_page_from_lru_list(lruvec, page, lru);
 
 			if (unlikely(PageCompound(page))) {
 				spin_unlock_irq(&zone->lru_lock);
@@ -1643,7 +1647,7 @@ static void move_active_pages_to_lru(struct zone *zone,
 		if (put_page_testzero(page)) {
 			__ClearPageLRU(page);
 			__ClearPageActive(page);
-			del_page_from_lru_list(zone, page, lru);
+			del_page_from_lru_list(lruvec, page, lru);
 
 			if (unlikely(PageCompound(page))) {
 				spin_unlock_irq(&zone->lru_lock);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 11/21
#29 lore raw

Re: [PATCH v3 11/21] mm: move page-to-lruvec translation upper

Quoted 4 lines · On Thu, 23 Feb 2012 17:52:33 +0400 Konstantin Khlebnikov…

move page_lruvec() out of add_page_to_lru_list() and del_page_from_lru_list() switch its first argument from zone to lruvec.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#30 lore raw

PATCH v3 12/21 mm: push lruvec into update_page_reclaim_stat()

In reply to Konstantin Khlebnikov

Push lruvec pointer into update_page_reclaim_stat()
* drop page argument
* drop active and file arguments, use lru instead

---
 mm/swap.c |   30 +++++++++---------------------
 1 files changed, 9 insertions(+), 21 deletions(-)
mm/swap.c +9 -21
index 0cbc558..1f5731e 100644
@@ -276,24 +276,19 @@ void rotate_reclaimable_page(struct page *page)
 	}
 }
 
-static void update_page_reclaim_stat(struct zone *zone, struct page *page,
-				     int file, int rotated)
+static void update_page_reclaim_stat(struct lruvec *lruvec, enum lru_list lru)
 {
-	struct zone_reclaim_stat *reclaim_stat;
-
-	reclaim_stat = &page_lruvec(page)->reclaim_stat;
+	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
+	int file = is_file_lru(lru);
 
 	reclaim_stat->recent_scanned[file]++;
-	if (rotated)
+	if (is_active_lru(lru))
 		reclaim_stat->recent_rotated[file]++;
 }
 
 static void __activate_page(struct page *page, void *arg)
 {
-	struct zone *zone = page_zone(page);
-
 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
-		int file = page_is_file_cache(page);
 		int lru = page_lru_base_type(page);
 		struct lruvec *lruvec = page_lruvec(page);
 
@@ -304,7 +299,7 @@ static void __activate_page(struct page *page, void *arg)
 		add_page_to_lru_list(lruvec, page, lru);
 		__count_vm_event(PGACTIVATE);
 
-		update_page_reclaim_stat(zone, page, file, 1);
+		update_page_reclaim_stat(lruvec, lru);
 	}
 }
 
@@ -443,7 +438,6 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 {
 	int lru, file;
 	bool active;
-	struct zone *zone = page_zone(page);
 	struct lruvec *lruvec;
 
 	if (!PageLRU(page))
@@ -484,7 +478,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 
 	if (active)
 		__count_vm_event(PGDEACTIVATE);
-	update_page_reclaim_stat(zone, page, file, 0);
+	update_page_reclaim_stat(lruvec, lru);
 }
 
 /*
@@ -649,9 +643,7 @@ EXPORT_SYMBOL(__pagevec_release);
 void lru_add_page_tail(struct zone* zone,
 		       struct page *page, struct page *page_tail)
 {
-	int active;
 	enum lru_list lru;
-	const int file = 0;
 	struct lruvec *lruvec = page_lruvec(page);
 
 	VM_BUG_ON(!PageHead(page));
@@ -664,13 +656,11 @@ void lru_add_page_tail(struct zone* zone,
 	if (page_evictable(page_tail, NULL)) {
 		if (PageActive(page)) {
 			SetPageActive(page_tail);
-			active = 1;
 			lru = LRU_ACTIVE_ANON;
 		} else {
-			active = 0;
 			lru = LRU_INACTIVE_ANON;
 		}
-		update_page_reclaim_stat(zone, page_tail, file, active);
+		update_page_reclaim_stat(lruvec, lru);
 	} else {
 		SetPageUnevictable(page_tail);
 		lru = LRU_UNEVICTABLE;
@@ -698,17 +688,15 @@ static void __pagevec_lru_add_fn(struct page *page, void *arg)
 {
 	enum lru_list lru = (enum lru_list)arg;
 	struct lruvec *lruvec = page_lruvec(page);
-	int file = is_file_lru(lru);
-	int active = is_active_lru(lru);
 
 	VM_BUG_ON(PageActive(page));
 	VM_BUG_ON(PageUnevictable(page));
 	VM_BUG_ON(PageLRU(page));
 
 	SetPageLRU(page);
-	if (active)
+	if (is_active_lru(lru))
 		SetPageActive(page);
-	update_page_reclaim_stat(lruvec_zone(lruvec), page, file, active);
+	update_page_reclaim_stat(lruvec, lru);
 	add_page_to_lru_list(lruvec, page, lru);
 }
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 12/21
#31 lore raw

Re: [PATCH v3 12/21] mm: push lruvec into update_page_reclaim_stat()

Quoted 5 lines · On Thu, 23 Feb 2012 17:52:38 +0400 Konstantin Khlebnikov…

Push lruvec pointer into update_page_reclaim_stat()
* drop page argument
* drop active and file arguments, use lru instead

Quoted 110 lines
---
 mm/swap.c |   30 +++++++++---------------------
 1 files changed, 9 insertions(+), 21 deletions(-)
mm/swap.c +9 -22
index 0cbc558..1f5731e 100644
@@ -276,24 +276,19 @@ void rotate_reclaimable_page(struct page *page)
 	}
 }
 
-static void update_page_reclaim_stat(struct zone *zone, struct page *page,
-				     int file, int rotated)
+static void update_page_reclaim_stat(struct lruvec *lruvec, enum lru_list lru)
 {
-	struct zone_reclaim_stat *reclaim_stat;
-
-	reclaim_stat = &page_lruvec(page)->reclaim_stat;
+	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
+	int file = is_file_lru(lru);
 
 	reclaim_stat->recent_scanned[file]++;
-	if (rotated)
+	if (is_active_lru(lru))
 		reclaim_stat->recent_rotated[file]++;
 }
 
 static void __activate_page(struct page *page, void *arg)
 {
-	struct zone *zone = page_zone(page);
-
 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
-		int file = page_is_file_cache(page);
 		int lru = page_lru_base_type(page);
 		struct lruvec *lruvec = page_lruvec(page);
 
@@ -304,7 +299,7 @@ static void __activate_page(struct page *page, void *arg)
 		add_page_to_lru_list(lruvec, page, lru);
 		__count_vm_event(PGACTIVATE);
 
-		update_page_reclaim_stat(zone, page, file, 1);
+		update_page_reclaim_stat(lruvec, lru);
 	}
 }
 
@@ -443,7 +438,6 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 {
 	int lru, file;
 	bool active;
-	struct zone *zone = page_zone(page);
 	struct lruvec *lruvec;
 
 	if (!PageLRU(page))
@@ -484,7 +478,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 
 	if (active)
 		__count_vm_event(PGDEACTIVATE);
-	update_page_reclaim_stat(zone, page, file, 0);
+	update_page_reclaim_stat(lruvec, lru);
 }
 
 /*
@@ -649,9 +643,7 @@ EXPORT_SYMBOL(__pagevec_release);
 void lru_add_page_tail(struct zone* zone,
 		       struct page *page, struct page *page_tail)
 {
-	int active;
 	enum lru_list lru;
-	const int file = 0;
 	struct lruvec *lruvec = page_lruvec(page);
 
 	VM_BUG_ON(!PageHead(page));
@@ -664,13 +656,11 @@ void lru_add_page_tail(struct zone* zone,
 	if (page_evictable(page_tail, NULL)) {
 		if (PageActive(page)) {
 			SetPageActive(page_tail);
-			active = 1;
 			lru = LRU_ACTIVE_ANON;
 		} else {
-			active = 0;
 			lru = LRU_INACTIVE_ANON;
 		}
-		update_page_reclaim_stat(zone, page_tail, file, active);
+		update_page_reclaim_stat(lruvec, lru);
 	} else {
 		SetPageUnevictable(page_tail);
 		lru = LRU_UNEVICTABLE;
@@ -698,17 +688,15 @@ static void __pagevec_lru_add_fn(struct page *page, void *arg)
 {
 	enum lru_list lru = (enum lru_list)arg;
 	struct lruvec *lruvec = page_lruvec(page);
-	int file = is_file_lru(lru);
-	int active = is_active_lru(lru);
 
 	VM_BUG_ON(PageActive(page));
 	VM_BUG_ON(PageUnevictable(page));
 	VM_BUG_ON(PageLRU(page));
 
 	SetPageLRU(page);
-	if (active)
+	if (is_active_lru(lru))
 		SetPageActive(page);
-	update_page_reclaim_stat(lruvec_zone(lruvec), page, file, active);
+	update_page_reclaim_stat(lruvec, lru);
 	add_page_to_lru_list(lruvec, page, lru);
 }
 
 
--
Quoted 5 lines

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#32 lore raw

PATCH v3 13/21 mm: push lruvecs from pagevec_lru_move_fn() to iterator

In reply to Konstantin Khlebnikov

Push lruvec pointer from pagevec_lru_move_fn() to iterator function. Push lruvec pointer into lru_add_page_tail()

---
 include/linux/swap.h |    2 +-
 mm/huge_memory.c     |    4 +++-
 mm/swap.c            |   27 +++++++++++++--------------
 3 files changed, 17 insertions(+), 16 deletions(-)
include/linux/swap.h +1 -1
index f7df3ea..8630354 100644
@@ -224,7 +224,7 @@ extern unsigned int nr_free_pagecache_pages(void);
 /* linux/mm/swap.c */
 extern void __lru_cache_add(struct page *, enum lru_list lru);
 extern void lru_cache_add_lru(struct page *, enum lru_list lru);
-extern void lru_add_page_tail(struct zone* zone,
+extern void lru_add_page_tail(struct lruvec *lruvec,
 			      struct page *page, struct page *page_tail);
 extern void activate_page(struct page *);
 extern void mark_page_accessed(struct page *);
mm/huge_memory.c +3 -1
index 91d3efb..09e7069 100644
@@ -1229,10 +1229,12 @@ static void __split_huge_page_refcount(struct page *page)
 {
 	int i;
 	struct zone *zone = page_zone(page);
+	struct lruvec *lruvec;
 	int tail_count = 0;
 
 	/* prevent PageLRU to go away from under us, and freeze lru stats */
 	spin_lock_irq(&zone->lru_lock);
+	lruvec = page_lruvec(page);
 	compound_lock(page);
 	/* complete memcg works before add pages to LRU */
 	mem_cgroup_split_huge_fixup(page);
@@ -1308,7 +1310,7 @@ static void __split_huge_page_refcount(struct page *page)
 		BUG_ON(!PageSwapBacked(page_tail));
 
 
-		lru_add_page_tail(zone, page, page_tail);
+		lru_add_page_tail(lruvec, page, page_tail);
 	}
 	atomic_sub(tail_count, &page->_count);
 	BUG_ON(atomic_read(&page->_count) <= 0);
mm/swap.c +13 -14
index 1f5731e..f7b5896 100644
@@ -204,7 +204,8 @@ void put_pages_list(struct list_head *pages)
 EXPORT_SYMBOL(put_pages_list);
 
 static void pagevec_lru_move_fn(struct pagevec *pvec,
-				void (*move_fn)(struct page *page, void *arg),
+				void (*move_fn)(struct lruvec *lruvec,
+						struct page *page, void *arg),
 				void *arg)
 {
 	int i;
@@ -214,6 +215,7 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,
 	for (i = 0; i < pagevec_count(pvec); i++) {
 		struct page *page = pvec->pages[i];
 		struct zone *pagezone = page_zone(page);
+		struct lruvec *lruvec;
 
 		if (pagezone != zone) {
 			if (zone)
@@ -222,7 +224,8 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,
 			spin_lock_irqsave(&zone->lru_lock, flags);
 		}
 
-		(*move_fn)(page, arg);
+		lruvec = page_lruvec(page);
+		(*move_fn)(lruvec, page, arg);
 	}
 	if (zone)
 		spin_unlock_irqrestore(&zone->lru_lock, flags);
@@ -230,13 +233,13 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,
 	pagevec_reinit(pvec);
 }
 
-static void pagevec_move_tail_fn(struct page *page, void *arg)
+static void pagevec_move_tail_fn(struct lruvec *lruvec,
+				 struct page *page, void *arg)
 {
 	int *pgmoved = arg;
 
 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 		enum lru_list lru = page_lru_base_type(page);
-		struct lruvec *lruvec = page_lruvec(page);
 
 		list_move_tail(&page->lru, &lruvec->pages_lru[lru]);
 		(*pgmoved)++;
@@ -286,11 +289,10 @@ static void update_page_reclaim_stat(struct lruvec *lruvec, enum lru_list lru)
 		reclaim_stat->recent_rotated[file]++;
 }
 
-static void __activate_page(struct page *page, void *arg)
+static void __activate_page(struct lruvec *lruvec, struct page *page, void *arg)
 {
 	if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 		int lru = page_lru_base_type(page);
-		struct lruvec *lruvec = page_lruvec(page);
 
 		del_page_from_lru_list(lruvec, page, lru);
 
@@ -434,11 +436,10 @@ void add_page_to_unevictable_list(struct page *page)
  * be write it out by flusher threads as this is much more effective
  * than the single-page writeout from reclaim.
  */
-static void lru_deactivate_fn(struct page *page, void *arg)
+static void lru_deactivate_fn(struct lruvec *lruvec, struct page *page, void *arg)
 {
 	int lru, file;
 	bool active;
-	struct lruvec *lruvec;
 
 	if (!PageLRU(page))
 		return;
@@ -454,7 +455,6 @@ static void lru_deactivate_fn(struct page *page, void *arg)
 
 	file = page_is_file_cache(page);
 	lru = page_lru_base_type(page);
-	lruvec = page_lruvec(page);
 	del_page_from_lru_list(lruvec, page, lru + active);
 	ClearPageActive(page);
 	ClearPageReferenced(page);
@@ -640,16 +640,15 @@ EXPORT_SYMBOL(__pagevec_release);
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 /* used by __split_huge_page_refcount() */
-void lru_add_page_tail(struct zone* zone,
+void lru_add_page_tail(struct lruvec *lruvec,
 		       struct page *page, struct page *page_tail)
 {
 	enum lru_list lru;
-	struct lruvec *lruvec = page_lruvec(page);
 
 	VM_BUG_ON(!PageHead(page));
 	VM_BUG_ON(PageCompound(page_tail));
 	VM_BUG_ON(PageLRU(page_tail));
-	VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&zone->lru_lock));
+	VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&lruvec_zone(lruvec)->lru_lock));
 
 	SetPageLRU(page_tail);
 
@@ -684,10 +683,10 @@ void lru_add_page_tail(struct zone* zone,
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
-static void __pagevec_lru_add_fn(struct page *page, void *arg)
+static void __pagevec_lru_add_fn(struct lruvec *lruvec,
+				 struct page *page, void *arg)
 {
 	enum lru_list lru = (enum lru_list)arg;
-	struct lruvec *lruvec = page_lruvec(page);
 
 	VM_BUG_ON(PageActive(page));
 	VM_BUG_ON(PageUnevictable(page));

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 13/21
#33 lore raw

Re: [PATCH v3 13/21] mm: push lruvecs from pagevec_lru_move_fn() to iterator

Quoted 4 lines · On Thu, 23 Feb 2012 17:52:42 +0400 Konstantin Khlebnikov…

Push lruvec pointer from pagevec_lru_move_fn() to iterator function. Push lruvec pointer into lru_add_page_tail()

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#34 lore raw

PATCH v3 14/21 mm: introduce lruvec locking primitives

In reply to Konstantin Khlebnikov

This is initial preparation for lru_lock splitting.

This locking primites designed to hide splitted nature of lru_lock and to avoid overhead for non-splitted lru_lock in non-memcg case.

* Lock via lruvec reference

lock_lruvec(lruvec, flags)
lock_lruvec_irq(lruvec)

* Lock via page reference

lock_page_lruvec(page, flags)
lock_page_lruvec_irq(page)
relock_page_lruvec(lruvec, page, flags) relock_page_lruvec_irq(lruvec, page) __relock_page_lruvec(lruvec, page) ( lruvec != NULL, page in same zone )

They always returns pointer to some locked lruvec, page anyway can be not in lru, PageLRU() sign is stable while we hold returned lruvec lock. Caller must guarantee page to lruvec reference validity.

* Lock via page, without stable page reference

__lock_page_lruvec_irq(&lruvec, page)

It returns true of lruvec succesfully locked and PageLRU is set. Initial lruvec can be NULL. Consequent calls must be in the same zone.

* Unlock

unlock_lruvec(lruvec, flags)
unlock_lruvec_irq(lruvec)

* Wait

wait_lruvec_unlock(lruvec)
Wait for lruvec unlock, caller must have stable reference to lruvec.

__wait_lruvec_unlock(lruvec)
Wait for lruvec unlock before locking other lrulock for same page, nothing if there only one possible lruvec per page.
Used at page-to-lruvec reference switching to stabilize PageLRU sign.

---
 mm/huge_memory.c |    8 +-
 mm/internal.h    |  176 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 mm/memcontrol.c  |   14 ++--
 mm/swap.c        |   58 ++++++------------
 mm/vmscan.c      |   77 ++++++++++--------------
 5 files changed, 237 insertions(+), 96 deletions(-)
mm/huge_memory.c +3 -5
index 09e7069..74996b8 100644
@@ -1228,13 +1228,11 @@ static int __split_huge_page_splitting(struct page *page,
 static void __split_huge_page_refcount(struct page *page)
 {
 	int i;
-	struct zone *zone = page_zone(page);
 	struct lruvec *lruvec;
 	int tail_count = 0;
 
 	/* prevent PageLRU to go away from under us, and freeze lru stats */
-	spin_lock_irq(&zone->lru_lock);
-	lruvec = page_lruvec(page);
+	lruvec = lock_page_lruvec_irq(page);
 	compound_lock(page);
 	/* complete memcg works before add pages to LRU */
 	mem_cgroup_split_huge_fixup(page);
@@ -1316,11 +1314,11 @@ static void __split_huge_page_refcount(struct page *page)
 	BUG_ON(atomic_read(&page->_count) <= 0);
 
 	__dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
-	__mod_zone_page_state(zone, NR_ANON_PAGES, HPAGE_PMD_NR);
+	__mod_zone_page_state(lruvec_zone(lruvec), NR_ANON_PAGES, HPAGE_PMD_NR);
 
 	ClearPageCompound(page);
 	compound_unlock(page);
-	spin_unlock_irq(&zone->lru_lock);
+	unlock_lruvec_irq(lruvec);
 
 	for (i = 1; i < HPAGE_PMD_NR; i++) {
 		struct page *page_tail = page + i;
mm/internal.h +176 -0

182 lines not on this page · raw message

mm/memcontrol.c +8 -6
index 83fa99b..aed1360 100644
@@ -3487,12 +3487,14 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
 	struct list_head *list;
 	struct page *busy;
 	struct zone *zone;
+	struct lruvec *lruvec;
 	int ret = 0;
 
 	zone = &NODE_DATA(node)->node_zones[zid];
 	mz = mem_cgroup_zoneinfo(memcg, node, zid);
-	list = &mz->lruvec.pages_lru[lru];
-	loop = mz->lruvec.pages_count[lru];
+	lruvec = &mz->lruvec;
+	list = &lruvec->pages_lru[lru];
+	loop = lruvec->pages_count[lru];
 	/* give some margin against EBUSY etc...*/
 	loop += 256;
 	busy = NULL;
@@ -3501,19 +3503,19 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
 		struct page *page;
 
 		ret = 0;
-		spin_lock_irqsave(&zone->lru_lock, flags);
+		lock_lruvec(lruvec, &flags);
 		if (list_empty(list)) {
-			spin_unlock_irqrestore(&zone->lru_lock, flags);
+			unlock_lruvec(lruvec, &flags);
 			break;
 		}
 		page = list_entry(list->prev, struct page, lru);
 		if (busy == page) {
 			list_move(&page->lru, list);
 			busy = NULL;
-			spin_unlock_irqrestore(&zone->lru_lock, flags);
+			unlock_lruvec(lruvec, &flags);
 			continue;
 		}
-		spin_unlock_irqrestore(&zone->lru_lock, flags);
+		unlock_lruvec(lruvec, &flags);
 
 		pc = lookup_page_cgroup(page);
 
mm/swap.c +18 -40

125 lines not on this page · raw message

mm/vmscan.c +32 -45

248 lines not on this page · raw message

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 14/21
#35 lore raw

Re: [PATCH v3 14/21] mm: introduce lruvec locking primitives

Quoted 39 lines · On Thu, 23 Feb 2012 17:52:47 +0400 Konstantin Khlebnikov…

This is initial preparation for lru_lock splitting.

This locking primites designed to hide splitted nature of lru_lock and to avoid overhead for non-splitted lru_lock in non-memcg case.

* Lock via lruvec reference

lock_lruvec(lruvec, flags)
lock_lruvec_irq(lruvec)

* Lock via page reference

lock_page_lruvec(page, flags)
lock_page_lruvec_irq(page)
relock_page_lruvec(lruvec, page, flags) relock_page_lruvec_irq(lruvec, page) __relock_page_lruvec(lruvec, page) ( lruvec != NULL, page in same zone )

They always returns pointer to some locked lruvec, page anyway can be not in lru, PageLRU() sign is stable while we hold returned lruvec lock. Caller must guarantee page to lruvec reference validity.

* Lock via page, without stable page reference

__lock_page_lruvec_irq(&lruvec, page)

It returns true of lruvec succesfully locked and PageLRU is set. Initial lruvec can be NULL. Consequent calls must be in the same zone.

* Unlock

unlock_lruvec(lruvec, flags)
unlock_lruvec_irq(lruvec)

* Wait

wait_lruvec_unlock(lruvec)
Wait for lruvec unlock, caller must have stable reference to lruvec.

Quoted 6 lines

__wait_lruvec_unlock(lruvec)
Wait for lruvec unlock before locking other lrulock for same page, nothing if there only one possible lruvec per page.
Used at page-to-lruvec reference switching to stabilize PageLRU sign.

O.K. I like this.

Hmm....Could you add a comment in memcg part ? (see below)

Quoted 89 lines
---
 mm/huge_memory.c |    8 +-
 mm/internal.h    |  176 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 mm/memcontrol.c  |   14 ++--
 mm/swap.c        |   58 ++++++------------
 mm/vmscan.c      |   77 ++++++++++--------------
 5 files changed, 237 insertions(+), 96 deletions(-)
mm/huge_memory.c +3 -5
index 09e7069..74996b8 100644
@@ -1228,13 +1228,11 @@ static int __split_huge_page_splitting(struct page *page,
 static void __split_huge_page_refcount(struct page *page)
 {
 	int i;
-	struct zone *zone = page_zone(page);
 	struct lruvec *lruvec;
 	int tail_count = 0;
 
 	/* prevent PageLRU to go away from under us, and freeze lru stats */
-	spin_lock_irq(&zone->lru_lock);
-	lruvec = page_lruvec(page);
+	lruvec = lock_page_lruvec_irq(page);
 	compound_lock(page);
 	/* complete memcg works before add pages to LRU */
 	mem_cgroup_split_huge_fixup(page);
@@ -1316,11 +1314,11 @@ static void __split_huge_page_refcount(struct page *page)
 	BUG_ON(atomic_read(&page->_count) <= 0);
 
 	__dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
-	__mod_zone_page_state(zone, NR_ANON_PAGES, HPAGE_PMD_NR);
+	__mod_zone_page_state(lruvec_zone(lruvec), NR_ANON_PAGES, HPAGE_PMD_NR);
 
 	ClearPageCompound(page);
 	compound_unlock(page);
-	spin_unlock_irq(&zone->lru_lock);
+	unlock_lruvec_irq(lruvec);
 
 	for (i = 1; i < HPAGE_PMD_NR; i++) {
 		struct page *page_tail = page + i;
mm/internal.h +40 -0
index ef49dbf..9454752 100644
@@ -13,6 +13,182 @@
 
 #include <linux/mm.h>
 
+static inline void lock_lruvec(struct lruvec *lruvec, unsigned long *flags)
+{
+	spin_lock_irqsave(&lruvec_zone(lruvec)->lru_lock, *flags);
+}
+
+static inline void lock_lruvec_irq(struct lruvec *lruvec)
+{
+	spin_lock_irq(&lruvec_zone(lruvec)->lru_lock);
+}
+
+static inline void unlock_lruvec(struct lruvec *lruvec, unsigned long *flags)
+{
+	spin_unlock_irqrestore(&lruvec_zone(lruvec)->lru_lock, *flags);
+}
+
+static inline void unlock_lruvec_irq(struct lruvec *lruvec)
+{
+	spin_unlock_irq(&lruvec_zone(lruvec)->lru_lock);
+}
+
+static inline void wait_lruvec_unlock(struct lruvec *lruvec)
+{
+	spin_unlock_wait(&lruvec_zone(lruvec)->lru_lock);
+}
+
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+
+/* Dynamic page to lruvec mapping */
+
+/* Lock other lruvec for other page in the same zone */
+static inline struct lruvec *__relock_page_lruvec(struct lruvec *locked_lruvec,
+						  struct page *page)
+{
+	/* Currenyly only one lru_lock per-zone */
+	return page_lruvec(page);
+}
+
+static inline struct lruvec *relock_page_lruvec_irq(struct lruvec *lruvec,
+						    struct page *page)
+{

mm/internal.h line 65

Quoted 10 lines
mm/internal.h +10 -0
+	struct zone *zone = page_zone(page);
+
+	if (!lruvec) {
+		spin_lock_irq(&zone->lru_lock);
+	} else if (zone != lruvec_zone(lruvec)) {
+		unlock_lruvec_irq(lruvec);
+		spin_lock_irq(&zone->lru_lock);
+	}
+	return page_lruvec(page);
+}

Could you add comments/caution to the caller

- !PageLRU(page) case ?
- Can the caller assume page_lruvec(page) == lruvec ? If no, which lru_vec is locked ?

etc...

Quoted 5 lines
mm/internal.h +5 -0
+
+static inline struct lruvec *relock_page_lruvec(struct lruvec *lruvec,
+						struct page *page,
+						unsigned long *flags)
+{

mm/internal.h line 80

Quoted 10 lines
mm/internal.h +10 -0
+	struct zone *zone = page_zone(page);
+
+	if (!lruvec) {
+		spin_lock_irqsave(&zone->lru_lock, *flags);
+	} else if (zone != lruvec_zone(lruvec)) {
+		unlock_lruvec(lruvec, flags);
+		spin_lock_irqsave(&zone->lru_lock, *flags);
+	}
+	return page_lruvec(page);
+}

Same here.

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#36 lore raw

Re: [PATCH v3 14/21] mm: introduce lruvec locking primitives

Quoted 159 lines · KAMEZAWA Hiroyuki
Quoted 39 lines · On Thu, 23 Feb 2012 17:52:47 +0400 Konstantin…

This is initial preparation for lru_lock splitting.

This locking primites designed to hide splitted nature of lru_lock and to avoid overhead for non-splitted lru_lock in non-memcg case.

* Lock via lruvec reference

lock_lruvec(lruvec, flags)
lock_lruvec_irq(lruvec)

* Lock via page reference

lock_page_lruvec(page, flags)
lock_page_lruvec_irq(page)
relock_page_lruvec(lruvec, page, flags) relock_page_lruvec_irq(lruvec, page) __relock_page_lruvec(lruvec, page) ( lruvec != NULL, page in same zone )

They always returns pointer to some locked lruvec, page anyway can be not in lru, PageLRU() sign is stable while we hold returned lruvec lock. Caller must guarantee page to lruvec reference validity.

* Lock via page, without stable page reference

__lock_page_lruvec_irq(&lruvec, page)

It returns true of lruvec succesfully locked and PageLRU is set. Initial lruvec can be NULL. Consequent calls must be in the same zone.

* Unlock

unlock_lruvec(lruvec, flags)
unlock_lruvec_irq(lruvec)

* Wait

wait_lruvec_unlock(lruvec)
Wait for lruvec unlock, caller must have stable reference to lruvec.

Quoted 6 lines

__wait_lruvec_unlock(lruvec)
Wait for lruvec unlock before locking other lrulock for same page, nothing if there only one possible lruvec per page.
Used at page-to-lruvec reference switching to stabilize PageLRU sign.

O.K. I like this.

Hmm....Could you add a comment in memcg part ? (see below)

Quoted 89 lines
---
  mm/huge_memory.c |    8 +-
  mm/internal.h    |  176 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
  mm/memcontrol.c  |   14 ++--
  mm/swap.c        |   58 ++++++------------
  mm/vmscan.c      |   77 ++++++++++--------------
  5 files changed, 237 insertions(+), 96 deletions(-)
mm/huge_memory.c +3 -5
index 09e7069..74996b8 100644
@@ -1228,13 +1228,11 @@ static int __split_huge_page_splitting(struct page *page,
  static void __split_huge_page_refcount(struct page *page)
  {
  	int i;
-	struct zone *zone = page_zone(page);
  	struct lruvec *lruvec;
  	int tail_count = 0;
 
  	/* prevent PageLRU to go away from under us, and freeze lru stats */
-	spin_lock_irq(&zone->lru_lock);
-	lruvec = page_lruvec(page);
+	lruvec = lock_page_lruvec_irq(page);
  	compound_lock(page);
  	/* complete memcg works before add pages to LRU */
  	mem_cgroup_split_huge_fixup(page);
@@ -1316,11 +1314,11 @@ static void __split_huge_page_refcount(struct page *page)
  	BUG_ON(atomic_read(&page->_count)<= 0);
 
  	__dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
-	__mod_zone_page_state(zone, NR_ANON_PAGES, HPAGE_PMD_NR);
+	__mod_zone_page_state(lruvec_zone(lruvec), NR_ANON_PAGES, HPAGE_PMD_NR);
 
  	ClearPageCompound(page);
  	compound_unlock(page);
-	spin_unlock_irq(&zone->lru_lock);
+	unlock_lruvec_irq(lruvec);
 
  	for (i = 1; i<  HPAGE_PMD_NR; i++) {
  		struct page *page_tail = page + i;
mm/internal.h +40 -0
index ef49dbf..9454752 100644
@@ -13,6 +13,182 @@
 
  #include<linux/mm.h>
 
+static inline void lock_lruvec(struct lruvec *lruvec, unsigned long *flags)
+{
+	spin_lock_irqsave(&lruvec_zone(lruvec)->lru_lock, *flags);
+}
+
+static inline void lock_lruvec_irq(struct lruvec *lruvec)
+{
+	spin_lock_irq(&lruvec_zone(lruvec)->lru_lock);
+}
+
+static inline void unlock_lruvec(struct lruvec *lruvec, unsigned long *flags)
+{
+	spin_unlock_irqrestore(&lruvec_zone(lruvec)->lru_lock, *flags);
+}
+
+static inline void unlock_lruvec_irq(struct lruvec *lruvec)
+{
+	spin_unlock_irq(&lruvec_zone(lruvec)->lru_lock);
+}
+
+static inline void wait_lruvec_unlock(struct lruvec *lruvec)
+{
+	spin_unlock_wait(&lruvec_zone(lruvec)->lru_lock);
+}
+
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+
+/* Dynamic page to lruvec mapping */
+
+/* Lock other lruvec for other page in the same zone */
+static inline struct lruvec *__relock_page_lruvec(struct lruvec *locked_lruvec,
+						  struct page *page)
+{
+	/* Currenyly only one lru_lock per-zone */
+	return page_lruvec(page);
+}
+
+static inline struct lruvec *relock_page_lruvec_irq(struct lruvec *lruvec,
+						    struct page *page)
+{

mm/internal.h line 65

Quoted 10 lines
mm/internal.h +10 -0
+	struct zone *zone = page_zone(page);
+
+	if (!lruvec) {
+		spin_lock_irq(&zone->lru_lock);
+	} else if (zone != lruvec_zone(lruvec)) {
+		unlock_lruvec_irq(lruvec);
+		spin_lock_irq(&zone->lru_lock);
+	}
+	return page_lruvec(page);
+}

Could you add comments/caution to the caller

Quoted 2 lines

- !PageLRU(page) case ?
- Can the caller assume page_lruvec(page) == lruvec ? If no, which lru_vec is locked ?

Yes, caller can assume page_lruvec(page) == lruvec. And PageLRU() is stable, it means it stays true or false while this lruvec is locked.

Quoted 23 lines

etc...

Quoted 5 lines
mm/internal.h +5 -0
+
+static inline struct lruvec *relock_page_lruvec(struct lruvec *lruvec,
+						struct page *page,
+						unsigned long *flags)
+{

mm/internal.h line 80

Quoted 10 lines
mm/internal.h +10 -0
+	struct zone *zone = page_zone(page);
+
+	if (!lruvec) {
+		spin_lock_irqsave(&zone->lru_lock, *flags);
+	} else if (zone != lruvec_zone(lruvec)) {
+		unlock_lruvec(lruvec, flags);
+		spin_lock_irqsave(&zone->lru_lock, *flags);
+	}
+	return page_lruvec(page);
+}

Same here.

Quoted 3 lines

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#37 lore raw

PATCH v3 15/21 mm: handle lruvec relocks on lumpy reclaim

In reply to Konstantin Khlebnikov

Prepare for lock splitting in lumly reclaim logic. Now move_active_pages_to_lru() and putback_inactive_pages() can put pages into different lruvecs.

* relock book before SetPageLRU()
* update reclaim_stat pointer after relocks
* return currently locked lruvec

---
 mm/vmscan.c |   45 +++++++++++++++++++++++++++++++++------------
 1 files changed, 33 insertions(+), 12 deletions(-)
mm/vmscan.c +33 -12
index a3941d1..6eeeb4b 100644
@@ -1114,6 +1114,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 		unsigned long *nr_scanned, struct scan_control *sc,
 		isolate_mode_t mode, int active, int file)
 {
+	struct lruvec *cursor_lruvec = lruvec;
 	struct list_head *src;
 	unsigned long nr_taken = 0;
 	unsigned long nr_lumpy_taken = 0;
@@ -1197,14 +1198,17 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 			    !PageSwapCache(cursor_page))
 				break;
 
+			/* Switch cursor_lruvec lock for lumpy isolate */
+			if (!__lock_page_lruvec_irq(&cursor_lruvec,
+						    cursor_page))
+				continue;
+
 			if (__isolate_lru_page(cursor_page, mode, file) == 0) {
 				unsigned int isolated_pages;
-				struct lruvec *cursor_lruvec;
 				int cursor_lru = page_lru(cursor_page);
 
 				list_move(&cursor_page->lru, dst);
 				isolated_pages = hpage_nr_pages(cursor_page);
-				cursor_lruvec = page_lruvec(cursor_page);
 				cursor_lruvec->pages_count[cursor_lru] -=
 								isolated_pages;
 				VM_BUG_ON((long)cursor_lruvec->
@@ -1235,6 +1239,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 			}
 		}
 
+		/* Restore original lruvec lock */
+		cursor_lruvec = __relock_page_lruvec(cursor_lruvec, page);
+
 		/* If we break out of the loop above, lumpy reclaim failed */
 		if (pfn < end_pfn)
 			nr_lumpy_failed++;
@@ -1325,7 +1332,10 @@ static int too_many_isolated(struct zone *zone, int file,
 	return isolated > inactive;
 }
 
-static noinline_for_stack void
+/*
+ * Returns currently locked lruvec
+ */
+static noinline_for_stack struct lruvec *
 putback_inactive_pages(struct lruvec *lruvec,
 		       struct list_head *page_list)
 {
@@ -1347,10 +1357,13 @@ putback_inactive_pages(struct lruvec *lruvec,
 			lock_lruvec_irq(lruvec);
 			continue;
 		}
+
+		lruvec = __relock_page_lruvec(lruvec, page);
+		reclaim_stat = &lruvec->reclaim_stat;
+
 		SetPageLRU(page);
 		lru = page_lru(page);
 
-		lruvec = page_lruvec(page);
 		add_page_to_lru_list(lruvec, page, lru);
 		if (is_active_lru(lru)) {
 			int file = is_file_lru(lru);
@@ -1375,6 +1388,8 @@ putback_inactive_pages(struct lruvec *lruvec,
 	 * To save our caller's stack, now use input list for pages to free.
 	 */
 	list_splice(&pages_to_free, page_list);
+
+	return lruvec;
 }
 
 static noinline_for_stack void
@@ -1544,7 +1559,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
 		__count_vm_events(KSWAPD_STEAL, nr_reclaimed);
 	__count_zone_vm_events(PGSTEAL, zone, nr_reclaimed);
 
-	putback_inactive_pages(lruvec, &page_list);
+	lruvec = putback_inactive_pages(lruvec, &page_list);
 
 	__mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon);
 	__mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file);
@@ -1603,12 +1618,15 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
  *
  * The downside is that we have to touch page->_count against each page.
  * But we had to alter page->flags anyway.
+ *
+ * Returns currently locked lruvec
  */
 
-static void move_active_pages_to_lru(struct lruvec *lruvec,
-				     struct list_head *list,
-				     struct list_head *pages_to_free,
-				     enum lru_list lru)
+static struct lruvec *
+move_active_pages_to_lru(struct lruvec *lruvec,
+			 struct list_head *list,
+			 struct list_head *pages_to_free,
+			 enum lru_list lru)
 {
 	unsigned long pgmoved = 0;
 	struct page *page;
@@ -1630,10 +1648,11 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
 
 		page = lru_to_page(list);
 
+		lruvec = __relock_page_lruvec(lruvec, page);
+
 		VM_BUG_ON(PageLRU(page));
 		SetPageLRU(page);
 
-		lruvec = page_lruvec(page);
 		list_move(&page->lru, &lruvec->pages_lru[lru]);
 		numpages = hpage_nr_pages(page);
 		lruvec->pages_count[lru] += numpages;
@@ -1655,6 +1674,8 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
 	__mod_zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru, pgmoved);
 	if (!is_active_lru(lru))
 		__count_vm_events(PGDEACTIVATE, pgmoved);
+
+	return lruvec;
 }
 
 static void shrink_active_list(unsigned long nr_to_scan,
@@ -1744,9 +1765,9 @@ static void shrink_active_list(unsigned long nr_to_scan,
 	 */
 	reclaim_stat->recent_rotated[file] += nr_rotated;
 
-	move_active_pages_to_lru(lruvec, &l_active, &l_hold,
+	lruvec = move_active_pages_to_lru(lruvec, &l_active, &l_hold,
 						LRU_ACTIVE + file * LRU_FILE);
-	move_active_pages_to_lru(lruvec, &l_inactive, &l_hold,
+	lruvec = move_active_pages_to_lru(lruvec, &l_inactive, &l_hold,
 						LRU_BASE   + file * LRU_FILE);
 	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
 	unlock_lruvec_irq(lruvec);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#38 lore raw

Re: [PATCH v3 15/21] mm: handle lruvec relocks on lumpy reclaim

Quoted 5 lines · On Thu, 23 Feb 2012 17:52:52 +0400 Konstantin Khlebnikov…

Prepare for lock splitting in lumly reclaim logic. Now move_active_pages_to_lru() and putback_inactive_pages() can put pages into different lruvecs.

* relock book before SetPageLRU()

lruvec ?

Quoted 140 lines

* update reclaim_stat pointer after relocks
* return currently locked lruvec

---
 mm/vmscan.c |   45 +++++++++++++++++++++++++++++++++------------
 1 files changed, 33 insertions(+), 12 deletions(-)
mm/vmscan.c +31 -10
index a3941d1..6eeeb4b 100644
@@ -1114,6 +1114,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 		unsigned long *nr_scanned, struct scan_control *sc,
 		isolate_mode_t mode, int active, int file)
 {
+	struct lruvec *cursor_lruvec = lruvec;
 	struct list_head *src;
 	unsigned long nr_taken = 0;
 	unsigned long nr_lumpy_taken = 0;
@@ -1197,14 +1198,17 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 			    !PageSwapCache(cursor_page))
 				break;
 
+			/* Switch cursor_lruvec lock for lumpy isolate */
+			if (!__lock_page_lruvec_irq(&cursor_lruvec,
+						    cursor_page))
+				continue;
+
 			if (__isolate_lru_page(cursor_page, mode, file) == 0) {
 				unsigned int isolated_pages;
-				struct lruvec *cursor_lruvec;
 				int cursor_lru = page_lru(cursor_page);
 
 				list_move(&cursor_page->lru, dst);
 				isolated_pages = hpage_nr_pages(cursor_page);
-				cursor_lruvec = page_lruvec(cursor_page);
 				cursor_lruvec->pages_count[cursor_lru] -=
 								isolated_pages;
 				VM_BUG_ON((long)cursor_lruvec->
@@ -1235,6 +1239,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 			}
 		}
 
+		/* Restore original lruvec lock */
+		cursor_lruvec = __relock_page_lruvec(cursor_lruvec, page);
+
 		/* If we break out of the loop above, lumpy reclaim failed */
 		if (pfn < end_pfn)
 			nr_lumpy_failed++;
@@ -1325,7 +1332,10 @@ static int too_many_isolated(struct zone *zone, int file,
 	return isolated > inactive;
 }
 
-static noinline_for_stack void
+/*
+ * Returns currently locked lruvec
+ */
+static noinline_for_stack struct lruvec *
 putback_inactive_pages(struct lruvec *lruvec,
 		       struct list_head *page_list)
 {
@@ -1347,10 +1357,13 @@ putback_inactive_pages(struct lruvec *lruvec,
 			lock_lruvec_irq(lruvec);
 			continue;
 		}
+
+		lruvec = __relock_page_lruvec(lruvec, page);
+		reclaim_stat = &lruvec->reclaim_stat;
+
 		SetPageLRU(page);
 		lru = page_lru(page);
 
-		lruvec = page_lruvec(page);
 		add_page_to_lru_list(lruvec, page, lru);
 		if (is_active_lru(lru)) {
 			int file = is_file_lru(lru);
@@ -1375,6 +1388,8 @@ putback_inactive_pages(struct lruvec *lruvec,
 	 * To save our caller's stack, now use input list for pages to free.
 	 */
 	list_splice(&pages_to_free, page_list);
+
+	return lruvec;
 }
 
 static noinline_for_stack void
@@ -1544,7 +1559,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
 		__count_vm_events(KSWAPD_STEAL, nr_reclaimed);
 	__count_zone_vm_events(PGSTEAL, zone, nr_reclaimed);
 
-	putback_inactive_pages(lruvec, &page_list);
+	lruvec = putback_inactive_pages(lruvec, &page_list);
 
 	__mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon);
 	__mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file);
@@ -1603,12 +1618,15 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
  *
  * The downside is that we have to touch page->_count against each page.
  * But we had to alter page->flags anyway.
+ *
+ * Returns currently locked lruvec
  */
 
-static void move_active_pages_to_lru(struct lruvec *lruvec,
-				     struct list_head *list,
-				     struct list_head *pages_to_free,
-				     enum lru_list lru)
+static struct lruvec *
+move_active_pages_to_lru(struct lruvec *lruvec,
+			 struct list_head *list,
+			 struct list_head *pages_to_free,
+			 enum lru_list lru)
 {
 	unsigned long pgmoved = 0;
 	struct page *page;
@@ -1630,10 +1648,11 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
 
 		page = lru_to_page(list);
 
+		lruvec = __relock_page_lruvec(lruvec, page);
+
 		VM_BUG_ON(PageLRU(page));
 		SetPageLRU(page);
 
-		lruvec = page_lruvec(page);
 		list_move(&page->lru, &lruvec->pages_lru[lru]);
 		numpages = hpage_nr_pages(page);
 		lruvec->pages_count[lru] += numpages;
@@ -1655,6 +1674,8 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
 	__mod_zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru, pgmoved);
 	if (!is_active_lru(lru))
 		__count_vm_events(PGDEACTIVATE, pgmoved);
+
+	return lruvec;
 }
 
 static void shrink_active_list(unsigned long nr_to_scan,
@@ -1744,9 +1765,9 @@ static void shrink_active_list(unsigned long nr_to_scan,
 	 */

mm/vmscan.c static void shrink_active_list(unsigned long nr_to_scan, · line 1773

Quoted 10 lines
mm/vmscan.c +2 -2
 	reclaim_stat->recent_rotated[file] += nr_rotated;
 
-	move_active_pages_to_lru(lruvec, &l_active, &l_hold,
+	lruvec = move_active_pages_to_lru(lruvec, &l_active, &l_hold,
 						LRU_ACTIVE + file * LRU_FILE);
-	move_active_pages_to_lru(lruvec, &l_inactive, &l_hold,
+	lruvec = move_active_pages_to_lru(lruvec, &l_inactive, &l_hold,
 						LRU_BASE   + file * LRU_FILE);
 	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
 	unlock_lruvec_irq(lruvec);

Hmm...could you add comments to each function as
"The caller should _lock_ lruvec before calling this functions.
This function returns a lruvec with _locked_. It may be different from passed one. And The callser should unlock lruvec"

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#39 lore raw

Re: [PATCH v3 15/21] mm: handle lruvec relocks on lumpy reclaim

Quoted 10 lines · KAMEZAWA Hiroyuki
Quoted 5 lines · On Thu, 23 Feb 2012 17:52:52 +0400 Konstantin…

Prepare for lock splitting in lumly reclaim logic. Now move_active_pages_to_lru() and putback_inactive_pages() can put pages into different lruvecs.

* relock book before SetPageLRU()

lruvec ?

yeah, this came from v1

Quoted 142 lines
Quoted 142 lines

* update reclaim_stat pointer after relocks
* return currently locked lruvec

---
  mm/vmscan.c |   45 +++++++++++++++++++++++++++++++++------------
  1 files changed, 33 insertions(+), 12 deletions(-)
mm/vmscan.c +31 -10
index a3941d1..6eeeb4b 100644
@@ -1114,6 +1114,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
  		unsigned long *nr_scanned, struct scan_control *sc,
  		isolate_mode_t mode, int active, int file)
  {
+	struct lruvec *cursor_lruvec = lruvec;
  	struct list_head *src;
  	unsigned long nr_taken = 0;
  	unsigned long nr_lumpy_taken = 0;
@@ -1197,14 +1198,17 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
  			    !PageSwapCache(cursor_page))
  				break;
 
+			/* Switch cursor_lruvec lock for lumpy isolate */
+			if (!__lock_page_lruvec_irq(&cursor_lruvec,
+						    cursor_page))
+				continue;
+
  			if (__isolate_lru_page(cursor_page, mode, file) == 0) {
  				unsigned int isolated_pages;
-				struct lruvec *cursor_lruvec;
  				int cursor_lru = page_lru(cursor_page);
 
  				list_move(&cursor_page->lru, dst);
  				isolated_pages = hpage_nr_pages(cursor_page);
-				cursor_lruvec = page_lruvec(cursor_page);
  				cursor_lruvec->pages_count[cursor_lru] -=
  								isolated_pages;
  				VM_BUG_ON((long)cursor_lruvec->
@@ -1235,6 +1239,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
  			}
  		}
 
+		/* Restore original lruvec lock */
+		cursor_lruvec = __relock_page_lruvec(cursor_lruvec, page);
+
  		/* If we break out of the loop above, lumpy reclaim failed */
  		if (pfn<  end_pfn)
  			nr_lumpy_failed++;
@@ -1325,7 +1332,10 @@ static int too_many_isolated(struct zone *zone, int file,
  	return isolated>  inactive;
  }
 
-static noinline_for_stack void
+/*
+ * Returns currently locked lruvec
+ */
+static noinline_for_stack struct lruvec *
  putback_inactive_pages(struct lruvec *lruvec,
  		       struct list_head *page_list)
  {
@@ -1347,10 +1357,13 @@ putback_inactive_pages(struct lruvec *lruvec,
  			lock_lruvec_irq(lruvec);
  			continue;
  		}
+
+		lruvec = __relock_page_lruvec(lruvec, page);
+		reclaim_stat =&lruvec->reclaim_stat;
+
  		SetPageLRU(page);
  		lru = page_lru(page);
 
-		lruvec = page_lruvec(page);
  		add_page_to_lru_list(lruvec, page, lru);
  		if (is_active_lru(lru)) {
  			int file = is_file_lru(lru);
@@ -1375,6 +1388,8 @@ putback_inactive_pages(struct lruvec *lruvec,
  	 * To save our caller's stack, now use input list for pages to free.
  	 */
  	list_splice(&pages_to_free, page_list);
+
+	return lruvec;
  }
 
  static noinline_for_stack void
@@ -1544,7 +1559,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
  		__count_vm_events(KSWAPD_STEAL, nr_reclaimed);
  	__count_zone_vm_events(PGSTEAL, zone, nr_reclaimed);
 
-	putback_inactive_pages(lruvec,&page_list);
+	lruvec = putback_inactive_pages(lruvec,&page_list);
 
  	__mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon);
  	__mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file);
@@ -1603,12 +1618,15 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
   *
   * The downside is that we have to touch page->_count against each page.
   * But we had to alter page->flags anyway.
+ *
+ * Returns currently locked lruvec
   */
 
-static void move_active_pages_to_lru(struct lruvec *lruvec,
-				     struct list_head *list,
-				     struct list_head *pages_to_free,
-				     enum lru_list lru)
+static struct lruvec *
+move_active_pages_to_lru(struct lruvec *lruvec,
+			 struct list_head *list,
+			 struct list_head *pages_to_free,
+			 enum lru_list lru)
  {
  	unsigned long pgmoved = 0;
  	struct page *page;
@@ -1630,10 +1648,11 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
 
  		page = lru_to_page(list);
 
+		lruvec = __relock_page_lruvec(lruvec, page);
+
  		VM_BUG_ON(PageLRU(page));
  		SetPageLRU(page);
 
-		lruvec = page_lruvec(page);
  		list_move(&page->lru,&lruvec->pages_lru[lru]);
  		numpages = hpage_nr_pages(page);
  		lruvec->pages_count[lru] += numpages;
@@ -1655,6 +1674,8 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
  	__mod_zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru, pgmoved);
  	if (!is_active_lru(lru))
  		__count_vm_events(PGDEACTIVATE, pgmoved);
+
+	return lruvec;
  }
 
  static void shrink_active_list(unsigned long nr_to_scan,
@@ -1744,9 +1765,9 @@ static void shrink_active_list(unsigned long nr_to_scan,
  	 */
  	reclaim_stat->recent_rotated[file] += nr_rotated;
 
Quoted 14 lines

mm/vmscan.c static void shrink_active_list(unsigned long nr_to_scan, · line 1773

Quoted 9 lines
mm/vmscan.c +2 -2
-	move_active_pages_to_lru(lruvec,&l_active,&l_hold,
+	lruvec = move_active_pages_to_lru(lruvec,&l_active,&l_hold,
  						LRU_ACTIVE + file * LRU_FILE);
-	move_active_pages_to_lru(lruvec,&l_inactive,&l_hold,
+	lruvec = move_active_pages_to_lru(lruvec,&l_inactive,&l_hold,
  						LRU_BASE   + file * LRU_FILE);
  	__mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
  	unlock_lruvec_irq(lruvec);

Hmm...could you add comments to each function as
"The caller should _lock_ lruvec before calling this functions.
This function returns a lruvec with _locked_. It may be different from passed one. And The callser should unlock lruvec"

Ok. Documenting is not my best side =)

Quoted 8 lines

Thanks,
-Kame

Quoted 6 lines

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email:<a href=mailto:"[email protected]"> [email protected]</a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#40 lore raw

PATCH v3 16/21 mm: handle lruvec relocks in compaction

In reply to Konstantin Khlebnikov

Prepare for lru_lock splitting in memory compaction code.

* disable irqs in acct_isolated() for __mod_zone_page_state(), lru_lock isn't required there.

---
 mm/compaction.c |   30 ++++++++++++++++--------------
 1 files changed, 16 insertions(+), 14 deletions(-)
mm/compaction.c +16 -14
index a976b28..54340e4 100644
@@ -224,8 +224,10 @@ static void acct_isolated(struct zone *zone, struct compact_control *cc)
 	list_for_each_entry(page, &cc->migratepages, lru)
 		count[!!page_is_file_cache(page)]++;
 
+	local_irq_disable();
 	__mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]);
 	__mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]);
+	local_irq_enable();
 }
 
 /* Similar to reclaim, but different enough that they don't share logic */
@@ -262,7 +264,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 	unsigned long nr_scanned = 0, nr_isolated = 0;
 	struct list_head *migratelist = &cc->migratepages;
 	isolate_mode_t mode = ISOLATE_ACTIVE|ISOLATE_INACTIVE;
-	struct lruvec *lruvec;
+	struct lruvec *lruvec = NULL;
 
 	/* Do not scan outside zone boundaries */
 	low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn);
@@ -294,25 +296,24 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 
 	/* Time to isolate some pages for migration */
 	cond_resched();
-	spin_lock_irq(&zone->lru_lock);
 	for (; low_pfn < end_pfn; low_pfn++) {
 		struct page *page;
-		bool locked = true;
 
 		/* give a chance to irqs before checking need_resched() */
 		if (!((low_pfn+1) % SWAP_CLUSTER_MAX)) {
-			spin_unlock_irq(&zone->lru_lock);
-			locked = false;
+			if (lruvec)
+				unlock_lruvec_irq(lruvec);
+			lruvec = NULL;
 		}
-		if (need_resched() || spin_is_contended(&zone->lru_lock)) {
-			if (locked)
-				spin_unlock_irq(&zone->lru_lock);
+		if (need_resched() ||
+		    (lruvec && spin_is_contended(&zone->lru_lock))) {
+			if (lruvec)
+				unlock_lruvec_irq(lruvec);
+			lruvec = NULL;
 			cond_resched();
-			spin_lock_irq(&zone->lru_lock);
 			if (fatal_signal_pending(current))
 				break;
-		} else if (!locked)
-			spin_lock_irq(&zone->lru_lock);
+		}
 
 		/*
 		 * migrate_pfn does not necessarily start aligned to a
@@ -359,7 +360,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 			continue;
 		}
 
-		if (!PageLRU(page))
+		if (!__lock_page_lruvec_irq(&lruvec, page))
 			continue;
 
 		/*
@@ -382,7 +383,6 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 		VM_BUG_ON(PageTransCompound(page));
 
 		/* Successfully isolated */
-		lruvec = page_lruvec(page);
 		del_page_from_lru_list(lruvec, page, page_lru(page));
 		list_add(&page->lru, migratelist);
 		cc->nr_migratepages++;
@@ -395,9 +395,11 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 		}
 	}
 
+	if (lruvec)
+		unlock_lruvec_irq(lruvec);
+
 	acct_isolated(zone, cc);
 
-	spin_unlock_irq(&zone->lru_lock);
 	cc->migrate_pfn = low_pfn;
 
 	trace_mm_compaction_isolate_migratepages(nr_scanned, nr_isolated);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#41 lore raw

Re: [PATCH v3 16/21] mm: handle lruvec relocks in compaction

Quoted 15 lines · On Thu, 23 Feb 2012 17:52:56 +0400 Konstantin Khlebnikov…

Prepare for lru_lock splitting in memory compaction code.

* disable irqs in acct_isolated() for __mod_zone_page_state(), lru_lock isn't required there.

---
 mm/compaction.c |   30 ++++++++++++++++--------------
 1 files changed, 16 insertions(+), 14 deletions(-)
mm/compaction.c +0 -0
index a976b28..54340e4 100644

mm/compaction.c static void acct_isolated(struct zone *zone, struct compact_control *cc) · line 230

Quoted 7 lines
mm/compaction.c +2 -0
@@ -224,8 +224,10 @@ static void acct_isolated(struct zone *zone, struct compact_control *cc)
 	list_for_each_entry(page, &cc->migratepages, lru)
 		count[!!page_is_file_cache(page)]++;
 
+	local_irq_disable();
 	__mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]);
 	__mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]);
+	local_irq_enable();

Why we need to disable Irq here ??

Quoted 48 lines
mm/compaction.c +10 -11
 }
 
 /* Similar to reclaim, but different enough that they don't share logic */
@@ -262,7 +264,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 	unsigned long nr_scanned = 0, nr_isolated = 0;
 	struct list_head *migratelist = &cc->migratepages;
 	isolate_mode_t mode = ISOLATE_ACTIVE|ISOLATE_INACTIVE;
-	struct lruvec *lruvec;
+	struct lruvec *lruvec = NULL;
 
 	/* Do not scan outside zone boundaries */
 	low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn);
@@ -294,25 +296,24 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 
 	/* Time to isolate some pages for migration */
 	cond_resched();
-	spin_lock_irq(&zone->lru_lock);
 	for (; low_pfn < end_pfn; low_pfn++) {
 		struct page *page;
-		bool locked = true;
 
 		/* give a chance to irqs before checking need_resched() */
 		if (!((low_pfn+1) % SWAP_CLUSTER_MAX)) {
-			spin_unlock_irq(&zone->lru_lock);
-			locked = false;
+			if (lruvec)
+				unlock_lruvec_irq(lruvec);
+			lruvec = NULL;
 		}
-		if (need_resched() || spin_is_contended(&zone->lru_lock)) {
-			if (locked)
-				spin_unlock_irq(&zone->lru_lock);
+		if (need_resched() ||
+		    (lruvec && spin_is_contended(&zone->lru_lock))) {
+			if (lruvec)
+				unlock_lruvec_irq(lruvec);
+			lruvec = NULL;
 			cond_resched();
-			spin_lock_irq(&zone->lru_lock);
 			if (fatal_signal_pending(current))
 				break;
-		} else if (!locked)
-			spin_lock_irq(&zone->lru_lock);
+		}
 
 		/*
 		 * migrate_pfn does not necessarily start aligned to a

mm/compaction.c static isolate_migrate_t isolate_migratepages(struct zone *zone, · line 364

Quoted 6 lines
mm/compaction.c +1 -1
@@ -359,7 +360,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 			continue;
 		}
 
-		if (!PageLRU(page))
+		if (!__lock_page_lruvec_irq(&lruvec, page))
 			continue;

Could you add more comments onto __lock_page_lruvec_irq() ?

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#42 lore raw

Re: [PATCH v3 16/21] mm: handle lruvec relocks in compaction

Quoted 21 lines · KAMEZAWA Hiroyuki
Quoted 18 lines · On Thu, 23 Feb 2012 17:52:56 +0400 Konstantin…

Prepare for lru_lock splitting in memory compaction code.

* disable irqs in acct_isolated() for __mod_zone_page_state(), lru_lock isn't required there.

---
  mm/compaction.c |   30 ++++++++++++++++--------------
  1 files changed, 16 insertions(+), 14 deletions(-)
mm/compaction.c +0 -0
index a976b28..54340e4 100644
@@ -224,8 +224,10 @@ static void acct_isolated(struct zone *zone, struct compact_control *cc)
  	list_for_each_entry(page,&cc->migratepages, lru)
  		count[!!page_is_file_cache(page)]++;
 
Quoted 6 lines

mm/compaction.c static void acct_isolated(struct zone *zone, struct compact_control *cc) · line 230

Quoted 4 lines
mm/compaction.c +2 -0
+	local_irq_disable();
  	__mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]);
  	__mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]);
+	local_irq_enable();

Why we need to disable Irq here ??

__mod_zone_page_state() want this to protect per-cpu counters, maybe preempt_disable() is enough.

Quoted 54 lines
Quoted 54 lines
mm/compaction.c +10 -11
  }
 
  /* Similar to reclaim, but different enough that they don't share logic */
@@ -262,7 +264,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
  	unsigned long nr_scanned = 0, nr_isolated = 0;
  	struct list_head *migratelist =&cc->migratepages;
  	isolate_mode_t mode = ISOLATE_ACTIVE|ISOLATE_INACTIVE;
-	struct lruvec *lruvec;
+	struct lruvec *lruvec = NULL;
 
  	/* Do not scan outside zone boundaries */
  	low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn);
@@ -294,25 +296,24 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 
  	/* Time to isolate some pages for migration */
  	cond_resched();
-	spin_lock_irq(&zone->lru_lock);
  	for (; low_pfn<  end_pfn; low_pfn++) {
  		struct page *page;
-		bool locked = true;
 
  		/* give a chance to irqs before checking need_resched() */
  		if (!((low_pfn+1) % SWAP_CLUSTER_MAX)) {
-			spin_unlock_irq(&zone->lru_lock);
-			locked = false;
+			if (lruvec)
+				unlock_lruvec_irq(lruvec);
+			lruvec = NULL;
  		}
-		if (need_resched() || spin_is_contended(&zone->lru_lock)) {
-			if (locked)
-				spin_unlock_irq(&zone->lru_lock);
+		if (need_resched() ||
+		    (lruvec&&  spin_is_contended(&zone->lru_lock))) {
+			if (lruvec)
+				unlock_lruvec_irq(lruvec);
+			lruvec = NULL;
  			cond_resched();
-			spin_lock_irq(&zone->lru_lock);
  			if (fatal_signal_pending(current))
  				break;
-		} else if (!locked)
-			spin_lock_irq(&zone->lru_lock);
+		}
 
  		/*
  		 * migrate_pfn does not necessarily start aligned to a
@@ -359,7 +360,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
  			continue;
  		}
 
Quoted 5 lines

mm/compaction.c static isolate_migrate_t isolate_migratepages(struct zone *zone, · line 364

Quoted 3 lines
mm/compaction.c +1 -1
-		if (!PageLRU(page))
+		if (!__lock_page_lruvec_irq(&lruvec, page))
  			continue;

Could you add more comments onto __lock_page_lruvec_irq() ?

Actually there is a very unlikely race with page free-realloc,
(which is fixed in Hugh's patchset, and surprisingly fixed in my old memory controller) thus this part will be redesigned.

Quoted 4 lines

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#43 lore raw

PATCH v3 17/21 mm: handle lruvec relock in memory controller

In reply to Konstantin Khlebnikov

Carefully relock lruvec lru lock at page memory cgroup change.

* In free_pn_rcu() wait for lruvec lock release.
Locking primitives keep lruvec pointer after successful lock held.

---
 mm/memcontrol.c |   19 +++++++++++++------
 1 files changed, 13 insertions(+), 6 deletions(-)
mm/memcontrol.c +13 -6
index aed1360..230f434 100644
@@ -2542,7 +2542,6 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,
 					enum charge_type ctype)
 {
 	struct page_cgroup *pc = lookup_page_cgroup(page);
-	struct zone *zone = page_zone(page);
 	struct lruvec *lruvec;
 	unsigned long flags;
 	bool removed = false;
@@ -2552,20 +2551,19 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,
 	 * is already on LRU. It means the page may on some other page_cgroup's
 	 * LRU. Take care of it.
 	 */
-	spin_lock_irqsave(&zone->lru_lock, flags);
+	lruvec = lock_page_lruvec(page, &flags);
 	if (PageLRU(page)) {
-		lruvec = page_lruvec(page);
 		del_page_from_lru_list(lruvec, page, page_lru(page));
 		ClearPageLRU(page);
 		removed = true;
 	}
 	__mem_cgroup_commit_charge(memcg, page, 1, pc, ctype);
 	if (removed) {
-		lruvec = page_lruvec(page);
+		lruvec = __relock_page_lruvec(lruvec, page);
 		add_page_to_lru_list(lruvec, page, page_lru(page));
 		SetPageLRU(page);
 	}
-	spin_unlock_irqrestore(&zone->lru_lock, flags);
+	unlock_lruvec(lruvec, &flags);
 }
 
 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
@@ -4624,7 +4622,16 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 
 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 {
-	kfree(memcg->info.nodeinfo[node]);
+	struct mem_cgroup_per_node *pn = memcg->info.nodeinfo[node];
+	int zone;
+
+	if (!pn)
+		return;
+
+	for (zone = 0; zone < MAX_NR_ZONES; zone++)
+		wait_lruvec_unlock(&pn->zoneinfo[zone].lruvec);
+
+	kfree(pn);
 }
 
 static struct mem_cgroup *mem_cgroup_alloc(void)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 17/21
#44 lore raw

Re: [PATCH v3 17/21] mm: handle lruvec relock in memory controller

Quoted 6 lines · On Thu, 23 Feb 2012 17:53:10 +0400 Konstantin Khlebnikov…

Carefully relock lruvec lru lock at page memory cgroup change.

* In free_pn_rcu() wait for lruvec lock release.
Locking primitives keep lruvec pointer after successful lock held.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#45 lore raw

PATCH v3 18/21 mm: add to lruvec isolated pages counters

In reply to Konstantin Khlebnikov

This patch adds into struct lruvec counter of isolated pages.
It is required for keeping lruvec alive till the isolated page putback. We cannot rely on resource counter in memory controller, because it does not account uncharged memory. And it much better to have common engine for dynamical lruvec management, than to tie all logic with memory cgroup magic. Plus this is useful information for memory reclaimer balancing.

There also appears per-cpu page-vectors for putting isolated pages back, and function add_page_to_evictable_list(). It is similar to lru_cache_add_lru() but it reuse page reference from caller and can adjust isolated pages counters. There also new function free_isolated_page_list() which is used at the end of shrink_page_list() for freeing pages and adjusting counters of isolated pages.

Memory cgroups code can shuffle pages between lruvecs without isolation if page is already isolated with someone else. Thus page lruvec reference is unstable even if page is isolated. It is stable only under lru_lock or if page reference count is zero. That's why we must always recheck page_lruvec() even on non-lumpy 0-order reclaim, where all pages are isolated from one lruvec.

Memory controller at moving pege between cgroups now adjust isolated pages counter for old lruvec before inserting page to new lruvec. Locking lruvec->lru_lock in mem_cgroup_adjust_isolated() also effectively stabilizes PageLRU() sign, so nobody will see PageLRU() under old lru_lock while page is already moved into other lruvec.

[ BTW, all lru-id arithmetic can be simplified if we devide unevictable list
into file and anon parts. After that we can swap bits in page->flags and calculate lru-id with single instruction ]

---
 include/linux/mmzone.h |   11 ++++-
 include/linux/swap.h   |    4 +-
 mm/compaction.c        |    1 
 mm/huge_memory.c       |    4 ++
 mm/internal.h          |    6 ++
 mm/ksm.c               |    2 -
 mm/memcontrol.c        |   39 ++++++++++++++--
 mm/migrate.c           |    2 -
 mm/rmap.c              |    2 -
 mm/swap.c              |   76 +++++++++++++++++++++++++++++++
 mm/vmscan.c            |  116 +++++++++++++++++++++++++++++++++++-------------
 11 files changed, 218 insertions(+), 45 deletions(-)
include/linux/mmzone.h +9 -2
index 82d5ff3..2e3a298 100644
@@ -137,13 +137,20 @@ enum lru_list {
 	LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
 	LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
 	LRU_UNEVICTABLE,
-	NR_LRU_LISTS
+	NR_EVICTABLE_LRU_LISTS = LRU_UNEVICTABLE,
+	NR_LRU_LISTS,
+	LRU_ISOLATED = NR_LRU_LISTS,
+	LRU_ISOLATED_ANON = LRU_ISOLATED,
+	LRU_ISOLATED_FILE,
+	NR_LRU_COUNTERS,
 };
 
 #define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
 
 #define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
 
+#define for_each_lru_counter(cnt) for (cnt = 0; cnt < NR_LRU_COUNTERS; cnt++)
+
 static inline int is_file_lru(enum lru_list lru)
 {
 	return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
@@ -298,7 +305,7 @@ struct zone_reclaim_stat {
 
 struct lruvec {
 	struct list_head	pages_lru[NR_LRU_LISTS];
-	unsigned long		pages_count[NR_LRU_LISTS];
+	unsigned long		pages_count[NR_LRU_COUNTERS];
 
 	struct zone_reclaim_stat	reclaim_stat;
 
include/linux/swap.h +3 -1
index 8630354..3a3ff2c 100644
@@ -234,7 +234,9 @@ extern void rotate_reclaimable_page(struct page *page);
 extern void deactivate_page(struct page *page);
 extern void swap_setup(void);
 
-extern void add_page_to_unevictable_list(struct page *page);
+extern void add_page_to_unevictable_list(struct page *page, bool isolated);
+extern void add_page_to_evictable_list(struct page *page,
+					enum lru_list lru, bool isolated);
 
 /**
  * lru_cache_add: add a page to the page lists
mm/compaction.c +1 -0
index 54340e4..fa74cbe 100644
@@ -384,6 +384,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 
 		/* Successfully isolated */
 		del_page_from_lru_list(lruvec, page, page_lru(page));
+		lruvec->pages_count[LRU_ISOLATED + page_is_file_cache(page)]++;
 		list_add(&page->lru, migratelist);
 		cc->nr_migratepages++;
 		nr_isolated++;
mm/huge_memory.c +4 -0
index 74996b8..46d9f44 100644
@@ -1316,6 +1316,10 @@ static void __split_huge_page_refcount(struct page *page)
 	__dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
 	__mod_zone_page_state(lruvec_zone(lruvec), NR_ANON_PAGES, HPAGE_PMD_NR);
 
+	/* Fixup isolated pages counter if head page currently isolated */
+	if (!PageLRU(page))
+		lruvec->pages_count[LRU_ISOLATED_ANON] -= HPAGE_PMD_NR-1;
+
 	ClearPageCompound(page);
 	compound_unlock(page);
 	unlock_lruvec_irq(lruvec);
mm/internal.h +5 -1
index 9454752..6dd2e70 100644
@@ -265,7 +265,11 @@ extern unsigned long highest_memmap_pfn;
  * in mm/vmscan.c:
  */
 extern int isolate_lru_page(struct page *page);
-extern void putback_lru_page(struct page *page);
+extern void __putback_lru_page(struct page *page, bool isolated);
+static inline void putback_lru_page(struct page *page)
+{
+	__putback_lru_page(page, true);
+}
 
 /*
  * in mm/page_alloc.c
mm/ksm.c +1 -1
index e20de58..109e6ec 100644
@@ -1592,7 +1592,7 @@ struct page *ksm_does_need_to_copy(struct page *page,
 		if (page_evictable(new_page, vma))
 			lru_cache_add_lru(new_page, LRU_ACTIVE_ANON);
 		else
-			add_page_to_unevictable_list(new_page);
+			add_page_to_unevictable_list(new_page, false);
 	}
 
 	return new_page;
mm/memcontrol.c +34 -5
index 230f434..4de8044 100644
@@ -697,7 +697,7 @@ mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
 
 	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
 
-	for_each_lru(lru) {
+	for_each_lru_counter(lru) {
 		if (BIT(lru) & lru_mask)
 			ret += mz->lruvec.pages_count[lru];
 	}
@@ -2354,6 +2354,17 @@ void mem_cgroup_split_huge_fixup(struct page *head)
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
+static void mem_cgroup_adjust_isolated(struct lruvec *lruvec,
+				       struct page *page, int delta)
+{
+	int file = page_is_file_cache(page);
+	unsigned long flags;
+
+	lock_lruvec(lruvec, &flags);
+	lruvec->pages_count[LRU_ISOLATED + file] += delta;
+	unlock_lruvec(lruvec, &flags);
+}
+
 /**
  * mem_cgroup_move_account - move account of the page
  * @page: the page
@@ -2452,6 +2463,7 @@ static int mem_cgroup_move_parent(struct page *page,
 	struct mem_cgroup *parent;
 	unsigned int nr_pages;
 	unsigned long uninitialized_var(flags);
+	struct lruvec *lruvec;
 	int ret;
 
 	/* Is ROOT ? */
@@ -2471,6 +2483,8 @@ static int mem_cgroup_move_parent(struct page *page,
 	if (ret)
 		goto put_back;
 
+	lruvec = page_lruvec(page);
+
 	if (nr_pages > 1)
 		flags = compound_lock_irqsave(page);
 
@@ -2480,8 +2494,11 @@ static int mem_cgroup_move_parent(struct page *page,
 
 	if (nr_pages > 1)
 		compound_unlock_irqrestore(page, flags);
+	if (!ret)
+		/* This also stabilize PageLRU() sign for lruvec lock holder. */
+		mem_cgroup_adjust_isolated(lruvec, page, -nr_pages);
 put_back:
-	putback_lru_page(page);
+	__putback_lru_page(page, !ret);
 put:
 	put_page(page);
 out:
@@ -3879,6 +3896,8 @@ enum {
 	MCS_INACTIVE_FILE,
 	MCS_ACTIVE_FILE,
 	MCS_UNEVICTABLE,
+	MCS_ISOLATED_ANON,
+	MCS_ISOLATED_FILE,
 	NR_MCS_STAT,
 };
 
@@ -3902,7 +3921,9 @@ struct {
 	{"active_anon", "total_active_anon"},
 	{"inactive_file", "total_inactive_file"},
 	{"active_file", "total_active_file"},
-	{"unevictable", "total_unevictable"}
+	{"unevictable", "total_unevictable"},
+	{"isolated_anon", "total_isolated_anon"},
+	{"isolated_file", "total_isolated_file"},
 };
 
 
@@ -3942,6 +3963,10 @@ mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
 	s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
 	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
 	s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
+	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ISOLATED_ANON));
+	s->stat[MCS_ISOLATED_ANON] += val * PAGE_SIZE;
+	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ISOLATED_FILE));
+	s->stat[MCS_ISOLATED_FILE] += val * PAGE_SIZE;
 }
 
 static void
@@ -5243,6 +5268,7 @@ retry:
 		struct page *page;
 		struct page_cgroup *pc;
 		swp_entry_t ent;
+		struct lruvec *lruvec;
 
 		if (!mc.precharge)
 			break;
@@ -5253,14 +5279,17 @@ retry:
 			page = target.page;
 			if (isolate_lru_page(page))
 				goto put;
+			lruvec = page_lruvec(page);
 			pc = lookup_page_cgroup(page);
 			if (!mem_cgroup_move_account(page, 1, pc,
 						     mc.from, mc.to, false)) {
 				mc.precharge--;
 				/* we uncharge from mc.from later. */
 				mc.moved_charge++;
-			}
-			putback_lru_page(page);
+				mem_cgroup_adjust_isolated(lruvec, page, -1);
+				__putback_lru_page(page, false);
+			} else
+				__putback_lru_page(page, true);
 put:			/* is_target_pte_for_mc() gets the page */
 			put_page(page);
 			break;
mm/migrate.c +1 -1
index df141f6..de13a0e 100644
@@ -868,7 +868,7 @@ out:
 	 * Move the new page to the LRU. If migration was not successful
 	 * then this will free the page.
 	 */
-	putback_lru_page(newpage);
+	__putback_lru_page(newpage, false);
 	if (result) {
 		if (rc)
 			*result = rc;
mm/rmap.c +1 -1
index aa547d4..06b5def9 100644
@@ -1139,7 +1139,7 @@ void page_add_new_anon_rmap(struct page *page,
 	if (page_evictable(page, vma))
 		lru_cache_add_lru(page, LRU_ACTIVE_ANON);
 	else
-		add_page_to_unevictable_list(page);
+		add_page_to_unevictable_list(page, false);
 }
 
 /**
mm/swap.c +75 -1

106 lines not on this page · raw message

mm/vmscan.c +84 -32

216 lines not on this page · raw message

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#46 lore raw

Re: [PATCH v3 18/21] mm: add to lruvec isolated pages counters

Review static int mem_cgroup_move_parent(struct page *page, · line 2504

Quoted 13 lines · Konstantin Khlebnikov
(quoted diff) +4 -1
@@ -2480,8 +2494,11 @@ static int mem_cgroup_move_parent(struct page *page,
 
         if (nr_pages>  1)
                 compound_unlock_irqrestore(page, flags);
+       if (!ret)
+               /* This also stabilize PageLRU() sign for lruvec lock holder. */
+               mem_cgroup_adjust_isolated(lruvec, page, -nr_pages);
  put_back:
-       putback_lru_page(page);
+       __putback_lru_page(page, !ret);
  put:
         put_page(page);
  out:

Oh, no. There must be !!ret

mm/memcontrol.c +1 -1
@@ -2482,7 +2482,7 @@ static int mem_cgroup_move_parent(struct page *page,
                /* This also stabilize PageLRU() sign for lruvec lock holder. */
                mem_cgroup_adjust_isolated(lruvec, page, -nr_pages);
 put_back:
-       __putback_lru_page(page, !ret);
+       __putback_lru_page(page, !!ret);
 put:
        put_page(page);
 out:

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#47 lore raw

Re: [PATCH v3 18/21] mm: add to lruvec isolated pages counters

In reply to Konstantin Khlebnikov
Quoted 25 lines · On Thu, 23 Feb 2012 17:53:14 +0400 Konstantin Khlebnikov…

This patch adds into struct lruvec counter of isolated pages.
It is required for keeping lruvec alive till the isolated page putback. We cannot rely on resource counter in memory controller, because it does not account uncharged memory. And it much better to have common engine for dynamical lruvec management, than to tie all logic with memory cgroup magic. Plus this is useful information for memory reclaimer balancing.

There also appears per-cpu page-vectors for putting isolated pages back, and function add_page_to_evictable_list(). It is similar to lru_cache_add_lru() but it reuse page reference from caller and can adjust isolated pages counters. There also new function free_isolated_page_list() which is used at the end of shrink_page_list() for freeing pages and adjusting counters of isolated pages.

Memory cgroups code can shuffle pages between lruvecs without isolation if page is already isolated with someone else. Thus page lruvec reference is unstable even if page is isolated. It is stable only under lru_lock or if page reference count is zero. That's why we must always recheck page_lruvec() even on non-lumpy 0-order reclaim, where all pages are isolated from one lruvec.

Memory controller at moving pege between cgroups now adjust isolated pages counter for old lruvec before inserting page to new lruvec. Locking lruvec->lru_lock in mem_cgroup_adjust_isolated() also effectively stabilizes PageLRU() sign, so nobody will see PageLRU() under old lru_lock while page is already moved into other lruvec.

Quoted 5 lines

[ BTW, all lru-id arithmetic can be simplified if we devide unevictable list
into file and anon parts. After that we can swap bits in page->flags and calculate lru-id with single instruction ]

Could you split the part showing statistics of isolated pages into another patch ?

Quoted 502 lines
---
 include/linux/mmzone.h |   11 ++++-
 include/linux/swap.h   |    4 +-
 mm/compaction.c        |    1 
 mm/huge_memory.c       |    4 ++
 mm/internal.h          |    6 ++
 mm/ksm.c               |    2 -
 mm/memcontrol.c        |   39 ++++++++++++++--
 mm/migrate.c           |    2 -
 mm/rmap.c              |    2 -
 mm/swap.c              |   76 +++++++++++++++++++++++++++++++
 mm/vmscan.c            |  116 +++++++++++++++++++++++++++++++++++-------------
 11 files changed, 218 insertions(+), 45 deletions(-)
include/linux/mmzone.h +9 -2
index 82d5ff3..2e3a298 100644
@@ -137,13 +137,20 @@ enum lru_list {
 	LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
 	LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
 	LRU_UNEVICTABLE,
-	NR_LRU_LISTS
+	NR_EVICTABLE_LRU_LISTS = LRU_UNEVICTABLE,
+	NR_LRU_LISTS,
+	LRU_ISOLATED = NR_LRU_LISTS,
+	LRU_ISOLATED_ANON = LRU_ISOLATED,
+	LRU_ISOLATED_FILE,
+	NR_LRU_COUNTERS,
 };
 
 #define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
 
 #define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
 
+#define for_each_lru_counter(cnt) for (cnt = 0; cnt < NR_LRU_COUNTERS; cnt++)
+
 static inline int is_file_lru(enum lru_list lru)
 {
 	return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
@@ -298,7 +305,7 @@ struct zone_reclaim_stat {
 
 struct lruvec {
 	struct list_head	pages_lru[NR_LRU_LISTS];
-	unsigned long		pages_count[NR_LRU_LISTS];
+	unsigned long		pages_count[NR_LRU_COUNTERS];
 
 	struct zone_reclaim_stat	reclaim_stat;
 
include/linux/swap.h +3 -1
index 8630354..3a3ff2c 100644
@@ -234,7 +234,9 @@ extern void rotate_reclaimable_page(struct page *page);
 extern void deactivate_page(struct page *page);
 extern void swap_setup(void);
 
-extern void add_page_to_unevictable_list(struct page *page);
+extern void add_page_to_unevictable_list(struct page *page, bool isolated);
+extern void add_page_to_evictable_list(struct page *page,
+					enum lru_list lru, bool isolated);
 
 /**
  * lru_cache_add: add a page to the page lists
mm/compaction.c +1 -0
index 54340e4..fa74cbe 100644
@@ -384,6 +384,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 
 		/* Successfully isolated */
 		del_page_from_lru_list(lruvec, page, page_lru(page));
+		lruvec->pages_count[LRU_ISOLATED + page_is_file_cache(page)]++;
 		list_add(&page->lru, migratelist);
 		cc->nr_migratepages++;
 		nr_isolated++;
mm/huge_memory.c +4 -0
index 74996b8..46d9f44 100644
@@ -1316,6 +1316,10 @@ static void __split_huge_page_refcount(struct page *page)
 	__dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
 	__mod_zone_page_state(lruvec_zone(lruvec), NR_ANON_PAGES, HPAGE_PMD_NR);
 
+	/* Fixup isolated pages counter if head page currently isolated */
+	if (!PageLRU(page))
+		lruvec->pages_count[LRU_ISOLATED_ANON] -= HPAGE_PMD_NR-1;
+
 	ClearPageCompound(page);
 	compound_unlock(page);
 	unlock_lruvec_irq(lruvec);
mm/internal.h +5 -1
index 9454752..6dd2e70 100644
@@ -265,7 +265,11 @@ extern unsigned long highest_memmap_pfn;
  * in mm/vmscan.c:
  */
 extern int isolate_lru_page(struct page *page);
-extern void putback_lru_page(struct page *page);
+extern void __putback_lru_page(struct page *page, bool isolated);
+static inline void putback_lru_page(struct page *page)
+{
+	__putback_lru_page(page, true);
+}
 
 /*
  * in mm/page_alloc.c
mm/ksm.c +1 -1
index e20de58..109e6ec 100644
@@ -1592,7 +1592,7 @@ struct page *ksm_does_need_to_copy(struct page *page,
 		if (page_evictable(new_page, vma))
 			lru_cache_add_lru(new_page, LRU_ACTIVE_ANON);
 		else
-			add_page_to_unevictable_list(new_page);
+			add_page_to_unevictable_list(new_page, false);
 	}
 
 	return new_page;
mm/memcontrol.c +34 -5
index 230f434..4de8044 100644
@@ -697,7 +697,7 @@ mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
 
 	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
 
-	for_each_lru(lru) {
+	for_each_lru_counter(lru) {
 		if (BIT(lru) & lru_mask)
 			ret += mz->lruvec.pages_count[lru];
 	}
@@ -2354,6 +2354,17 @@ void mem_cgroup_split_huge_fixup(struct page *head)
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
+static void mem_cgroup_adjust_isolated(struct lruvec *lruvec,
+				       struct page *page, int delta)
+{
+	int file = page_is_file_cache(page);
+	unsigned long flags;
+
+	lock_lruvec(lruvec, &flags);
+	lruvec->pages_count[LRU_ISOLATED + file] += delta;
+	unlock_lruvec(lruvec, &flags);
+}
+
 /**
  * mem_cgroup_move_account - move account of the page
  * @page: the page
@@ -2452,6 +2463,7 @@ static int mem_cgroup_move_parent(struct page *page,
 	struct mem_cgroup *parent;
 	unsigned int nr_pages;
 	unsigned long uninitialized_var(flags);
+	struct lruvec *lruvec;
 	int ret;
 
 	/* Is ROOT ? */
@@ -2471,6 +2483,8 @@ static int mem_cgroup_move_parent(struct page *page,
 	if (ret)
 		goto put_back;
 
+	lruvec = page_lruvec(page);
+
 	if (nr_pages > 1)
 		flags = compound_lock_irqsave(page);
 
@@ -2480,8 +2494,11 @@ static int mem_cgroup_move_parent(struct page *page,
 
 	if (nr_pages > 1)
 		compound_unlock_irqrestore(page, flags);
+	if (!ret)
+		/* This also stabilize PageLRU() sign for lruvec lock holder. */
+		mem_cgroup_adjust_isolated(lruvec, page, -nr_pages);
 put_back:
-	putback_lru_page(page);
+	__putback_lru_page(page, !ret);
 put:
 	put_page(page);
 out:
@@ -3879,6 +3896,8 @@ enum {
 	MCS_INACTIVE_FILE,
 	MCS_ACTIVE_FILE,
 	MCS_UNEVICTABLE,
+	MCS_ISOLATED_ANON,
+	MCS_ISOLATED_FILE,
 	NR_MCS_STAT,
 };
 
@@ -3902,7 +3921,9 @@ struct {
 	{"active_anon", "total_active_anon"},
 	{"inactive_file", "total_inactive_file"},
 	{"active_file", "total_active_file"},
-	{"unevictable", "total_unevictable"}
+	{"unevictable", "total_unevictable"},
+	{"isolated_anon", "total_isolated_anon"},
+	{"isolated_file", "total_isolated_file"},
 };
 
 
@@ -3942,6 +3963,10 @@ mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
 	s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
 	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
 	s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
+	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ISOLATED_ANON));
+	s->stat[MCS_ISOLATED_ANON] += val * PAGE_SIZE;
+	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ISOLATED_FILE));
+	s->stat[MCS_ISOLATED_FILE] += val * PAGE_SIZE;
 }
 
 static void
@@ -5243,6 +5268,7 @@ retry:
 		struct page *page;
 		struct page_cgroup *pc;
 		swp_entry_t ent;
+		struct lruvec *lruvec;
 
 		if (!mc.precharge)
 			break;
@@ -5253,14 +5279,17 @@ retry:
 			page = target.page;
 			if (isolate_lru_page(page))
 				goto put;
+			lruvec = page_lruvec(page);
 			pc = lookup_page_cgroup(page);
 			if (!mem_cgroup_move_account(page, 1, pc,
 						     mc.from, mc.to, false)) {
 				mc.precharge--;
 				/* we uncharge from mc.from later. */
 				mc.moved_charge++;
-			}
-			putback_lru_page(page);
+				mem_cgroup_adjust_isolated(lruvec, page, -1);
+				__putback_lru_page(page, false);
+			} else
+				__putback_lru_page(page, true);
 put:			/* is_target_pte_for_mc() gets the page */
 			put_page(page);
 			break;
mm/migrate.c +1 -1
index df141f6..de13a0e 100644
@@ -868,7 +868,7 @@ out:
 	 * Move the new page to the LRU. If migration was not successful
 	 * then this will free the page.
 	 */
-	putback_lru_page(newpage);
+	__putback_lru_page(newpage, false);
 	if (result) {
 		if (rc)
 			*result = rc;
mm/rmap.c +1 -1
index aa547d4..06b5def9 100644
@@ -1139,7 +1139,7 @@ void page_add_new_anon_rmap(struct page *page,
 	if (page_evictable(page, vma))
 		lru_cache_add_lru(page, LRU_ACTIVE_ANON);
 	else
-		add_page_to_unevictable_list(page);
+		add_page_to_unevictable_list(page, false);
 }
 
 /**
mm/swap.c +75 -1
index 3689e3d..998c71c 100644
@@ -37,6 +37,8 @@
 int page_cluster;
 
 static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
+static DEFINE_PER_CPU(struct pagevec[NR_EVICTABLE_LRU_LISTS],
+					   lru_add_isolated_pvecs);
 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
 
@@ -381,6 +383,67 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)
 	__lru_cache_add(page, lru);
 }
 
+static void __lru_add_isolated_fn(struct lruvec *lruvec,
+				  struct page *page, void *arg)
+{
+	enum lru_list lru = (enum lru_list)arg;
+
+	VM_BUG_ON(PageActive(page));
+	VM_BUG_ON(PageUnevictable(page));
+	VM_BUG_ON(PageLRU(page));
+
+	SetPageLRU(page);
+	if (is_active_lru(lru))
+		SetPageActive(page);
+	update_page_reclaim_stat(lruvec, lru);
+	add_page_to_lru_list(lruvec, page, lru);
+	lruvec->pages_count[LRU_ISOLATED + is_file_lru(lru)] -=
+						hpage_nr_pages(page);
+}
+
+static void __lru_add_isolated(struct pagevec *pvec, enum lru_list lru)
+{
+	VM_BUG_ON(is_unevictable_lru(lru));
+	pagevec_lru_move_fn(pvec, __lru_add_isolated_fn, (void *)lru);
+}
+
+/**
+ * add_page_to_evictable_list - add page to lru list
+ * @page	the page to be added into the lru list
+ * @lru		lru list id
+ * @isolated	need to adjust isolated pages counter
+ *
+ * Like lru_cache_add_lru() but reuses caller's reference to page and
+ * taking care about isolated pages counter on lruvec if isolated = true.
+ */
+void add_page_to_evictable_list(struct page *page,
+				enum lru_list lru, bool isolated)
+{
+	struct pagevec *pvec;
+
+	if (PageActive(page)) {
+		VM_BUG_ON(PageUnevictable(page));
+		ClearPageActive(page);
+	} else if (PageUnevictable(page)) {
+		VM_BUG_ON(PageActive(page));
+		ClearPageUnevictable(page);
+	}
+
+	VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
+
+	preempt_disable();
+	if (isolated) {
+		pvec = __this_cpu_ptr(lru_add_isolated_pvecs + lru);
+		if (!pagevec_add(pvec, page))
+			__lru_add_isolated(pvec, lru);
+	} else {
+		pvec = __this_cpu_ptr(lru_add_pvecs + lru);
+		if (!pagevec_add(pvec, page))
+			__pagevec_lru_add(pvec, lru);
+	}
+	preempt_enable();
+}
+
 /**
  * add_page_to_unevictable_list - add a page to the unevictable list
  * @page:  the page to be added to the unevictable list
@@ -391,7 +454,7 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)
  * while it's locked or otherwise "invisible" to other tasks.  This is
  * difficult to do when using the pagevec cache, so bypass that.
  */
-void add_page_to_unevictable_list(struct page *page)
+void add_page_to_unevictable_list(struct page *page, bool isolated)
 {
 	struct lruvec *lruvec;
 
@@ -399,6 +462,10 @@ void add_page_to_unevictable_list(struct page *page)
 	SetPageUnevictable(page);
 	SetPageLRU(page);
 	add_page_to_lru_list(lruvec, page, LRU_UNEVICTABLE);
+	if (isolated) {
+		int type = LRU_ISOLATED + page_is_file_cache(page);
+		lruvec->pages_count[type] -= hpage_nr_pages(page);
+	}
 	unlock_lruvec_irq(lruvec);
 }
 
@@ -485,6 +552,13 @@ static void drain_cpu_pagevecs(int cpu)
 			__pagevec_lru_add(pvec, lru);
 	}
 
+	pvecs = per_cpu(lru_add_isolated_pvecs, cpu);
+	for_each_evictable_lru(lru) {
+		pvec = &pvecs[lru];
+		if (pagevec_count(pvec))
+			__lru_add_isolated(pvec, lru);
+	}
+
 	pvec = &per_cpu(lru_rotate_pvecs, cpu);
 	if (pagevec_count(pvec)) {
 		unsigned long flags;
mm/vmscan.c +43 -26
index 6eeeb4b..a1ff010 100644
@@ -42,6 +42,7 @@
 #include <linux/sysctl.h>
 #include <linux/oom.h>
 #include <linux/prefetch.h>
+#include <trace/events/kmem.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>
@@ -585,15 +586,16 @@ int remove_mapping(struct address_space *mapping, struct page *page)
 }
 
 /**
- * putback_lru_page - put previously isolated page onto appropriate LRU list
+ * __putback_lru_page - put previously isolated page onto appropriate LRU list
  * @page: page to be put back to appropriate lru list
+ * @isolated: isolated pages counter update required
  *
  * Add previously isolated @page to appropriate LRU list.
  * Page may still be unevictable for other reasons.
  *
  * lru_lock must not be held, interrupts must be enabled.
  */
-void putback_lru_page(struct page *page)
+void __putback_lru_page(struct page *page, bool isolated)
 {
 	int lru;
 	int active = !!TestClearPageActive(page);
@@ -612,14 +614,16 @@ redo:
 		 * We know how to handle that.
 		 */
 		lru = active + page_lru_base_type(page);
-		lru_cache_add_lru(page, lru);
+		add_page_to_evictable_list(page, lru, isolated);
+		if (was_unevictable)
+			count_vm_event(UNEVICTABLE_PGRESCUED);
 	} else {
 		/*
 		 * Put unevictable pages directly on zone's unevictable
 		 * list.
 		 */
 		lru = LRU_UNEVICTABLE;
-		add_page_to_unevictable_list(page);
+		add_page_to_unevictable_list(page, isolated);
 		/*
 		 * When racing with an mlock or AS_UNEVICTABLE clearing
 		 * (page is unlocked) make sure that if the other thread
@@ -631,30 +635,26 @@ redo:
 		 * The other side is TestClearPageMlocked() or shmem_lock().
 		 */
 		smp_mb();
-	}
-
-	/*
-	 * page's status can change while we move it among lru. If an evictable
-	 * page is on unevictable list, it never be freed. To avoid that,
-	 * check after we added it to the list, again.
-	 */
-	if (lru == LRU_UNEVICTABLE && page_evictable(page, NULL)) {
-		if (!isolate_lru_page(page)) {
-			put_page(page);
-			goto redo;
-		}
-		/* This means someone else dropped this page from LRU
-		 * So, it will be freed or putback to LRU again. There is
-		 * nothing to do here.
+		/*
+		 * page's status can change while we move it among lru.
+		 * If an evictable page is on unevictable list, it never be freed.
+		 * To avoid that, check after we added it to the list, again.
 		 */
+		if (page_evictable(page, NULL)) {
+			if (!isolate_lru_page(page)) {
+				isolated = true;
+				put_page(page);
+				goto redo;
+			}
+			/* This means someone else dropped this page from LRU
+			 * So, it will be freed or putback to LRU again. There is
+			 * nothing to do here.
+			 */
+		}
+		put_page(page);		/* drop ref from isolate */
+		if (!was_unevictable)
+			count_vm_event(UNEVICTABLE_PGCULLED);
 	}
-
-	if (was_unevictable && lru != LRU_UNEVICTABLE)
-		count_vm_event(UNEVICTABLE_PGRESCUED);
-	else if (!was_unevictable && lru == LRU_UNEVICTABLE)
-		count_vm_event(UNEVICTABLE_PGCULLED);
-
-	put_page(page);		/* drop ref from isolate */
 }
 
 enum page_references {
@@ -724,6 +724,48 @@ static enum page_references page_check_references(struct page *page,
 }
 
 /*
+ * Free a list of isolated 0-order pages
+ */
+static void free_isolated_page_list(struct lruvec *lruvec,
+				    struct list_head *list, int cold)
+{
+	struct page *page, *next;
+	unsigned long nr_pages[2];
+	struct list_head queue;
+
+again:
+	INIT_LIST_HEAD(&queue);
+	nr_pages[0] = nr_pages[1] = 0;
+
+	list_for_each_entry_safe(page, next, list, lru) {
+		if (unlikely(lruvec != page_lruvec(page))) {
+			list_add_tail(&page->lru, &queue);
+			continue;

mm/vmscan.c static enum page_references page_check_references(struct page *page, · line 753

Quoted 10 lines
mm/vmscan.c +10 -0
+		}
+		nr_pages[page_is_file_cache(page)]++;
+		trace_mm_page_free_batched(page, cold);
+		free_hot_cold_page(page, cold);
+	}
+
+	lock_lruvec_irq(lruvec);
+	lruvec->pages_count[LRU_ISOLATED_ANON] -= nr_pages[0];
+	lruvec->pages_count[LRU_ISOLATED_FILE] -= nr_pages[1];
+	unlock_lruvec_irq(lruvec);

Is it guaranteed that all pages are from the same lruvec ?

Thanks,
-Kame

Quoted 104 lines
mm/vmscan.c +31 -7
+
+	/*
+	 * Usually there will be only one iteration, because
+	 * at 0-order reclaim all pages are from one lruvec
+	 * if we didn't raced with memory cgroup shuffling.
+	 */
+	if (unlikely(!list_empty(&queue))) {
+		list_replace(&queue, list);
+		lruvec = page_lruvec(list_first_entry(list,
+					struct page, lru));
+		goto again;
+	}
+}
+
+/*
  * shrink_page_list() returns the number of reclaimed pages
  */
 static unsigned long shrink_page_list(struct list_head *page_list,
@@ -986,7 +1028,7 @@ keep_lumpy:
 	if (nr_dirty && nr_dirty == nr_congested && global_reclaim(sc))
 		zone_set_flag(lruvec_zone(lruvec), ZONE_CONGESTED);
 
-	free_hot_cold_page_list(&free_pages, 1);
+	free_isolated_page_list(lruvec, &free_pages, 1);
 
 	list_splice(&ret_pages, page_list);
 	count_vm_events(PGACTIVATE, pgactivate);
@@ -1206,11 +1248,14 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 			if (__isolate_lru_page(cursor_page, mode, file) == 0) {
 				unsigned int isolated_pages;
 				int cursor_lru = page_lru(cursor_page);
+				int cur_file = page_is_file_cache(cursor_page);
 
 				list_move(&cursor_page->lru, dst);
 				isolated_pages = hpage_nr_pages(cursor_page);
 				cursor_lruvec->pages_count[cursor_lru] -=
 								isolated_pages;
+				cursor_lruvec->pages_count[LRU_ISOLATED +
+						cur_file] += isolated_pages;
 				VM_BUG_ON((long)cursor_lruvec->
 						pages_count[cursor_lru] < 0);
 				nr_taken += isolated_pages;
@@ -1248,6 +1293,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 	}
 
 	lruvec->pages_count[lru] -= nr_taken - nr_lumpy_taken;
+	lruvec->pages_count[LRU_ISOLATED + file] += nr_taken - nr_lumpy_taken;
 	VM_BUG_ON((long)lruvec->pages_count[lru] < 0);
 
 	*nr_scanned = scan;
@@ -1296,11 +1342,14 @@ int isolate_lru_page(struct page *page)
 
 		lruvec = lock_page_lruvec_irq(page);
 		if (PageLRU(page)) {
+			int file = page_is_file_cache(page);
 			int lru = page_lru(page);
 			ret = 0;
 			get_page(page);
 			ClearPageLRU(page);
 			del_page_from_lru_list(lruvec, page, lru);
+			lruvec->pages_count[LRU_ISOLATED + file] +=
+							hpage_nr_pages(page);
 		}
 		unlock_lruvec_irq(lruvec);
 	}
@@ -1347,7 +1396,7 @@ putback_inactive_pages(struct lruvec *lruvec,
 	 */
 	while (!list_empty(page_list)) {
 		struct page *page = lru_to_page(page_list);
-		int lru;
+		int numpages, lru, file;
 
 		VM_BUG_ON(PageLRU(page));
 		list_del(&page->lru);
@@ -1363,13 +1412,13 @@ putback_inactive_pages(struct lruvec *lruvec,
 
 		SetPageLRU(page);
 		lru = page_lru(page);
+		file = is_file_lru(lru);
+		numpages = hpage_nr_pages(page);
 
 		add_page_to_lru_list(lruvec, page, lru);
-		if (is_active_lru(lru)) {
-			int file = is_file_lru(lru);
-			int numpages = hpage_nr_pages(page);
+		lruvec->pages_count[LRU_ISOLATED + file] -= numpages;
+		if (is_active_lru(lru))
 			reclaim_stat->recent_rotated[file] += numpages;
-		}
 		if (put_page_testzero(page)) {
 			__ClearPageLRU(page);
 			__ClearPageActive(page);
@@ -1656,6 +1705,9 @@ move_active_pages_to_lru(struct lruvec *lruvec,
 		list_move(&page->lru, &lruvec->pages_lru[lru]);
 		numpages = hpage_nr_pages(page);
 		lruvec->pages_count[lru] += numpages;
+		/* There should be no mess between file and anon pages */
+		lruvec->pages_count[LRU_ISOLATED +
+				    is_file_lru(lru)] -= numpages;
 		pgmoved += numpages;
 
 		if (put_page_testzero(page)) {
 
--
Quoted 5 lines

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#48 lore raw

PATCH v3 19/21 memcg: check lru vectors emptiness in pre-destroy

In reply to Konstantin Khlebnikov

We must abort cgroup destroying if it still not empty, resource counter cannot catch isolated uncharged pages.

---
 mm/memcontrol.c |   10 +++++++++-
 1 files changed, 9 insertions(+), 1 deletions(-)
mm/memcontrol.c +9 -1
index 4de8044..fbeff85 100644
@@ -4859,8 +4859,16 @@ free_out:
 static int mem_cgroup_pre_destroy(struct cgroup *cont)
 {
 	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
+	int ret;
+
+	ret = mem_cgroup_force_empty(memcg, false);
+	if (ret)
+		return ret;
 
-	return mem_cgroup_force_empty(memcg, false);
+	if (mem_cgroup_nr_lru_pages(memcg, -1))
+		return -EBUSY;
+
+	return 0;
 }
 
 static void mem_cgroup_destroy(struct cgroup *cont)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Acked-by on v3 19/21
#49 lore raw

Re: [PATCH v3 19/21] memcg: check lru vectors emptiness in pre-destroy

Quoted 4 lines · On Thu, 23 Feb 2012 17:53:19 +0400 Konstantin Khlebnikov…

We must abort cgroup destroying if it still not empty, resource counter cannot catch isolated uncharged pages.

I like this.

Quoted 28 lines
---
 mm/memcontrol.c |   10 +++++++++-
 1 files changed, 9 insertions(+), 1 deletions(-)
mm/memcontrol.c +9 -2
index 4de8044..fbeff85 100644
@@ -4859,8 +4859,16 @@ free_out:
 static int mem_cgroup_pre_destroy(struct cgroup *cont)
 {
 	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
+	int ret;
+
+	ret = mem_cgroup_force_empty(memcg, false);
+	if (ret)
+		return ret;
 
-	return mem_cgroup_force_empty(memcg, false);
+	if (mem_cgroup_nr_lru_pages(memcg, -1))
+		return -EBUSY;
+
+	return 0;
 }
 
 static void mem_cgroup_destroy(struct cgroup *cont)
 
--
Quoted 5 lines

To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#50 lore raw

PATCH v3 20/21 mm: split zone->lru_lock

In reply to Konstantin Khlebnikov

Looks like all ready for splitting zone->lru_lock into per-lruvec pieces.

lruvec locking loop protected with rcu, actually there is irq-disabling instead of rcu_read_lock(). Memory controller already releases its lru-vectors after syncronize_rcu() in cgroup_diput(). Probably it should be replaced with synchronize_sched()

---
 include/linux/mmzone.h |    3 +-
 mm/compaction.c        |    2 +
 mm/internal.h          |   66 +++++++++++++++++++++++++-----------------------
 mm/page_alloc.c        |    2 +
 mm/swap.c              |    2 +
 5 files changed, 40 insertions(+), 35 deletions(-)
include/linux/mmzone.h +2 -1
index 2e3a298..9880150 100644
@@ -304,6 +304,8 @@ struct zone_reclaim_stat {
 };
 
 struct lruvec {
+	spinlock_t		lru_lock;
+
 	struct list_head	pages_lru[NR_LRU_LISTS];
 	unsigned long		pages_count[NR_LRU_COUNTERS];
 
@@ -386,7 +388,6 @@ struct zone {
 	ZONE_PADDING(_pad1_)
 
 	/* Fields commonly accessed by the page reclaim scanner */
-	spinlock_t		lru_lock;
 	struct lruvec		lruvec;
 
 	unsigned long		pages_scanned;	   /* since last reclaim */
mm/compaction.c +1 -1
index fa74cbe..8661bb58 100644
@@ -306,7 +306,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 			lruvec = NULL;
 		}
 		if (need_resched() ||
-		    (lruvec && spin_is_contended(&zone->lru_lock))) {
+		    (lruvec && spin_is_contended(&lruvec->lru_lock))) {
 			if (lruvec)
 				unlock_lruvec_irq(lruvec);
 			lruvec = NULL;
mm/internal.h +35 -31

128 lines not on this page · raw message

mm/page_alloc.c +1 -1
index ab42446..beadcc9 100644
@@ -4294,6 +4294,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)
 	enum lru_list lru;
 
 	memset(lruvec, 0, sizeof(struct lruvec));
+	spin_lock_init(&lruvec->lru_lock);
 	for_each_lru(lru)
 		INIT_LIST_HEAD(&lruvec->pages_lru[lru]);
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
@@ -4369,7 +4370,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 #endif
 		zone->name = zone_names[j];
 		spin_lock_init(&zone->lock);
-		spin_lock_init(&zone->lru_lock);
 		zone_seqlock_init(zone);
 		zone->zone_pgdat = pgdat;
 
mm/swap.c +1 -1
index 998c71c..8156181 100644
@@ -700,7 +700,7 @@ void lru_add_page_tail(struct lruvec *lruvec,
 	VM_BUG_ON(!PageHead(page));
 	VM_BUG_ON(PageCompound(page_tail));
 	VM_BUG_ON(PageLRU(page_tail));
-	VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&lruvec_zone(lruvec)->lru_lock));
+	VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&lruvec->lru_lock));
 
 	SetPageLRU(page_tail);
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#51 lore raw

Re: [PATCH v3 20/21] mm: split zone->lru_lock

Quoted 7 lines · On Thu, 23 Feb 2012 17:53:23 +0400 Konstantin Khlebnikov…

Looks like all ready for splitting zone->lru_lock into per-lruvec pieces.

lruvec locking loop protected with rcu, actually there is irq-disabling instead of rcu_read_lock(). Memory controller already releases its lru-vectors after syncronize_rcu() in cgroup_diput(). Probably it should be replaced with synchronize_sched()

Do we need rcu_read_lock() even if we check isolated pages at pre_destroy() ? If pre_destroy() ends, pages under a memcg being destroyed were moved to other cgroup while it's usolated.

So,
- PageLRU(page) guarantees lruvec is valid.
- if !PageLRU(page), the caller of lru_lock should know what it does. Once isolated, pre_destroy() never ends and page_lruvec(page) is always stable.

Thanks,
-Kame

Quoted 205 lines
---
 include/linux/mmzone.h |    3 +-
 mm/compaction.c        |    2 +
 mm/internal.h          |   66 +++++++++++++++++++++++++-----------------------
 mm/page_alloc.c        |    2 +
 mm/swap.c              |    2 +
 5 files changed, 40 insertions(+), 35 deletions(-)
include/linux/mmzone.h +2 -1
index 2e3a298..9880150 100644
@@ -304,6 +304,8 @@ struct zone_reclaim_stat {
 };
 
 struct lruvec {
+	spinlock_t		lru_lock;
+
 	struct list_head	pages_lru[NR_LRU_LISTS];
 	unsigned long		pages_count[NR_LRU_COUNTERS];
 
@@ -386,7 +388,6 @@ struct zone {
 	ZONE_PADDING(_pad1_)
 
 	/* Fields commonly accessed by the page reclaim scanner */
-	spinlock_t		lru_lock;
 	struct lruvec		lruvec;
 
 	unsigned long		pages_scanned;	   /* since last reclaim */
mm/compaction.c +1 -1
index fa74cbe..8661bb58 100644
@@ -306,7 +306,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
 			lruvec = NULL;
 		}
 		if (need_resched() ||
-		    (lruvec && spin_is_contended(&zone->lru_lock))) {
+		    (lruvec && spin_is_contended(&lruvec->lru_lock))) {
 			if (lruvec)
 				unlock_lruvec_irq(lruvec);
 			lruvec = NULL;
mm/internal.h +35 -31
index 6dd2e70..9a9fd53 100644
@@ -15,27 +15,27 @@
 
 static inline void lock_lruvec(struct lruvec *lruvec, unsigned long *flags)
 {
-	spin_lock_irqsave(&lruvec_zone(lruvec)->lru_lock, *flags);
+	spin_lock_irqsave(&lruvec->lru_lock, *flags);
 }
 
 static inline void lock_lruvec_irq(struct lruvec *lruvec)
 {
-	spin_lock_irq(&lruvec_zone(lruvec)->lru_lock);
+	spin_lock_irq(&lruvec->lru_lock);
 }
 
 static inline void unlock_lruvec(struct lruvec *lruvec, unsigned long *flags)
 {
-	spin_unlock_irqrestore(&lruvec_zone(lruvec)->lru_lock, *flags);
+	spin_unlock_irqrestore(&lruvec->lru_lock, *flags);
 }
 
 static inline void unlock_lruvec_irq(struct lruvec *lruvec)
 {
-	spin_unlock_irq(&lruvec_zone(lruvec)->lru_lock);
+	spin_unlock_irq(&lruvec->lru_lock);
 }
 
 static inline void wait_lruvec_unlock(struct lruvec *lruvec)
 {
-	spin_unlock_wait(&lruvec_zone(lruvec)->lru_lock);
+	spin_unlock_wait(&lruvec->lru_lock);
 }
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
@@ -46,37 +46,39 @@ static inline void wait_lruvec_unlock(struct lruvec *lruvec)
 static inline struct lruvec *__relock_page_lruvec(struct lruvec *locked_lruvec,
 						  struct page *page)
 {
-	/* Currenyly only one lru_lock per-zone */
-	return page_lruvec(page);
+	struct lruvec *lruvec;
+
+	do {
+		lruvec = page_lruvec(page);
+		if (likely(lruvec == locked_lruvec))
+			return lruvec;
+		spin_unlock(&locked_lruvec->lru_lock);
+		spin_lock(&lruvec->lru_lock);
+		locked_lruvec = lruvec;
+	} while (1);
 }
 
 static inline struct lruvec *relock_page_lruvec_irq(struct lruvec *lruvec,
 						    struct page *page)
 {
-	struct zone *zone = page_zone(page);
-
 	if (!lruvec) {
-		spin_lock_irq(&zone->lru_lock);
-	} else if (zone != lruvec_zone(lruvec)) {
-		unlock_lruvec_irq(lruvec);
-		spin_lock_irq(&zone->lru_lock);
+		local_irq_disable();
+		lruvec = page_lruvec(page);
+		spin_lock(&lruvec->lru_lock);
 	}
-	return page_lruvec(page);
+	return __relock_page_lruvec(lruvec, page);
 }
 
 static inline struct lruvec *relock_page_lruvec(struct lruvec *lruvec,
 						struct page *page,
 						unsigned long *flags)
 {
-	struct zone *zone = page_zone(page);
-
 	if (!lruvec) {
-		spin_lock_irqsave(&zone->lru_lock, *flags);
-	} else if (zone != lruvec_zone(lruvec)) {
-		unlock_lruvec(lruvec, flags);
-		spin_lock_irqsave(&zone->lru_lock, *flags);
+		local_irq_save(*flags);
+		lruvec = page_lruvec(page);
+		spin_lock(&lruvec->lru_lock);
 	}
-	return page_lruvec(page);
+	return __relock_page_lruvec(lruvec, page);
 }
 
 /*
@@ -87,22 +89,24 @@ static inline struct lruvec *relock_page_lruvec(struct lruvec *lruvec,
 static inline bool __lock_page_lruvec_irq(struct lruvec **lruvec,
 					  struct page *page)
 {
-	struct zone *zone;
 	bool ret = false;
 
+	rcu_read_lock();
+	/*
+	 * If we see there PageLRU(), it means page has valid lruvec link.
+	 * We need protect whole operation with single rcu-interval, otherwise
+	 * lruvec which hold this LRU sign can run out before we secure it.
+	 */
 	if (PageLRU(page)) {
 		if (!*lruvec) {
-			zone = page_zone(page);
-			spin_lock_irq(&zone->lru_lock);
-		} else
-			zone = lruvec_zone(*lruvec);
-
-		if (PageLRU(page)) {
 			*lruvec = page_lruvec(page);
+			lock_lruvec_irq(*lruvec);
+		}
+		*lruvec = __relock_page_lruvec(*lruvec, page);
+		if (PageLRU(page))
 			ret = true;
-		} else
-			*lruvec = &zone->lruvec;
 	}
+	rcu_read_unlock();
 
 	return ret;
 }
@@ -110,7 +114,7 @@ static inline bool __lock_page_lruvec_irq(struct lruvec **lruvec,
 /* Wait for lruvec unlock before locking other lruvec for the same page */
 static inline void __wait_lruvec_unlock(struct lruvec *lruvec)
 {
-	/* Currently only one lru_lock per-zone */
+	wait_lruvec_unlock(lruvec);
 }
 
 #else /* CONFIG_CGROUP_MEM_RES_CTLR */
mm/page_alloc.c +1 -1
index ab42446..beadcc9 100644
@@ -4294,6 +4294,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)
 	enum lru_list lru;
 
 	memset(lruvec, 0, sizeof(struct lruvec));
+	spin_lock_init(&lruvec->lru_lock);
 	for_each_lru(lru)
 		INIT_LIST_HEAD(&lruvec->pages_lru[lru]);
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
@@ -4369,7 +4370,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 #endif
 		zone->name = zone_names[j];
 		spin_lock_init(&zone->lock);
-		spin_lock_init(&zone->lru_lock);
 		zone_seqlock_init(zone);
 		zone->zone_pgdat = pgdat;
 
mm/swap.c +0 -0
index 998c71c..8156181 100644

mm/swap.c void lru_add_page_tail(struct lruvec *lruvec, · line 706

Quoted 8 lines
mm/swap.c +1 -1
@@ -700,7 +700,7 @@ void lru_add_page_tail(struct lruvec *lruvec,
 	VM_BUG_ON(!PageHead(page));
 	VM_BUG_ON(PageCompound(page_tail));
 	VM_BUG_ON(PageLRU(page_tail));
-	VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&lruvec_zone(lruvec)->lru_lock));
+	VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&lruvec->lru_lock));
 
 	SetPageLRU(page_tail);
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#52 lore raw

Re: [PATCH v3 20/21] mm: split zone->lru_lock

Quoted 11 lines · KAMEZAWA Hiroyuki
Quoted 7 lines · On Thu, 23 Feb 2012 17:53:23 +0400 Konstantin…

Looks like all ready for splitting zone->lru_lock into per-lruvec pieces.

lruvec locking loop protected with rcu, actually there is irq-disabling instead of rcu_read_lock(). Memory controller already releases its lru-vectors after syncronize_rcu() in cgroup_diput(). Probably it should be replaced with synchronize_sched()

Quoted 8 lines

Do we need rcu_read_lock() even if we check isolated pages at pre_destroy() ? If pre_destroy() ends, pages under a memcg being destroyed were moved to other cgroup while it's usolated.

So,
- PageLRU(page) guarantees lruvec is valid.
- if !PageLRU(page), the caller of lru_lock should know what it does. Once isolated, pre_destroy() never ends and page_lruvec(page) is always stable.

This will be racy, because lruvec can be changed and released between checking PageLRU() and page_lruvec() or between page_lruvec() and spin_lock(), so we need to protect whole operation with single locking interval.

Quoted 213 lines

Thanks,
-Kame

Quoted 209 lines
---
  include/linux/mmzone.h |    3 +-
  mm/compaction.c        |    2 +
  mm/internal.h          |   66 +++++++++++++++++++++++++-----------------------
  mm/page_alloc.c        |    2 +
  mm/swap.c              |    2 +
  5 files changed, 40 insertions(+), 35 deletions(-)
include/linux/mmzone.h +2 -1
index 2e3a298..9880150 100644
@@ -304,6 +304,8 @@ struct zone_reclaim_stat {
  };
 
  struct lruvec {
+	spinlock_t		lru_lock;
+
  	struct list_head	pages_lru[NR_LRU_LISTS];
  	unsigned long		pages_count[NR_LRU_COUNTERS];
 
@@ -386,7 +388,6 @@ struct zone {
  	ZONE_PADDING(_pad1_)
 
  	/* Fields commonly accessed by the page reclaim scanner */
-	spinlock_t		lru_lock;
  	struct lruvec		lruvec;
 
  	unsigned long		pages_scanned;	   /* since last reclaim */
mm/compaction.c +1 -1
index fa74cbe..8661bb58 100644
@@ -306,7 +306,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
  			lruvec = NULL;
  		}
  		if (need_resched() ||
-		    (lruvec&&  spin_is_contended(&zone->lru_lock))) {
+		    (lruvec&&  spin_is_contended(&lruvec->lru_lock))) {
  			if (lruvec)
  				unlock_lruvec_irq(lruvec);
  			lruvec = NULL;
mm/internal.h +35 -31
index 6dd2e70..9a9fd53 100644
@@ -15,27 +15,27 @@
 
  static inline void lock_lruvec(struct lruvec *lruvec, unsigned long *flags)
  {
-	spin_lock_irqsave(&lruvec_zone(lruvec)->lru_lock, *flags);
+	spin_lock_irqsave(&lruvec->lru_lock, *flags);
  }
 
  static inline void lock_lruvec_irq(struct lruvec *lruvec)
  {
-	spin_lock_irq(&lruvec_zone(lruvec)->lru_lock);
+	spin_lock_irq(&lruvec->lru_lock);
  }
 
  static inline void unlock_lruvec(struct lruvec *lruvec, unsigned long *flags)
  {
-	spin_unlock_irqrestore(&lruvec_zone(lruvec)->lru_lock, *flags);
+	spin_unlock_irqrestore(&lruvec->lru_lock, *flags);
  }
 
  static inline void unlock_lruvec_irq(struct lruvec *lruvec)
  {
-	spin_unlock_irq(&lruvec_zone(lruvec)->lru_lock);
+	spin_unlock_irq(&lruvec->lru_lock);
  }
 
  static inline void wait_lruvec_unlock(struct lruvec *lruvec)
  {
-	spin_unlock_wait(&lruvec_zone(lruvec)->lru_lock);
+	spin_unlock_wait(&lruvec->lru_lock);
  }
 
  #ifdef CONFIG_CGROUP_MEM_RES_CTLR
@@ -46,37 +46,39 @@ static inline void wait_lruvec_unlock(struct lruvec *lruvec)
  static inline struct lruvec *__relock_page_lruvec(struct lruvec *locked_lruvec,
  						  struct page *page)
  {
-	/* Currenyly only one lru_lock per-zone */
-	return page_lruvec(page);
+	struct lruvec *lruvec;
+
+	do {
+		lruvec = page_lruvec(page);
+		if (likely(lruvec == locked_lruvec))
+			return lruvec;
+		spin_unlock(&locked_lruvec->lru_lock);
+		spin_lock(&lruvec->lru_lock);
+		locked_lruvec = lruvec;
+	} while (1);
  }
 
  static inline struct lruvec *relock_page_lruvec_irq(struct lruvec *lruvec,
  						    struct page *page)
  {
-	struct zone *zone = page_zone(page);
-
  	if (!lruvec) {
-		spin_lock_irq(&zone->lru_lock);
-	} else if (zone != lruvec_zone(lruvec)) {
-		unlock_lruvec_irq(lruvec);
-		spin_lock_irq(&zone->lru_lock);
+		local_irq_disable();
+		lruvec = page_lruvec(page);
+		spin_lock(&lruvec->lru_lock);
  	}
-	return page_lruvec(page);
+	return __relock_page_lruvec(lruvec, page);
  }
 
  static inline struct lruvec *relock_page_lruvec(struct lruvec *lruvec,
  						struct page *page,
  						unsigned long *flags)
  {
-	struct zone *zone = page_zone(page);
-
  	if (!lruvec) {
-		spin_lock_irqsave(&zone->lru_lock, *flags);
-	} else if (zone != lruvec_zone(lruvec)) {
-		unlock_lruvec(lruvec, flags);
-		spin_lock_irqsave(&zone->lru_lock, *flags);
+		local_irq_save(*flags);
+		lruvec = page_lruvec(page);
+		spin_lock(&lruvec->lru_lock);
  	}
-	return page_lruvec(page);
+	return __relock_page_lruvec(lruvec, page);
  }
 
  /*
@@ -87,22 +89,24 @@ static inline struct lruvec *relock_page_lruvec(struct lruvec *lruvec,
  static inline bool __lock_page_lruvec_irq(struct lruvec **lruvec,
  					  struct page *page)
  {
-	struct zone *zone;
  	bool ret = false;
 
+	rcu_read_lock();
+	/*
+	 * If we see there PageLRU(), it means page has valid lruvec link.
+	 * We need protect whole operation with single rcu-interval, otherwise
+	 * lruvec which hold this LRU sign can run out before we secure it.
+	 */
  	if (PageLRU(page)) {
  		if (!*lruvec) {
-			zone = page_zone(page);
-			spin_lock_irq(&zone->lru_lock);
-		} else
-			zone = lruvec_zone(*lruvec);
-
-		if (PageLRU(page)) {
  			*lruvec = page_lruvec(page);
+			lock_lruvec_irq(*lruvec);
+		}
+		*lruvec = __relock_page_lruvec(*lruvec, page);
+		if (PageLRU(page))
  			ret = true;
-		} else
-			*lruvec =&zone->lruvec;
  	}
+	rcu_read_unlock();
 
  	return ret;
  }
@@ -110,7 +114,7 @@ static inline bool __lock_page_lruvec_irq(struct lruvec **lruvec,
  /* Wait for lruvec unlock before locking other lruvec for the same page */
  static inline void __wait_lruvec_unlock(struct lruvec *lruvec)
  {
-	/* Currently only one lru_lock per-zone */
+	wait_lruvec_unlock(lruvec);
  }
 
  #else /* CONFIG_CGROUP_MEM_RES_CTLR */
mm/page_alloc.c +1 -1
index ab42446..beadcc9 100644
@@ -4294,6 +4294,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)
  	enum lru_list lru;
 
  	memset(lruvec, 0, sizeof(struct lruvec));
+	spin_lock_init(&lruvec->lru_lock);
  	for_each_lru(lru)
  		INIT_LIST_HEAD(&lruvec->pages_lru[lru]);
  #ifdef CONFIG_CGROUP_MEM_RES_CTLR
@@ -4369,7 +4370,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
  #endif
  		zone->name = zone_names[j];
  		spin_lock_init(&zone->lock);
-		spin_lock_init(&zone->lru_lock);
  		zone_seqlock_init(zone);
  		zone->zone_pgdat = pgdat;
 
mm/swap.c +1 -1
index 998c71c..8156181 100644
@@ -700,7 +700,7 @@ void lru_add_page_tail(struct lruvec *lruvec,
  	VM_BUG_ON(!PageHead(page));
  	VM_BUG_ON(PageCompound(page_tail));
  	VM_BUG_ON(PageLRU(page_tail));
-	VM_BUG_ON(NR_CPUS != 1&&  !spin_is_locked(&lruvec_zone(lruvec)->lru_lock));
+	VM_BUG_ON(NR_CPUS != 1&&  !spin_is_locked(&lruvec->lru_lock));
 
Quoted 5 lines

mm/swap.c void lru_add_page_tail(struct lruvec *lruvec, · line 706

Quoted 4 lines
mm/swap.c +0 -0
  	SetPageLRU(page_tail);
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#53 lore raw

PATCH v3 21/21 mm: zone lru vectors interleaving

In reply to Konstantin Khlebnikov

Split zones into several lru vectors with pfn-based interleaving. Thus we can redeuce lru_lock contention without using cgroups.

By default there 4 lru with 16Mb interleaving.

---
 include/linux/huge_mm.h    |    3 ++
 include/linux/memcontrol.h |    2 +-
 include/linux/mm.h         |   45 +++++++++++++++++++++++++++++------
 include/linux/mmzone.h     |    4 ++-
 mm/Kconfig                 |   16 +++++++++++++
 mm/internal.h              |   19 ++++++++++++++-
 mm/memcontrol.c            |   56 ++++++++++++++++++++++++--------------------
 mm/page_alloc.c            |    7 +++---
 mm/vmscan.c                |   18 ++++++++++----
 9 files changed, 124 insertions(+), 46 deletions(-)
include/linux/huge_mm.h +3 -0
index 1b92129..3a45cb3 100644
@@ -107,6 +107,9 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
 #if HPAGE_PMD_ORDER > MAX_ORDER
 #error "hugepages can't be allocated by the buddy allocator"
 #endif
+#if HPAGE_PMD_ORDER > CONFIG_PAGE_LRU_INTERLEAVING
+#error "zone lru interleaving order lower than huge page order"
+#endif
 extern int hugepage_madvise(struct vm_area_struct *vma,
 			    unsigned long *vm_flags, int advice);
 extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
include/linux/memcontrol.h +1 -1
index c3e46b0..b137d4c 100644
@@ -199,7 +199,7 @@ static inline void mem_cgroup_uncharge_cache_page(struct page *page)
 static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
 						    struct mem_cgroup *memcg)
 {
-	return &zone->lruvec;
+	return zone->lruvec;
 }
 
 static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
include/linux/mm.h +37 -8

71 lines not on this page · raw message

include/linux/mmzone.h +2 -2
index 9880150..a52f423 100644
@@ -311,7 +311,7 @@ struct lruvec {
 
 	struct zone_reclaim_stat	reclaim_stat;
 
-#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+#if defined(CONFIG_CGROUP_MEM_RES_CTLR) || (CONFIG_PAGE_LRU_SPLIT != 1)
 	struct zone		*zone;
 	struct pglist_data	*node;
 #endif
@@ -388,7 +388,7 @@ struct zone {
 	ZONE_PADDING(_pad1_)
 
 	/* Fields commonly accessed by the page reclaim scanner */
-	struct lruvec		lruvec;
+	struct lruvec		lruvec[CONFIG_PAGE_LRU_SPLIT];
 
 	unsigned long		pages_scanned;	   /* since last reclaim */
 	unsigned long		flags;		   /* zone flags, see below */
mm/Kconfig +16 -0
index 2613c91..48ff866 100644
@@ -183,6 +183,22 @@ config SPLIT_PTLOCK_CPUS
 	default "999999" if DEBUG_SPINLOCK || DEBUG_LOCK_ALLOC
 	default "4"
 
+config PAGE_LRU_SPLIT
+	int "Memory lru lists per zone"
+	default	4 if EXPERIMENTAL && SPARSEMEM_VMEMMAP
+	default 1
+	help
+	  The number of lru lists in each memory zone for interleaving.
+	  Allows to redeuce lru_lock contention, but adds some overhead.
+	  Without SPARSEMEM_VMEMMAP might be costly. "1" means no split.
+
+config PAGE_LRU_INTERLEAVING
+	int "Memory lru lists interleaving page-order"
+	default	12
+	help
+	  Page order for lru lists interleaving. By default 12 (16Mb).
+	  Must be greater than huge-page order.
+	  With CONFIG_PAGE_LRU_SPLIT=1 has no effect.
 #
 # support for memory compaction
 config COMPACTION
mm/internal.h +18 -1
index 9a9fd53..f429911 100644
@@ -13,6 +13,15 @@
 
 #include <linux/mm.h>
 
+#define for_each_zone_id(zone_id) \
+	for ( zone_id = 0 ; zone_id < MAX_NR_ZONES ; zone_id++ )
+
+#define for_each_lruvec_id(lruvec_id) \
+	for ( lruvec_id = 0 ; lruvec_id < CONFIG_PAGE_LRU_SPLIT ; lruvec_id++ )
+
+#define for_each_zone_and_lruvec_id(zone_id, lruvec_id) \
+	for_each_zone_id(zone_id) for_each_lruvec_id(lruvec_id)
+
 static inline void lock_lruvec(struct lruvec *lruvec, unsigned long *flags)
 {
 	spin_lock_irqsave(&lruvec->lru_lock, *flags);
@@ -125,7 +134,15 @@ static inline void __wait_lruvec_unlock(struct lruvec *lruvec)
 static inline struct lruvec *__relock_page_lruvec(struct lruvec *locked_lruvec,
 						  struct page *page)
 {
-	/* Currently ony one lruvec per-zone */
+#if CONFIG_PAGE_LRU_SPLIT != 1
+	struct lruvec *lruvec = page_lruvec(page);
+
+	if (unlikely(lruvec != locked_lruvec)) {
+		spin_unlock(&locked_lruvec->lru_lock);
+		spin_lock(&lruvec->lru_lock);
+		locked_lruvec = lruvec;
+	}
+#endif
 	return locked_lruvec;
 }
 
mm/memcontrol.c +30 -26

163 lines not on this page · raw message

mm/page_alloc.c +4 -3
index beadcc9..9b0cc92 100644
@@ -4297,7 +4297,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)
 	spin_lock_init(&lruvec->lru_lock);
 	for_each_lru(lru)
 		INIT_LIST_HEAD(&lruvec->pages_lru[lru]);
-#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+#if defined(CONFIG_CGROUP_MEM_RES_CTLR) || (CONFIG_PAGE_LRU_SPLIT != 1)
 	lruvec->node = zone->zone_pgdat;
 	lruvec->zone = zone;
 #endif
@@ -4312,7 +4312,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)
 static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 		unsigned long *zones_size, unsigned long *zholes_size)
 {
-	enum zone_type j;
+	enum zone_type j, lruvec_id;
 	int nid = pgdat->node_id;
 	unsigned long zone_start_pfn = pgdat->node_start_pfn;
 	int ret;
@@ -4374,7 +4374,8 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 		zone->zone_pgdat = pgdat;
 
 		zone_pcp_init(zone);
-		init_zone_lruvec(zone, &zone->lruvec);
+		for_each_lruvec_id(lruvec_id)
+			init_zone_lruvec(zone, &zone->lruvec[lruvec_id]);
 		zap_zone_vm_stats(zone);
 		zone->flags = 0;
 		if (!size)
mm/vmscan.c +13 -5
index a1ff010..aaf2b0e 100644
@@ -2202,12 +2202,14 @@ static void shrink_zone(int priority, struct zone *zone,
 	};
 	struct mem_cgroup *memcg;
 	struct lruvec *lruvec;
+	int lruvec_id;
 
 	memcg = mem_cgroup_iter(root, NULL, &reclaim);
 	do {
 		lruvec = mem_cgroup_zone_lruvec(zone, memcg);
 
-		shrink_lruvec(priority, lruvec, sc);
+		for_each_lruvec_id(lruvec_id)
+			shrink_lruvec(priority, lruvec + lruvec_id, sc);
 
 		/*
 		 * Limit reclaim has historically picked one memcg and
@@ -2529,6 +2531,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,
 		.target_mem_cgroup = memcg,
 	};
 	struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
+	int lruvec_id;
 
 	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
 			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
@@ -2544,7 +2547,8 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,
 	 * will pick up pages from other mem cgroup's as well. We hack
 	 * the priority and make it zero.
 	 */
-	shrink_lruvec(0, lruvec, &sc);
+	for_each_lruvec_id(lruvec_id)
+		shrink_lruvec(0, lruvec + lruvec_id, &sc);
 
 	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
 
@@ -2599,6 +2603,7 @@ static void age_active_anon(struct zone *zone, struct scan_control *sc,
 			    int priority)
 {
 	struct mem_cgroup *memcg;
+	int lruvec_id;
 
 	if (!total_swap_pages)
 		return;
@@ -2607,9 +2612,12 @@ static void age_active_anon(struct zone *zone, struct scan_control *sc,
 	do {
 		struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
 
-		if (inactive_anon_is_low(lruvec))
-			shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
-					   sc, priority, 0);
+		for_each_lruvec_id(lruvec_id) {
+			if (inactive_anon_is_low(lruvec + lruvec_id))
+				shrink_active_list(SWAP_CLUSTER_MAX,
+						   lruvec + lruvec_id,
+						   sc, priority, 0);
+		}
 
 		memcg = mem_cgroup_iter(NULL, memcg, NULL);
 	} while (memcg);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#54 lore raw

Re: [PATCH v3 21/21] mm: zone lru vectors interleaving

mm/page_alloc.c void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec) · line 4315

Quoted 6 lines · Konstantin Khlebnikov
mm/page_alloc.c +1 -1
@@ -4312,7 +4312,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)
  static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                unsigned long *zones_size, unsigned long *zholes_size)
  {
-       enum zone_type j;
+       enum zone_type j, lruvec_id;

Like other cases in the patch,

          int lruvec_id;

looks clearer

mm/page_alloc.c static void __paginginit free_area_init_core(struct pglist_data *pgdat, · line 4381

Quoted 13 lines
mm/page_alloc.c +2 -1
        int nid = pgdat->node_id;
        unsigned long zone_start_pfn = pgdat->node_start_pfn;
        int ret;
@@ -4374,7 +4374,8 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                zone->zone_pgdat = pgdat;
 
                zone_pcp_init(zone);
-               init_zone_lruvec(zone, &zone->lruvec);
+               for_each_lruvec_id(lruvec_id)
+                       init_zone_lruvec(zone, &zone->lruvec[lruvec_id]);
                zap_zone_vm_stats(zone);
                zone->flags = 0;
                if (!size)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#55 lore raw

Re: [PATCH v3 21/21] mm: zone lru vectors interleaving

In reply to Konstantin Khlebnikov

mm/Kconfig config SPLIT_PTLOCK_CPUS · line 193

Quoted 8 lines
mm/Kconfig +8 -0
+config PAGE_LRU_SPLIT
+	int "Memory lru lists per zone"
+	default	4 if EXPERIMENTAL && SPARSEMEM_VMEMMAP
+	default 1
+	help
+	  The number of lru lists in each memory zone for interleaving.
+	  Allows to redeuce lru_lock contention, but adds some overhead.
+	  Without SPARSEMEM_VMEMMAP might be costly. "1" means no split.

Could you turn those two numbers into a boot option? Compile time parameters are nasty to use.

I suppose it's ok to have an upper limit.

mm/Kconfig config SPLIT_PTLOCK_CPUS · line 201

Quoted 8 lines
mm/Kconfig +8 -0
+
+config PAGE_LRU_INTERLEAVING
+	int "Memory lru lists interleaving page-order"
+	default	12
+	help
+	  Page order for lru lists interleaving. By default 12 (16Mb).
+	  Must be greater than huge-page order.
+	  With CONFIG_PAGE_LRU_SPLIT=1 has no effect.

-Andi

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#56 lore raw

PATCH 1/2 mm: configure lruvec split by boot options

This patch adds boot options:
lruvec_split=%u by default 1, limited by CONFIG_PAGE_LRU_SPLIT lruvec_interleaving=%u by default CONFIG_PAGE_LRU_INTERLEAVING

---
 include/linux/mm.h |    5 ++++-
 mm/internal.h      |    2 +-
 mm/page_alloc.c    |   29 +++++++++++++++++++++++++++++
 3 files changed, 34 insertions(+), 2 deletions(-)
include/linux/mm.h +4 -1
index d14db10..f042a34 100644
@@ -737,12 +737,15 @@ static inline int page_lruvec_id(struct page *page)
 
 #else /* CONFIG_PAGE_LRU_SPLIT */
 
+extern unsigned lruvec_split;
+extern unsigned lruvec_interleaving;
+
 static inline int page_lruvec_id(struct page *page)
 {
 
 	unsigned long pfn = page_to_pfn(page);
 
-	return (pfn >> CONFIG_PAGE_LRU_INTERLEAVING) % CONFIG_PAGE_LRU_SPLIT;
+	return (pfn >> lruvec_interleaving) % lruvec_split;
 }
 
 #endif /* CONFIG_PAGE_LRU_SPLIT */
mm/internal.h +1 -1
index f429911..be7415b 100644
@@ -17,7 +17,7 @@
 	for ( zone_id = 0 ; zone_id < MAX_NR_ZONES ; zone_id++ )
 
 #define for_each_lruvec_id(lruvec_id) \
-	for ( lruvec_id = 0 ; lruvec_id < CONFIG_PAGE_LRU_SPLIT ; lruvec_id++ )
+	for ( lruvec_id = 0 ; lruvec_id < lruvec_split ; lruvec_id++ )
 
 #define for_each_zone_and_lruvec_id(zone_id, lruvec_id) \
 	for_each_zone_id(zone_id) for_each_lruvec_id(lruvec_id)
mm/page_alloc.c +29 -0
index 9b0cc92..1a899fa 100644
@@ -4303,6 +4303,35 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)
 #endif
 }
 
+#if CONFIG_PAGE_LRU_SPLIT != 1
+
+unsigned lruvec_split = 1;
+unsigned lruvec_interleaving = CONFIG_PAGE_LRU_INTERLEAVING;
+
+static int __init set_lruvec_split(char *arg)
+{
+	if (!kstrtouint(arg, 0, &lruvec_split) &&
+	    lruvec_split >= 1 &&
+	    lruvec_split <= CONFIG_PAGE_LRU_SPLIT)
+		return 0;
+	lruvec_split = 1;
+	return 1;
+}
+early_param("lruvec_split", set_lruvec_split);
+
+static int __init set_lruvec_interleaving(char *arg)
+{
+	if (!kstrtouint(arg, 0, &lruvec_interleaving) &&
+	    lruvec_interleaving >= HPAGE_PMD_ORDER &&
+	    lruvec_interleaving <= BITS_PER_LONG)
+		return 0;
+	lruvec_split = 1;
+	return 1;
+}
+early_param("lruvec_interleaving", set_lruvec_interleaving);
+
+#endif
+
 /*
  * Set up the zone data structures:
  *   - mark all pages reserved

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#57 lore raw

PATCH 2/2 mm: show zone lruvec state in /proc/zoneinfo

In reply to Andi Kleen
---
 mm/vmstat.c |   23 +++++++++++++++++++++++
 1 files changed, 23 insertions(+), 0 deletions(-)
mm/vmstat.c +23 -0
index 2c813e1..2e77a19 100644
@@ -20,6 +20,8 @@
 #include <linux/writeback.h>
 #include <linux/compaction.h>
 
+#include "internal.h"
+
 #ifdef CONFIG_VM_EVENT_COUNTERS
 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
 EXPORT_PER_CPU_SYMBOL(vm_event_states);
@@ -1020,6 +1022,27 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
 		   "\n  start_pfn:         %lu",
 		   zone->all_unreclaimable,
 		   zone->zone_start_pfn);
+	seq_printf(m, "\n  lruvecs");
+	for_each_lruvec_id(i) {
+		struct lruvec *lruvec = zone->lruvec + i;
+		enum lru_list lru;
+
+		seq_printf(m,
+			   "\n    lruvec: %i",
+			   i);
+		for_each_lru(lru)
+			seq_printf(m,
+			   "\n              %s: %lu",
+			   vmstat_text[NR_LRU_BASE + lru],
+			   lruvec->pages_count[lru]);
+		seq_printf(m,
+			   "\n              %s: %lu"
+			   "\n              %s: %lu",
+			   vmstat_text[NR_ISOLATED_ANON],
+			   lruvec->pages_count[LRU_ISOLATED_ANON],
+			   vmstat_text[NR_ISOLATED_FILE],
+			   lruvec->pages_count[LRU_ISOLATED_FILE]);
+	}
 	seq_putc(m, '\n');
 }
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#58 lore raw
In reply to Konstantin Khlebnikov
Quoted 8 lines · Konstantin Khlebnikov

v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving

Quoted 5 lines

This patchset is based on next-20120210

git: https://github.com/koct9i/linux/commits/lruvec-v3

---

I am seeing an improvement of about 7% in throughput in a workload where I am doing parallel reading of files that are mmaped. The contention on lru_lock used to be 13% in the cpu profile on the __pagevec_lru_add code path. Now lock contention on this path drops to about 0.6%. I have 40 hyper-threaded enabled cpu cores running 80 mmaped file reading processes.

So initial testing of this patch set looks encouraging.

Tim

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#59 lore raw
Quoted 15 lines · Tim Chen
Quoted 8 lines · Konstantin Khlebnikov

v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving

Quoted 5 lines

This patchset is based on next-20120210

git: https://github.com/koct9i/linux/commits/lruvec-v3

---

Quoted 8 lines

I am seeing an improvement of about 7% in throughput in a workload where I am doing parallel reading of files that are mmaped. The contention on lru_lock used to be 13% in the cpu profile on the __pagevec_lru_add code path. Now lock contention on this path drops to about 0.6%. I have 40 hyper-threaded enabled cpu cores running 80 mmaped file reading processes.

So initial testing of this patch set looks encouraging.

That's great!

Quoted 3 lines

Tim

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#60 lore raw
In reply to Konstantin Khlebnikov
Quoted 8 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin Khlebnikov…

v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving

Quoted 4 lines

This patchset is based on next-20120210

git: https://github.com/koct9i/linux/commits/lruvec-v3

I wonder.... I just wonder...if we can split a lruvec in a zone into small pieces of lruvec and have splitted LRU-lock per them, do we need per-memcg-lrulock ?

It seems per-memcg-lrulock can be much bigger lock than small-lruvec-lock. (depends on configuraton) and much more complicated..and have to take care of many things.. If unit of splitting can be specified by boot option, it seems admins can split a big memcg's per-memcg-lru lock into more small pieces.

BTW, how to think of default size of splitting ? I wonder splitting lru into the number of cpus per a node can be a choice. Each cpu may have a chance to set prefered-pfn-range at page allocation with additional patches.

Thanks,
-Kame

Quoted 37 lines

---

Konstantin Khlebnikov (21):
memcg: unify inactive_ratio calculation
memcg: make mm_match_cgroup() hirarchical
memcg: fix page_referencies cgroup filter on global reclaim
memcg: use vm_swappiness from target memory cgroup
mm: rename lruvec->lists into lruvec->pages_lru
mm: lruvec linking functions
mm: add lruvec->pages_count
mm: unify inactive_list_is_low()
mm: add lruvec->reclaim_stat
mm: kill struct mem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec into update_page_reclaim_stat()
mm: push lruvecs from pagevec_lru_move_fn() to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: split zone->lru_lock
mm: zone lru vectors interleaving

 include/linux/huge_mm.h    |    3 
 include/linux/memcontrol.h |   75 ------
 include/linux/mm.h         |   66 +++++
 include/linux/mm_inline.h  |   19 +-
 include/linux/mmzone.h     |   39 ++-
 include/linux/swap.h       |    6 
 mm/Kconfig                 |   16 +
 mm/compaction.c            |   31 +--
 mm/huge_memory.c           |   14 +
 mm/internal.h              |  204 +++++++++++++++++
 mm/ksm.c                   |    2 
Quoted 12 lines
 mm/memcontrol.c            |  343 +++++++++++-----------------
 mm/migrate.c               |    2 
 mm/page_alloc.c            |   70 +-----
 mm/rmap.c                  |    2 
 mm/swap.c                  |  217 ++++++++++--------
 mm/vmscan.c                |  534 ++++++++++++++++++++++++--------------------
 mm/vmstat.c                |    6 
 18 files changed, 932 insertions(+), 717 deletions(-)
Signature · 1 line
Signature

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#61 lore raw
Quoted 11 lines · KAMEZAWA Hiroyuki
Quoted 8 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin…

v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving

Quoted 7 lines
Quoted 4 lines

This patchset is based on next-20120210

git: https://github.com/koct9i/linux/commits/lruvec-v3

I wonder.... I just wonder...if we can split a lruvec in a zone into small pieces of lruvec and have splitted LRU-lock per them, do we need per-memcg-lrulock ?

What per-memcg-lrulock? I don't have it.
last patch splits lruvecs in memcg with the same factor.

Quoted 5 lines

It seems per-memcg-lrulock can be much bigger lock than small-lruvec-lock. (depends on configuraton) and much more complicated..and have to take care of many things.. If unit of splitting can be specified by boot option, it seems admins can split a big memcg's per-memcg-lru lock into more small pieces.

lruvec count per memcg can be arbitrary and changeable if cgroup is empty. This is not in this patch, but it's really easy.

Quoted 4 lines

BTW, how to think of default size of splitting ? I wonder splitting lru into the number of cpus per a node can be a choice. Each cpu may have a chance to set prefered-pfn-range at page allocation with additional patches.

If we rework page to memcg linking and add direct lruvec-id into page->flags, we will able to change lruvec before inserting page to lru.
Thus each cpu will always insert pages into its own lruvec in zone. I have not thought about races yet, but this would be perfect solution.

Quoted 55 lines

Thanks,
-Kame

Quoted 49 lines

---

Konstantin Khlebnikov (21):
memcg: unify inactive_ratio calculation
memcg: make mm_match_cgroup() hirarchical
memcg: fix page_referencies cgroup filter on global reclaim
memcg: use vm_swappiness from target memory cgroup
mm: rename lruvec->lists into lruvec->pages_lru
mm: lruvec linking functions
mm: add lruvec->pages_count
mm: unify inactive_list_is_low()
mm: add lruvec->reclaim_stat
mm: kill struct mem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec into update_page_reclaim_stat()
mm: push lruvecs from pagevec_lru_move_fn() to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: split zone->lru_lock
mm: zone lru vectors interleaving

  include/linux/huge_mm.h    |    3
  include/linux/memcontrol.h |   75 ------
  include/linux/mm.h         |   66 +++++
  include/linux/mm_inline.h  |   19 +-
  include/linux/mmzone.h     |   39 ++-
  include/linux/swap.h       |    6
  mm/Kconfig                 |   16 +
  mm/compaction.c            |   31 +--
  mm/huge_memory.c           |   14 +
  mm/internal.h              |  204 +++++++++++++++++
  mm/ksm.c                   |    2
  mm/memcontrol.c            |  343 +++++++++++-----------------
  mm/migrate.c               |    2
  mm/page_alloc.c            |   70 +-----
  mm/rmap.c                  |    2
  mm/swap.c                  |  217 ++++++++++--------
  mm/vmscan.c                |  534 ++++++++++++++++++++++++--------------------
  mm/vmstat.c                |    6
  18 files changed, 932 insertions(+), 717 deletions(-)

--
Signature

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#62 lore raw
Quoted 14 lines · On Sat, 25 Feb 2012 09:31:01 +0400 Konstantin Khlebnikov…
Quoted 13 lines · KAMEZAWA Hiroyuki
Quoted 10 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin…

v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving

This patchset is based on next-20120210

Quoted 9 lines
Quoted 5 lines
Quoted 2 lines

git: https://github.com/koct9i/linux/commits/lruvec-v3

I wonder.... I just wonder...if we can split a lruvec in a zone into small pieces of lruvec and have splitted LRU-lock per them, do we need per-memcg-lrulock ?

What per-memcg-lrulock? I don't have it.
last patch splits lruvecs in memcg with the same factor.

Okay, I missed it.

Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#63 lore raw
In reply to Konstantin Khlebnikov
Quoted 8 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin Khlebnikov…

v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving

Quoted 4 lines

This patchset is based on next-20120210

git: https://github.com/koct9i/linux/commits/lruvec-v3

I'm sorry I can't have enough review time in these days but the whole series seems good to me.

BTW, how about trying to merge patch 1/21 -> 13or14/21 first ?
This series adds many changes to various place under /mm. So, step-by-step merging will be better I think.
(Just says as this because I tend to split a long series of patch to
small sets of patches and merge them one by one to reduce my own maintainance cost.)

At final lock splitting, performance number should be in changelog.

Thanks,
-Kame

Quoted 37 lines

---

Konstantin Khlebnikov (21):
memcg: unify inactive_ratio calculation
memcg: make mm_match_cgroup() hirarchical
memcg: fix page_referencies cgroup filter on global reclaim
memcg: use vm_swappiness from target memory cgroup
mm: rename lruvec->lists into lruvec->pages_lru
mm: lruvec linking functions
mm: add lruvec->pages_count
mm: unify inactive_list_is_low()
mm: add lruvec->reclaim_stat
mm: kill struct mem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec into update_page_reclaim_stat()
mm: push lruvecs from pagevec_lru_move_fn() to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: split zone->lru_lock
mm: zone lru vectors interleaving

 include/linux/huge_mm.h    |    3 
 include/linux/memcontrol.h |   75 ------
 include/linux/mm.h         |   66 +++++
 include/linux/mm_inline.h  |   19 +-
 include/linux/mmzone.h     |   39 ++-
 include/linux/swap.h       |    6 
 mm/Kconfig                 |   16 +
 mm/compaction.c            |   31 +--
 mm/huge_memory.c           |   14 +
 mm/internal.h              |  204 +++++++++++++++++
 mm/ksm.c                   |    2 
Quoted 19 lines
 mm/memcontrol.c            |  343 +++++++++++-----------------
 mm/migrate.c               |    2 
 mm/page_alloc.c            |   70 +-----
 mm/rmap.c                  |    2 
 mm/swap.c                  |  217 ++++++++++--------
 mm/vmscan.c                |  534 ++++++++++++++++++++++++--------------------
 mm/vmstat.c                |    6 
 18 files changed, 932 insertions(+), 717 deletions(-)
Signature · 8 lines
Signature

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

#64 lore raw
Quoted 19 lines · KAMEZAWA Hiroyuki
Quoted 8 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin…

v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving

Quoted 4 lines

This patchset is based on next-20120210

git: https://github.com/koct9i/linux/commits/lruvec-v3

I'm sorry I can't have enough review time in these days but the whole series seems good to me.

Quoted 5 lines

BTW, how about trying to merge patch 1/21 -> 13or14/21 first ?
This series adds many changes to various place under /mm. So, step-by-step merging will be better I think.
(Just says as this because I tend to split a long series of patch to
small sets of patches and merge them one by one to reduce my own maintainance cost.)

I agree, the amount of required cleanups has exceeded the limit. I'll send them as separate patchset, maybe tomorrow.

Quoted 2 lines

At final lock splitting, performance number should be in changelog.

For sure

Quoted 61 lines

Thanks,
-Kame

Quoted 56 lines

---

Konstantin Khlebnikov (21):
memcg: unify inactive_ratio calculation
memcg: make mm_match_cgroup() hirarchical
memcg: fix page_referencies cgroup filter on global reclaim
memcg: use vm_swappiness from target memory cgroup
mm: rename lruvec->lists into lruvec->pages_lru
mm: lruvec linking functions
mm: add lruvec->pages_count
mm: unify inactive_list_is_low()
mm: add lruvec->reclaim_stat
mm: kill struct mem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec into update_page_reclaim_stat()
mm: push lruvecs from pagevec_lru_move_fn() to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: split zone->lru_lock
mm: zone lru vectors interleaving

  include/linux/huge_mm.h    |    3
  include/linux/memcontrol.h |   75 ------
  include/linux/mm.h         |   66 +++++
  include/linux/mm_inline.h  |   19 +-
  include/linux/mmzone.h     |   39 ++-
  include/linux/swap.h       |    6
  mm/Kconfig                 |   16 +
  mm/compaction.c            |   31 +--
  mm/huge_memory.c           |   14 +
  mm/internal.h              |  204 +++++++++++++++++
  mm/ksm.c                   |    2
  mm/memcontrol.c            |  343 +++++++++++-----------------
  mm/migrate.c               |    2
  mm/page_alloc.c            |   70 +-----
  mm/rmap.c                  |    2
  mm/swap.c                  |  217 ++++++++++--------
  mm/vmscan.c                |  534 ++++++++++++++++++++++++--------------------
  mm/vmstat.c                |    6
  18 files changed, 932 insertions(+), 717 deletions(-)

--
Signature

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email:<a href=mailto:"[email protected]"> [email protected]</a>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>